diff --git a/docker/oap/docker-entrypoint.sh b/docker/oap/docker-entrypoint.sh index 06cca183038f3f30717e75224eab8ea060943a8a..4187ddbc961edf73891743373a970c4272e34afa 100755 --- a/docker/oap/docker-entrypoint.sh +++ b/docker/oap/docker-entrypoint.sh @@ -136,7 +136,15 @@ generateStorageMySQL() { cat <> ${var_application_file} storage: mysql: - metadataQueryMaxSize: \${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: ${SW_DATA_SOURCE_USER:root} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: \${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} EOT } diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md index d2d02616a23e65a10d0c1114fdca3c0f6a8f79b4..c0fc435089e657e040030c8e427396bccc8f149a 100644 --- a/docs/en/setup/backend/backend-storage.md +++ b/docs/en/setup/backend/backend-storage.md @@ -159,11 +159,18 @@ Please download MySQL driver by yourself. Copy the connection driver jar to `oap ```yaml storage: mysql: + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: ${SW_DATA_SOURCE_USER:root} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} ``` - -All connection related settings including link url, username and password -are in `datasource-settings.properties`. -This setting file follow [HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool document. +All connection related settings including link url, username and password are in `application.yml`. +Here are some of the settings, please follow [HikariCP](https://github.com/brettwooldridge/HikariCP) connection pool document for all the settings. ## TiDB Currently tested TiDB in version 2.0.9, and Mysql Client driver in version 8.0.13. @@ -172,10 +179,18 @@ Active TiDB as storage, set storage provider to **mysql**. ```yaml storage: mysql: + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: ${SW_DATA_SOURCE_USER:root} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} ``` - -All connection related settings including link url, username and password -are in `datasource-settings.properties`. And these settings can refer to the configuration of *MySQL* above. +All connection related settings including link url, username and password are in `application.yml`. +These settings can refer to the configuration of *MySQL* above. ## ElasticSearch 5 ElasticSearch 5 is incompatible with ElasticSearch 6 Java client jar, so it could not be included in native distribution. diff --git a/oap-server/server-library/library-module/src/main/java/org/apache/skywalking/oap/server/library/module/ModuleDefine.java b/oap-server/server-library/library-module/src/main/java/org/apache/skywalking/oap/server/library/module/ModuleDefine.java index 0f95a2bc5fde35c699e6494f38b484940a600af1..60982fd7096ffb30f4366157b9698af8c5bb74d0 100644 --- a/oap-server/server-library/library-module/src/main/java/org/apache/skywalking/oap/server/library/module/ModuleDefine.java +++ b/oap-server/server-library/library-module/src/main/java/org/apache/skywalking/oap/server/library/module/ModuleDefine.java @@ -18,12 +18,13 @@ package org.apache.skywalking.oap.server.library.module; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + import java.lang.reflect.Field; import java.util.Enumeration; import java.util.Properties; import java.util.ServiceLoader; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; /** * A module definition. @@ -93,7 +94,7 @@ public abstract class ModuleDefine implements ModuleProviderHolder { } loadedProvider.prepare(); } - + private void copyProperties(ModuleConfig dest, Properties src, String moduleName, String providerName) throws IllegalAccessException { if (dest == null) { @@ -103,7 +104,6 @@ public abstract class ModuleDefine implements ModuleProviderHolder { while (propertyNames.hasMoreElements()) { String propertyName = (String)propertyNames.nextElement(); Class destClass = dest.getClass(); - try { Field field = getDeclaredField(destClass, propertyName); field.setAccessible(true); diff --git a/oap-server/server-starter/pom.xml b/oap-server/server-starter/pom.xml index e46fe5640de7bcae1c0be04cd83e420325a53579..716e828ab1720d5fb6799c5b2a0c5698842400c8 100644 --- a/oap-server/server-starter/pom.xml +++ b/oap-server/server-starter/pom.xml @@ -242,7 +242,6 @@ log4j2.xml alarm-settings.yml component-libraries.yml - datasource-settings.properties endpoint_naming_rules.properties official_analysis.oal gateways.yml diff --git a/oap-server/server-starter/src/main/assembly/application.yml b/oap-server/server-starter/src/main/assembly/application.yml index 9741096878ad1cdd8c43ba3fa1ea837b34151ab0..c18191dba5e8400d32219d3040d3eb8438d687e0 100644 --- a/oap-server/server-starter/src/main/assembly/application.yml +++ b/oap-server/server-starter/src/main/assembly/application.yml @@ -99,7 +99,15 @@ storage: user: ${SW_STORAGE_H2_USER:sa} metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} # mysql: -# metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} +# properties: +# jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} +# dataSource.user: ${SW_DATA_SOURCE_USER:root} +# dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} +# dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} +# dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} +# dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} +# dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} +# metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} receiver-sharing-server: default: receiver-register: diff --git a/oap-server/server-starter/src/main/assembly/assembly.xml b/oap-server/server-starter/src/main/assembly/assembly.xml index 36bf489cc55641fd9253bcdefff7670cb438c0ae..a1c87e9e6882f4f9df574c941383a38eff5f9e5d 100755 --- a/oap-server/server-starter/src/main/assembly/assembly.xml +++ b/oap-server/server-starter/src/main/assembly/assembly.xml @@ -39,7 +39,6 @@ application.yml alarm-settings.yml alarm-settings-sample.yml - datasource-settings.properties gateways.yml diff --git a/oap-server/server-starter/src/main/assembly/datasource-settings.properties b/oap-server/server-starter/src/main/assembly/datasource-settings.properties deleted file mode 100644 index 70b009915e41e042161163a2a6ef877f9f251a7d..0000000000000000000000000000000000000000 --- a/oap-server/server-starter/src/main/assembly/datasource-settings.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -jdbcUrl=jdbc:mysql://localhost:3306/swtest -dataSource.user=root -dataSource.password=root@1234 -dataSource.cachePrepStmts=true -dataSource.prepStmtCacheSize=250 -dataSource.prepStmtCacheSqlLimit=2048 -dataSource.useServerPrepStmts=true -dataSource.useLocalSessionState=true -dataSource.rewriteBatchedStatements=true -dataSource.cacheResultSetMetadata=true -dataSource.cacheServerConfiguration=true -dataSource.elideSetAutoCommits=true -dataSource.maintainTimeStats=false \ No newline at end of file diff --git a/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java b/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java index f8908ea010f6704a0635356b55719272dfdf0e20..64dc7cf0078ca2dd8e9210e31c70eb9fec929396 100644 --- a/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java +++ b/oap-server/server-starter/src/main/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoader.java @@ -18,10 +18,6 @@ package org.apache.skywalking.oap.server.starter.config; -import java.io.FileNotFoundException; -import java.io.Reader; -import java.util.Map; -import java.util.Properties; import org.apache.skywalking.apm.util.PropertyPlaceholderHelper; import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration; import org.apache.skywalking.oap.server.library.util.CollectionUtils; @@ -30,6 +26,11 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.yaml.snakeyaml.Yaml; +import java.io.FileNotFoundException; +import java.io.Reader; +import java.util.Map; +import java.util.Properties; + /** * Initialize collector settings with following sources. Use application.yml as primary setting, and fix missing setting * by default settings in application-default.yml. @@ -61,21 +62,25 @@ public class ApplicationConfigLoader implements ConfigLoader 0) { logger.info("Get a module define from application.yml, module name: {}", moduleName); ApplicationConfiguration.ModuleConfiguration moduleConfiguration = configuration.addModule(moduleName); - providerConfig.forEach((name, propertiesConfig) -> { - logger.info("Get a provider define belong to {} module, provider name: {}", moduleName, name); + providerConfig.forEach((providerName, propertiesConfig) -> { + logger.info("Get a provider define belong to {} module, provider name: {}", moduleName, providerName); Properties properties = new Properties(); if (propertiesConfig != null) { - propertiesConfig.forEach((key, value) -> { - properties.put(key, value); - final Object replaceValue = yaml.load(PropertyPlaceholderHelper.INSTANCE - .replacePlaceholders(value + "", properties)); - if (replaceValue != null) { - properties.replace(key, replaceValue); + propertiesConfig.forEach((propertyName, propertyValue) -> { + if (propertyValue instanceof Map) { + Properties subProperties = new Properties(); + ((Map) propertyValue).forEach((key, value) -> { + subProperties.put(key, value); + replacePropertyAndLog(key, value, subProperties, providerName); + }); + properties.put(propertyName, subProperties); + } else { + properties.put(propertyName, propertyValue); + replacePropertyAndLog(propertyName, propertyValue, properties, providerName); } - logger.info("The property with key: {}, value: {}, in {} provider", key, replaceValue.toString(), name); }); } - moduleConfiguration.addProviderConfiguration(name, properties); + moduleConfiguration.addProviderConfiguration(providerName, properties); }); } else { logger.warn("Get a module define from application.yml, but no provider define, use default, module name: {}", moduleName); @@ -86,6 +91,14 @@ public class ApplicationConfigLoader implements ConfigLoader prop : System.getProperties().entrySet()) { diff --git a/oap-server/server-starter/src/main/resources/application.yml b/oap-server/server-starter/src/main/resources/application.yml index b16976cb70b1c8b0e35e7e70f12f62980195d866..8b0d0fe200d01a63e82d49178d52c3eb10b80599 100755 --- a/oap-server/server-starter/src/main/resources/application.yml +++ b/oap-server/server-starter/src/main/resources/application.yml @@ -98,7 +98,15 @@ storage: # user: ${SW_STORAGE_H2_USER:sa} # metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} # mysql: -# metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} +# properties: +# jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} +# dataSource.user: ${SW_DATA_SOURCE_USER:root} +# dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} +# dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} +# dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} +# dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} +# dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} +# metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} receiver-sharing-server: default: receiver-register: diff --git a/oap-server/server-starter/src/main/resources/datasource-settings.properties b/oap-server/server-starter/src/main/resources/datasource-settings.properties deleted file mode 100755 index 70b009915e41e042161163a2a6ef877f9f251a7d..0000000000000000000000000000000000000000 --- a/oap-server/server-starter/src/main/resources/datasource-settings.properties +++ /dev/null @@ -1,31 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# -# - -jdbcUrl=jdbc:mysql://localhost:3306/swtest -dataSource.user=root -dataSource.password=root@1234 -dataSource.cachePrepStmts=true -dataSource.prepStmtCacheSize=250 -dataSource.prepStmtCacheSqlLimit=2048 -dataSource.useServerPrepStmts=true -dataSource.useLocalSessionState=true -dataSource.rewriteBatchedStatements=true -dataSource.cacheResultSetMetadata=true -dataSource.cacheServerConfiguration=true -dataSource.elideSetAutoCommits=true -dataSource.maintainTimeStats=false \ No newline at end of file diff --git a/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoaderTestCase.java b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoaderTestCase.java new file mode 100644 index 0000000000000000000000000000000000000000..9365db5c74daf6e76d9b4d00e3629ec81638eaee --- /dev/null +++ b/oap-server/server-starter/src/test/java/org/apache/skywalking/oap/server/starter/config/ApplicationConfigLoaderTestCase.java @@ -0,0 +1,52 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.starter.config; + +import org.apache.skywalking.oap.server.library.module.ApplicationConfiguration; +import org.junit.Before; +import org.junit.Test; + +import java.util.Properties; + +import static org.hamcrest.CoreMatchers.instanceOf; +import static org.hamcrest.CoreMatchers.is; +import static org.junit.Assert.assertThat; + +/** + * @author panjuan + */ +public class ApplicationConfigLoaderTestCase { + + private ApplicationConfiguration applicationConfiguration; + + @Before + public void setUp() throws ConfigFileNotFoundException { + ApplicationConfigLoader configLoader = new ApplicationConfigLoader(); + applicationConfiguration = configLoader.load(); + } + + @Test + public void testLoadConfig() { + Properties providerConfig = applicationConfiguration.getModuleConfiguration("storage").getProviderConfiguration("mysql"); + assertThat(providerConfig.get("metadataQueryMaxSize"), is(5000)); + assertThat(providerConfig.get("properties"), instanceOf(Properties.class)); + Properties properties = (Properties) providerConfig.get("properties"); + assertThat(properties.get("jdbcUrl"), is("jdbc:mysql://localhost:3306/swtest")); + } +} diff --git a/oap-server/server-starter/src/test/resources/application.yml b/oap-server/server-starter/src/test/resources/application.yml new file mode 100755 index 0000000000000000000000000000000000000000..20ee234e836e2cb6e1f402da08fc72d190fb78c5 --- /dev/null +++ b/oap-server/server-starter/src/test/resources/application.yml @@ -0,0 +1,179 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster: + standalone: +# Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+ +# library the oap-libs folder with your ZooKeeper 3.4.x library. +# zookeeper: +# nameSpace: ${SW_NAMESPACE:""} +# hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181} +# #Retry Policy +# baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries +# maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry +# # Enable ACL +# enableACL: ${SW_ZK_ENABLE_ACL:false} # disable ACL in default +# schema: ${SW_ZK_SCHEMA:digest} # only support digest schema +# expression: ${SW_ZK_EXPRESSION:skywalking:skywalking} +# kubernetes: +# watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60} +# namespace: ${SW_CLUSTER_K8S_NAMESPACE:default} +# labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking} +# uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID} +# consul: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 +# hostPort: ${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} +# nacos: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# # Nacos Configuration namespace +# namespace: ${SW_CLUSTER_NACOS_NAMESPACE:"public"} +# hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} +# etcd: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# etcd cluster nodes, example: 10.0.0.1:2379,10.0.0.2:2379,10.0.0.3:2379 +# hostPort: ${SW_CLUSTER_ETCD_HOST_PORT:localhost:2379} +core: + default: + # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate + # Receiver: Receive agent data, Level 1 aggregate + # Aggregator: Level 2 aggregate + role: ${SW_CORE_ROLE:Mixed} # Mixed/Receiver/Aggregator + restHost: ${SW_CORE_REST_HOST:0.0.0.0} + restPort: ${SW_CORE_REST_PORT:12800} + restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/} + gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0} + gRPCPort: ${SW_CORE_GRPC_PORT:11800} + downsampling: + - Hour + - Day + - Month + # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. + enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute + recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute + minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute + hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour + dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day + monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month + # Cache metric data for 1 minute to reduce database queries, and if the OAP cluster changes within that minute, + # the metrics may not be accurate within that minute. + enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true} +storage: + mysql: + properties: + jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: ${SW_DATA_SOURCE_USER:root} + dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} +receiver-sharing-server: + default: +receiver-register: + default: +receiver-trace: + default: + bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} + sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. + slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. +receiver-jvm: + default: +receiver-clr: + default: +#receiver-so11y: +# default: +service-mesh: + default: + bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} +istio-telemetry: + default: +envoy-metric: + default: +# alsHTTPAnalysis: ${SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS:k8s-mesh} +#receiver_zipkin: +# default: +# host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0} +# port: ${SW_RECEIVER_ZIPKIN_PORT:9411} +# contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/} +#receiver_jaeger: +# default: +# gRPCHost: ${SW_RECEIVER_JAEGER_HOST:0.0.0.0} +# gRPCPort: ${SW_RECEIVER_JAEGER_PORT:14250} +query: + graphql: + path: ${SW_QUERY_GRAPHQL_PATH:/graphql} +alarm: + default: +telemetry: + none: +# prometheus: +# host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0} +# port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234} +# so11y: +# prometheusExporterEnabled: ${SW_TELEMETRY_SO11Y_PROMETHEUS_ENABLED:true} +# prometheusExporterHost: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0} +# prometheusExporterPort: ${SW_TELEMETRY_PROMETHEUS_PORT:1234} +configuration: + none: +# apollo: +# apolloMeta: http://106.12.25.204:8080 +# apolloCluster: default +# # apolloEnv: # defaults to null +# appId: skywalking +# period: 5 +# nacos: +# # Nacos Server Host +# serverAddr: 127.0.0.1 +# # Nacos Server Port +# port: 8848 +# # Nacos Configuration Group +# group: 'skywalking' +# # Nacos Configuration namespace +# namespace: '' +# # Unit seconds, sync period. Default fetch every 60 seconds. +# period : 5 +# # the name of current cluster, set the name if you want to upstream system known. +# clusterName: "default" +# zookeeper: +# period : 60 # Unit seconds, sync period. Default fetch every 60 seconds. +# nameSpace: /default +# hostPort: localhost:2181 +# #Retry Policy +# baseSleepTimeMs: 1000 # initial amount of time to wait between retries +# maxRetries: 3 # max number of times to retry +# etcd: +# period : 60 # Unit seconds, sync period. Default fetch every 60 seconds. +# group : 'skywalking' +# serverAddr: localhost:2379 +# clusterName: "default" +# consul: +# # Consul host and ports, separated by comma, e.g. 1.2.3.4:8500,2.3.4.5:8500 +# hostAndPorts: ${consul.address} +# # Sync period in seconds. Defaults to 60 seconds. +# period: 1 + +#exporter: +# grpc: +# targetHost: ${SW_EXPORTER_GRPC_HOST:127.0.0.1} +# targetPort: ${SW_EXPORTER_GRPC_PORT:9870} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..92ad903b9750cfaddd14a59bc6918a3953cdcdc3 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageConfig.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; + +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +import java.util.Properties; + +/** + * + * @author panjuan + */ +@Setter +@Getter +public final class MySQLStorageConfig extends ModuleConfig { + + private int metadataQueryMaxSize = 5000; + private Properties properties; +} diff --git a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java index 76c81c73d3ec39451435047a24c984397de44665..e0a1b2880b4ac330233d5c6ca91a37b59ea400d6 100644 --- a/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java +++ b/oap-server/server-storage-plugin/storage-jdbc-hikaricp-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/jdbc/mysql/MySQLStorageProvider.java @@ -18,19 +18,47 @@ package org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql; -import java.io.IOException; -import java.util.Properties; import org.apache.skywalking.oap.server.core.CoreModule; -import org.apache.skywalking.oap.server.core.storage.*; -import org.apache.skywalking.oap.server.core.storage.cache.*; -import org.apache.skywalking.oap.server.core.storage.query.*; +import org.apache.skywalking.oap.server.core.storage.IBatchDAO; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.IRegisterLockDAO; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.cache.IEndpointInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.cache.IServiceInstanceInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.cache.IServiceInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; import org.apache.skywalking.oap.server.core.storage.ttl.GeneralStorageTTL; import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; -import org.apache.skywalking.oap.server.library.module.*; -import org.apache.skywalking.oap.server.library.util.ResourceUtils; -import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.*; -import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.*; -import org.slf4j.*; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2BatchDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2EndpointInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2HistoryDeleteDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetadataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetricsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2NetworkAddressInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2RegisterLockDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2RegisterLockInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ServiceInstanceInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ServiceInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2StorageDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TopNRecordsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TopologyQueryDAO; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; /** * MySQL storage provider should be secondary choice for production usage as SkyWalking storage solution. It enhanced @@ -40,18 +68,18 @@ import org.slf4j.*; * this storage implementation, we could also use this in MySQL-compatible projects, such as, Apache ShardingSphere, * TiDB * - * @author wusheng, peng-yongsheng + * @author wusheng, peng-yongsheng, panjuan */ public class MySQLStorageProvider extends ModuleProvider { private static final Logger logger = LoggerFactory.getLogger(MySQLStorageProvider.class); - private H2StorageConfig config; + private MySQLStorageConfig config; private JDBCHikariCPClient mysqlClient; private H2RegisterLockDAO lockDAO; public MySQLStorageProvider() { - config = new H2StorageConfig(); + config = new MySQLStorageConfig(); } @Override public String name() { @@ -66,15 +94,8 @@ public class MySQLStorageProvider extends ModuleProvider { return config; } - @Override public void prepare() throws ServiceNotProvidedException, ModuleStartException { - Properties settings = new Properties(); - try { - settings.load(ResourceUtils.read("datasource-settings.properties")); - } catch (IOException e) { - throw new ModuleStartException("load datasource setting file failure.", e); - } - - mysqlClient = new JDBCHikariCPClient(settings); + @Override public void prepare() throws ServiceNotProvidedException { + mysqlClient = new JDBCHikariCPClient(config.getProperties()); this.registerServiceImplementation(IBatchDAO.class, new H2BatchDAO(mysqlClient)); this.registerServiceImplementation(StorageDAO.class, new H2StorageDAO(mysqlClient)); @@ -96,7 +117,7 @@ public class MySQLStorageProvider extends ModuleProvider { this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new H2TopNRecordsQueryDAO(mysqlClient)); this.registerServiceImplementation(ILogQueryDAO.class, new MySQLLogQueryDAO(mysqlClient)); } - + @Override public void start() throws ServiceNotProvidedException, ModuleStartException { try { mysqlClient.connect(); @@ -110,7 +131,7 @@ public class MySQLStorageProvider extends ModuleProvider { } } - @Override public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + @Override public void notifyAfterCompleted() throws ServiceNotProvidedException { }