diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 3f68fa915288db8a2d59e2ab4a31306156c17124..6016d83e586f6785da891adc354b4de164d9a1d3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -39,17 +39,19 @@ jobs: ./mvnw --batch-mode -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install ./mvnw --batch-mode -f test/e2e/pom.xml -pl e2e-base clean install - name: Single Node Tests(JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-single-service + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-single-service + - name: Single Node Tests(InfluxDB/JDK8) + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-influxdb - name: Single Node Tests(MySQL/JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-mysql + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-mysql - name: Single Node Tests(JDK9) - run: export E2E_VERSION=jdk9-1.3 && bash -x test/e2e/run.sh e2e-single-service + run: export E2E_VERSION=jdk9-1.5 && bash -x test/e2e/run.sh e2e-single-service - name: Single Node Tests(JDK11) - run: export E2E_VERSION=jdk11-1.3 && bash -x test/e2e/run.sh e2e-single-service + run: export E2E_VERSION=jdk11-1.5 && bash -x test/e2e/run.sh e2e-single-service - name: Single Node Tests(JDK12) - run: export E2E_VERSION=jdk12-1.3 && bash -x test/e2e/run.sh e2e-single-service + run: export E2E_VERSION=jdk12-1.5 && bash -x test/e2e/run.sh e2e-single-service - name: Agent Reboot Tests(JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-agent-reboot + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-agent-reboot Cluster: runs-on: ubuntu-latest @@ -71,15 +73,19 @@ jobs: ./mvnw --batch-mode -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install ./mvnw --batch-mode -f test/e2e/pom.xml -pl e2e-base clean install - name: Cluster Tests (ES6/ZK/JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner --storage=elasticsearch + - name: Cluster Tests (InfluxDB/ZK/JDK8) + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner --storage=influxdb - name: Cluster With Gateway Tests (ES6/ZK/JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner - name: Cluster Tests (ES7/ZK/JDK8) - run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner + run: export E2E_VERSION=jdk8-1.5 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-cluster/e2e-cluster-test-runner --storage=elasticsearch - name: TTL ES Tests(JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es - name: TTL ES7 Tests(JDK8) - run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es + run: export E2E_VERSION=jdk8-1.5 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-es + - name: TTL InfluxDB Tests(JDK8) + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-ttl/e2e-ttl-influxdb Compatibilities: runs-on: ubuntu-latest @@ -95,7 +101,7 @@ jobs: ./mvnw --batch-mode -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install ./mvnw --batch-mode -f test/e2e/pom.xml -pl e2e-base clean install - name: 6.x Agents & 7.x Backend - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-6.x-agent-7.x-oap-compatibility + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-6.x-agent-7.x-oap-compatibility Profile: runs-on: ubuntu-latest @@ -111,10 +117,12 @@ jobs: ./mvnw --batch-mode -Dcheckstyle.skip -Drat.skip -T2 -Dmaven.compile.fork -Dmaven.compiler.maxmem=3072 -DskipTests clean install ./mvnw --batch-mode -f test/e2e/pom.xml -pl e2e-base clean install - name: Profile Tests H2(JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=h2 + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=h2 - name: Profile Tests MySQL(JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=mysql + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=mysql - name: Profile Tests ES6(JDK8) - run: export E2E_VERSION=jdk8-1.3 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=elasticsearch + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=elasticsearch + - name: Profile Tests InfluxDB(JDK8) + run: export E2E_VERSION=jdk8-1.5 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=influxdb - name: Profile Tests ES7(JDK8) - run: export E2E_VERSION=jdk8-1.3 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=elasticsearch + run: export E2E_VERSION=jdk8-1.5 DIST_PACKAGE=apache-skywalking-apm-bin-es7.tar.gz ES_VERSION=7.4.2 && bash -x test/e2e/run.sh e2e-profile/e2e-profile-test-runner --storage=elasticsearch diff --git a/dist-material/application.yml b/dist-material/application.yml index 39f9ef683cf3fd5842a12f02e0754e6e787879d0..8012e410576dfddda4164bba608cd1a65ac9fb88 100644 --- a/dist-material/application.yml +++ b/dist-material/application.yml @@ -139,6 +139,31 @@ storage: # dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} # dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} # metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} +# influx: +# # Metadata storage provider configuration +# metabaseType: ${SW_STORAGE_METABASE_TYPE:H2} # There are 2 options as Metabase provider, H2 or MySQL. +# h2Props: +# dataSourceClassName: ${SW_STORAGE_METABASE_DRIVER:org.h2.jdbcx.JdbcDataSource} +# dataSource.url: ${SW_STORAGE_METABASE_URL:jdbc:h2:mem:skywalking-oap-db} +# dataSource.user: ${SW_STORAGE_METABASE_USER:sa} +# dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:} +# mysqlProps: +# jdbcUrl: ${SW_STORAGE_METABASE_URL:"jdbc:mysql://localhost:3306/swtest"} +# dataSource.user: ${SW_STORAGE_METABASE_USER:root} +# dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:root@1234} +# dataSource.cachePrepStmts: ${SW_STORAGE_METABASE_CACHE_PREP_STMTS:true} +# dataSource.prepStmtCacheSize: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_SIZE:250} +# dataSource.prepStmtCacheSqlLimit: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_LIMIT:2048} +# dataSource.useServerPrepStmts: ${SW_STORAGE_METABASE_USE_SERVER_PREP_STMTS:true} +# metadataQueryMaxSize: ${SW_STORAGE_METABASE_QUERY_MAX_SIZE:5000} +# # InfluxDB configuration +# url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} +# user: ${SW_STORAGE_INFLUXDB_USER:root} +# password: ${SW_STORAGE_INFLUXDB_PASSWORD:} +# database: ${SW_STORAGE_INFLUXDB_DATABASE:skywalking} +# actions: ${SW_STORAGE_INFLUXDB_ACTIONS:1000} # the number of actions to collect +# duration: ${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) +# fetchTaskLogMaxSize: ${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request receiver-sharing-server: default: receiver-register: diff --git a/dist-material/release-docs/LICENSE b/dist-material/release-docs/LICENSE index 103234af05c9ec99d5e8a3033c7525c3411d39d8..b72e0bf55f0c0fe85d1a9b2e64529421eccf5723 100755 --- a/dist-material/release-docs/LICENSE +++ b/dist-material/release-docs/LICENSE @@ -323,7 +323,10 @@ The text of each license is the standard Apache 2.0 license. etcd4j 2.17.0: https://github.com/jurmous/etcd4j Apache 2.0 javaassist 3.25.0-GA: https://github.com/jboss-javassist/javassist Apache 2.0 jackson-module-afterburner 2.9.5: https://github.com/FasterXML/jackson-modules-base, Apache 2.0 - perfmark-api 0.19.0: https://github.com/perfmark/perfmark + perfmark-api 0.19.0: https://github.com/perfmark/perfmark, Apache 2.0 + moshi 1.5.0: https://github.com/square/moshi, Apache 2.0 + logging-interceptor 3.13.1: https://github.com/square/okhttp/tree/master/okhttp-logging-interceptor, Apache 2.0 + msgpack-core 0.8.16: https://github.com/msgpack/msgpack-java, Apache 2.0 ======================================================================== MIT licenses @@ -341,6 +344,7 @@ The text of each license is also included at licenses/LICENSE-[project].txt. bcprov-jdk15on 1.55: http://www.bouncycastle.org/licence.html , MIT minimal-json 0.9.5: https://github.com/ralfstx/minimal-json, MIT checker-qual 2.8.1: https://github.com/typetools/checker-framework, MIT + influxdb-java 2.15: https://github.com/influxdata/influxdb-java, MIT ======================================================================== MIT licenses diff --git a/dist-material/release-docs/NOTICE b/dist-material/release-docs/NOTICE index e32065b7d26b4c294390a782123edc072cc217e1..9ea8d9113fa22fe04bdbb020d2a66d1ff049022f 100755 --- a/dist-material/release-docs/NOTICE +++ b/dist-material/release-docs/NOTICE @@ -905,4 +905,13 @@ be obtained at: * HOMEPAGE: * https://github.com/catapult-project/catapult ------- \ No newline at end of file +------ + +=========================================================================== +MessagePackage Notice +=========================================================================== + +This product includes the software developed by third-party: + + * Google Guava https://code.google.com/p/guava-libraries/ (APL2) + * sbt-extras: https://github.com/paulp/sbt-extras (BSD) (LICENSE.sbt-extras.txt) \ No newline at end of file diff --git a/dist-material/release-docs/licenses/LICENSE-influxdb-java.txt b/dist-material/release-docs/licenses/LICENSE-influxdb-java.txt new file mode 100644 index 0000000000000000000000000000000000000000..4be564520aad001990daa55574b3c3d101ca4a8f --- /dev/null +++ b/dist-material/release-docs/licenses/LICENSE-influxdb-java.txt @@ -0,0 +1,22 @@ + +The MIT License (MIT) + +Copyright (c) 2014-2017 Stefan Majer + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/docker/oap-es7/docker-entrypoint.sh b/docker/oap-es7/docker-entrypoint.sh index b1ab944354a47007e212237ca09c89383ebc05c6..54f508a9ef9f26c8689fc589a9ef4204977afe42 100755 --- a/docker/oap-es7/docker-entrypoint.sh +++ b/docker/oap-es7/docker-entrypoint.sh @@ -149,6 +149,37 @@ storage: EOT } +generateStorageInfluxDB() { + cat <> ${var_application_file} +storage: + influx: + # Metadata storage provider configuration + metabaseType: \${SW_STORAGE_METABASE_TYPE:H2} # There are 2 options as Metabase provider, H2 or MySQL. + h2Props: + dataSourceClassName: \${SW_STORAGE_METABASE_DRIVER:org.h2.jdbcx.JdbcDataSource} + dataSource.url: \${SW_STORAGE_METABASE_URL:jdbc:h2:mem:skywalking-oap-db} + dataSource.user: \${SW_STORAGE_METABASE_USER:sa} + dataSource.password: \${SW_STORAGE_METABASE_PASSWORD:} + mysqlProps: + jdbcUrl: \${SW_STORAGE_METABASE_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: \${SW_STORAGE_METABASE_USER:root} + dataSource.password: \${SW_STORAGE_METABASE_PASSWORD:root@1234} + dataSource.cachePrepStmts: \${SW_STORAGE_METABASE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: \${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: \${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: \${SW_STORAGE_METABASE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: \${SW_STORAGE_METABASE_QUERY_MAX_SIZE:5000} + # InfluxDB configuration + url: \${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} + user: \${SW_STORAGE_INFLUXDB_USER:root} + password: \${SW_STORAGE_INFLUXDB_PASSWORD:} + database: \${SW_STORAGE_INFLUXDB_DATABASE:skywalking} + actions: \${SW_STORAGE_INFLUXDB_ACTIONS:1000} # the number of actions to collect + duration: \${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) + fetchTaskLogMaxSize: \${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request +EOT +} + generateConfigurationNone() { cat <> ${var_application_file} configuration: diff --git a/docker/oap/docker-entrypoint.sh b/docker/oap/docker-entrypoint.sh index 7b61db5d7232c82e308958d32464877a4e7e275e..8e469014ee1c831338bcb0d6f1fb7c58b05cdeb7 100755 --- a/docker/oap/docker-entrypoint.sh +++ b/docker/oap/docker-entrypoint.sh @@ -150,6 +150,37 @@ storage: EOT } +generateStorageInfluxDB() { + cat <> ${var_application_file} +storage: + influx: + # Metadata storage provider configuration + metabaseType: \${SW_STORAGE_METABASE_TYPE:H2} # There are 2 options as Metabase provider, H2 or MySQL. + h2Props: + dataSourceClassName: \${SW_STORAGE_METABASE_DRIVER:org.h2.jdbcx.JdbcDataSource} + dataSource.url: \${SW_STORAGE_METABASE_URL:jdbc:h2:mem:skywalking-oap-db} + dataSource.user: \${SW_STORAGE_METABASE_USER:sa} + dataSource.password: \${SW_STORAGE_METABASE_PASSWORD:} + mysqlProps: + jdbcUrl: \${SW_STORAGE_METABASE_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: \${SW_STORAGE_METABASE_USER:root} + dataSource.password: \${SW_STORAGE_METABASE_PASSWORD:root@1234} + dataSource.cachePrepStmts: \${SW_STORAGE_METABASE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: \${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: \${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: \${SW_STORAGE_METABASE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: \${SW_STORAGE_METABASE_QUERY_MAX_SIZE:5000} + # InfluxDB configuration + url: \${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} + user: \${SW_STORAGE_INFLUXDB_USER:root} + password: \${SW_STORAGE_INFLUXDB_PASSWORD:} + database: \${SW_STORAGE_INFLUXDB_DATABASE:skywalking} + actions: \${SW_STORAGE_INFLUXDB_ACTIONS:1000} # the number of actions to collect + duration: \${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) + fetchTaskLogMaxSize: \${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request +EOT +} + generateConfigurationNone() { cat <> ${var_application_file} configuration: diff --git a/docs/en/setup/backend/backend-storage.md b/docs/en/setup/backend/backend-storage.md index ee1adf83cd0fff096134f606dafc53bca519fef4..72169ba9a4bec9ee1de0c4499383bd0613869bb5 100644 --- a/docs/en/setup/backend/backend-storage.md +++ b/docs/en/setup/backend/backend-storage.md @@ -7,6 +7,7 @@ Native supported storage - ElasticSearch 6, 7 - MySQL - TiDB +- InfluxDB Redistribution version with supported storage. - ElasticSearch 5 @@ -249,6 +250,39 @@ storage: All connection related settings including link url, username and password are in `application.yml`. These settings can refer to the configuration of *MySQL* above. +## InfluxDB +InfluxDB as storage since SkyWalking 7.0. It depends on `H2/MySQL` storage-plugin to store `metadata` like `Inventory` and `ProfileTask`. So, when we set `InfluxDB` as storage provider. We need to configure properties of InfluxDB and Metabase. + +```yaml +storage: + influx: + # Metadata storage provider configuration + metabaseType: ${SW_STORAGE_METABASE_TYPE:H2} # There are 2 options as Metabase provider, H2 or MySQL. + h2Props: + dataSourceClassName: ${SW_STORAGE_METABASE_DRIVER:org.h2.jdbcx.JdbcDataSource} + dataSource.url: ${SW_STORAGE_METABASE_URL:jdbc:h2:mem:skywalking-oap-db} + dataSource.user: ${SW_STORAGE_METABASE_USER:sa} + dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:} + mysqlProps: + jdbcUrl: ${SW_STORAGE_METABASE_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: ${SW_STORAGE_METABASE_USER:root} + dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_STORAGE_METABASE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_STORAGE_METABASE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_METABASE_QUERY_MAX_SIZE:5000} + # InfluxDB configuration + url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} + user: ${SW_STORAGE_INFLUXDB_USER:root} + password: ${SW_STORAGE_INFLUXDB_PASSWORD:} + database: ${SW_STORAGE_INFLUXDB_DATABASE:skywalking} + actions: ${SW_STORAGE_INFLUXDB_ACTIONS:1000} # the number of actions to collect + duration: ${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) + fetchTaskLogMaxSize: ${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request +``` +All connection related settings including link url, username and password are in `application.yml`. The Metadata storage provider settings can refer to the configuration of **H2/MySQL** above. + ## ElasticSearch 5 ElasticSearch 5 is incompatible with ElasticSearch 6 Java client jar, so it could not be included in native distribution. [OpenSkyWalking/SkyWalking-With-Es5x-Storage](https://github.com/OpenSkywalking/SkyWalking-With-Es5x-Storage) repo includes the distribution version. diff --git a/oap-server/server-bootstrap/pom.xml b/oap-server/server-bootstrap/pom.xml index a0c8a69d2da279291dcb5819cb1eea0f91f00144..4aa47c9132a61c900bdccf75cfa399b3e4705b92 100644 --- a/oap-server/server-bootstrap/pom.xml +++ b/oap-server/server-bootstrap/pom.xml @@ -130,6 +130,11 @@ storage-jdbc-hikaricp-plugin ${project.version} + + org.apache.skywalking + storage-influxdb-plugin + ${project.version} + diff --git a/oap-server/server-bootstrap/src/main/resources/application.yml b/oap-server/server-bootstrap/src/main/resources/application.yml index e7f63ac603bfcbeeefab52f918d7da956827d493..a97537b399ab5672a89464f77b844e1963322072 100755 --- a/oap-server/server-bootstrap/src/main/resources/application.yml +++ b/oap-server/server-bootstrap/src/main/resources/application.yml @@ -139,6 +139,31 @@ storage: # dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} # dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} # metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} +# influx: +# # Metadata storage provider configuration +# metabaseType: ${SW_STORAGE_METABASE_TYPE:H2} # There are 2 options as Metabase provider, H2 or MySQL. +# h2Props: +# dataSourceClassName: ${SW_STORAGE_METABASE_DRIVER:org.h2.jdbcx.JdbcDataSource} +# dataSource.url: ${SW_STORAGE_METABASE_URL:jdbc:h2:mem:skywalking-oap-db} +# dataSource.user: ${SW_STORAGE_METABASE_USER:sa} +# dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:} +# mysqlProps: +# jdbcUrl: ${SW_STORAGE_METABASE_URL:"jdbc:mysql://localhost:3306/swtest"} +# dataSource.user: ${SW_STORAGE_METABASE_USER:root} +# dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:root@1234} +# dataSource.cachePrepStmts: ${SW_STORAGE_METABASE_CACHE_PREP_STMTS:true} +# dataSource.prepStmtCacheSize: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_SIZE:250} +# dataSource.prepStmtCacheSqlLimit: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_LIMIT:2048} +# dataSource.useServerPrepStmts: ${SW_STORAGE_METABASE_USE_SERVER_PREP_STMTS:true} +# metadataQueryMaxSize: ${SW_STORAGE_METABASE_QUERY_MAX_SIZE:5000} +# # InfluxDB configuration +# url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} +# user: ${SW_STORAGE_INFLUXDB_USER:root} +# password: ${SW_STORAGE_INFLUXDB_PASSWORD:} +# database: ${SW_STORAGE_INFLUXDB_DATABASE:skywalking} +# actions: ${SW_STORAGE_INFLUXDB_ACTIONS:1000} # the number of actions to collect +# duration: ${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) +# fetchTaskLogMaxSize: ${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request receiver-sharing-server: default: authentication: ${SW_AUTHENTICATION:""} diff --git a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java index c5a43939b285313567f6739b684a5dfc1787d0d1..4948c1de70c130786cf520ac7653954827484ddf 100644 --- a/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java +++ b/oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/TimeBucket.java @@ -32,10 +32,119 @@ public class TimeBucket { return getTimeBucket(time, Downsampling.Second); } + /** + * Record time bucket format in Minute Unit. + * + * @param time Timestamp + * @return time in minute format. + */ public static long getMinuteTimeBucket(long time) { return getTimeBucket(time, Downsampling.Minute); } + /** + * Convert TimeBucket to Timestamp in millisecond. + * + * @param timeBucket long + * @return timestamp in millisecond unit + */ + public static long getTimestamp(long timeBucket) { + if (isSecondBucket(timeBucket)) { + return getTimestamp(timeBucket, Downsampling.Second); + } else if (isMinuteBucket(timeBucket)) { + return getTimestamp(timeBucket, Downsampling.Minute); + } else if (isHourBucket(timeBucket)) { + return getTimestamp(timeBucket, Downsampling.Hour); + } else if (isDayBucket(timeBucket)) { + return getTimestamp(timeBucket, Downsampling.Day); + } else if (isMonthBucket(timeBucket)) { + return getTimestamp(timeBucket, Downsampling.Month); + } else { + throw new UnexpectedException("Unknown downsampling value."); + } + } + + /** + * The format of timeBucket in minute Unit is "yyyyMMddHHmmss", so which means the TimeBucket must be between + * 10000000000000 and 99999999999999. + */ + public static boolean isSecondBucket(long timeBucket) { + return timeBucket < 99999999999999L && timeBucket > 10000000000000L; + } + + /** + * The format of timeBucket in minute Unit is "yyyyMMddHHmm", so which means the TimeBucket must be between + * 100000000000 and 999999999999. + */ + public static boolean isMinuteBucket(long timeBucket) { + return timeBucket < 999999999999L && timeBucket > 100000000000L; + } + + /** + * The format of timeBucket in hour Unit is "yyyyMMddHH", so which means the TimeBucket must be between 1000000000 and + * 9999999999. + */ + public static boolean isHourBucket(long timeBucket) { + return timeBucket < 9999999999L && timeBucket > 1000000000L; + } + + /** + * The format of timeBucket in day Unit is "yyyyMMdd", so which means the TimeBucket must be between 10000000 and + * 99999999. + */ + public static boolean isDayBucket(long timeBucket) { + return timeBucket < 99999999L && timeBucket > 10000000L; + } + + /** + * The format of timeBucket in month Unit is "yyyyMM", so which means the TimeBucket must be between 100000 and + * 999999. + */ + public static boolean isMonthBucket(long timeBucket) { + return timeBucket < 999999L && timeBucket > 100000L; + } + + /** + * Convert TimeBucket to Timestamp in millisecond. + * + * @param timeBucket long + * @param downsampling Downsampling + * @return timestamp in millisecond unit + */ + public static long getTimestamp(long timeBucket, Downsampling downsampling) { + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(0); + switch (downsampling) { + case Second: + calendar.set(Calendar.SECOND, (int) (timeBucket % 100)); + timeBucket /= 100; + case Minute: + calendar.set(Calendar.MINUTE, (int) (timeBucket % 100)); + timeBucket /= 100; + case Hour: + calendar.set(Calendar.HOUR_OF_DAY, (int) (timeBucket % 100)); + timeBucket /= 100; + case Day: + calendar.set(Calendar.DAY_OF_MONTH, (int) (timeBucket % 100)); + timeBucket /= 100; + case Month: + calendar.set(Calendar.MONTH, (int) (timeBucket % 100) - 1); + calendar.set(Calendar.YEAR, (int) (timeBucket / 100)); + break; + default: + throw new UnexpectedException("Unknown downsampling value."); + } + + return calendar.getTimeInMillis(); + } + + /** + * Record time bucket format in Downsampling Unit. + * + * @param time Timestamp + * @param downsampling Downsampling + * @return time in downsampling format. + */ public static long getTimeBucket(long time, Downsampling downsampling) { Calendar calendar = Calendar.getInstance(); calendar.setTimeInMillis(time); diff --git a/oap-server/server-storage-plugin/pom.xml b/oap-server/server-storage-plugin/pom.xml index 5826106469e6fab8712c41f29f304f8c5c388037..a034b3634d421fa6be00989c945b6ec5c4a8e3f2 100644 --- a/oap-server/server-storage-plugin/pom.xml +++ b/oap-server/server-storage-plugin/pom.xml @@ -34,6 +34,7 @@ storage-elasticsearch7-plugin storage-zipkin-plugin storage-jaeger-plugin + storage-influxdb-plugin \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/pom.xml b/oap-server/server-storage-plugin/storage-influxdb-plugin/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..985926ce8568237e1cf77d7a9db2c49f6eb8cde2 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/pom.xml @@ -0,0 +1,48 @@ + + + + + + server-storage-plugin + org.apache.skywalking + 7.0.0-SNAPSHOT + + 4.0.0 + + storage-influxdb-plugin + jar + + + 2.15 + + + + + org.apache.skywalking + storage-jdbc-hikaricp-plugin + ${project.version} + + + + org.influxdb + influxdb-java + ${influxdb-jave.version} + + + \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java new file mode 100644 index 0000000000000000000000000000000000000000..495a453518a1c4f9bd81cddf6e4291db46078dde --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxClient.java @@ -0,0 +1,191 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.TimeUnit; +import lombok.extern.slf4j.Slf4j; +import okhttp3.OkHttpClient; +import org.apache.skywalking.oap.server.core.analysis.Downsampling; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.library.client.Client; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.influxdb.InfluxDB; +import org.influxdb.InfluxDBFactory; +import org.influxdb.dto.BatchPoints; +import org.influxdb.dto.Point; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.time.TimeInterval; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.ti; + +/** + * InfluxDB connection maintainer, provides base data write/query API. + */ +@Slf4j +public class InfluxClient implements Client { + private InfluxStorageConfig config; + private InfluxDB influx; + + /** + * A constant, the name of time field in Time-series database. + */ + public static final String TIME = "time"; + /** + * A constant, the name of tag of time_bucket. + */ + public static final String TAG_TIME_BUCKET = "_time_bucket"; + + private final String database; + + public InfluxClient(InfluxStorageConfig config) { + this.config = config; + this.database = config.getDatabase(); + } + + public final String getDatabase() { + return database; + } + + @Override + public void connect() { + influx = InfluxDBFactory.connect(config.getUrl(), config.getUser(), config.getPassword(), + new OkHttpClient.Builder().readTimeout(3, TimeUnit.MINUTES) + .writeTimeout(3, TimeUnit.MINUTES), + InfluxDB.ResponseFormat.MSGPACK + ); + influx.query(new Query("CREATE DATABASE " + database)); + + influx.enableBatch(config.getActions(), config.getDuration(), TimeUnit.MILLISECONDS); + influx.setDatabase(database); + } + + /** + * To get a connection of InfluxDB. + * + * @return InfluxDB's connection + */ + private InfluxDB getInflux() { + return influx; + } + + /** + * Execute a query against InfluxDB and return a set of {@link QueryResult.Result}s. Normally, InfluxDB supports + * combining multiple statements into one query, so that we do get multi-results. + * + * @throws IOException if there is an error on the InfluxDB server or communication error. + */ + public List query(Query query) throws IOException { + if (log.isDebugEnabled()) { + log.debug("SQL Statement: {}", query.getCommand()); + } + + try { + QueryResult result = getInflux().query(query); + if (result.hasError()) { + throw new IOException(result.getError()); + } + return result.getResults(); + } catch (Exception e) { + throw new IOException(e.getMessage() + System.lineSeparator() + "SQL Statement: " + query.getCommand(), e); + } + } + + /** + * Execute a query against InfluxDB with a single statement. + * + * @throws IOException if there is an error on the InfluxDB server or communication error + */ + public List queryForSeries(Query query) throws IOException { + List results = query(query); + + if (CollectionUtils.isEmpty(results)) { + return null; + } + return results.get(0).getSeries(); + } + + /** + * Execute a query against InfluxDB with a single statement but return a single {@link QueryResult.Series}. + * + * @throws IOException if there is an error on the InfluxDB server or communication error + */ + public QueryResult.Series queryForSingleSeries(Query query) throws IOException { + List series = queryForSeries(query); + if (CollectionUtils.isEmpty(series)) { + return null; + } + return series.get(0); + } + + /** + * Data management, to drop a time-series by measurement and time-series name specified. If an exception isn't + * thrown, it means execution success. Notice, drop series don't support to drop series by range + * + * @throws IOException if there is an error on the InfluxDB server or communication error + */ + public void dropSeries(String measurement, long timeBucket) throws IOException { + Query query = new Query("DROP SERIES FROM " + measurement + " WHERE time_bucket='" + timeBucket + "'"); + QueryResult result = getInflux().query(query); + + if (result.hasError()) { + throw new IOException("Statement: " + query.getCommand() + ", ErrorMsg: " + result.getError()); + } + } + + public void deleteByQuery(String measurement, long timestamp) throws IOException { + this.query(new Query("delete from " + measurement + " where time < " + timestamp + "ms")); + } + + /** + * Write a {@link Point} into InfluxDB. Note that, the {@link Point} is written into buffer of InfluxDB Client and + * wait for buffer flushing. + */ + public void write(Point point) { + getInflux().write(point); + } + + /** + * A batch operation of write. {@link Point}s flush directly. + */ + public void write(BatchPoints points) { + getInflux().write(points); + } + + @Override + public void shutdown() throws IOException { + influx.close(); + } + + /** + * Convert to InfluxDB {@link TimeInterval}. + */ + public static TimeInterval timeInterval(long timeBucket, Downsampling downsampling) { + return ti(TimeBucket.getTimestamp(timeBucket, downsampling), "ms"); + } + + /** + * Convert to InfluxDB {@link TimeInterval}. + */ + public static TimeInterval timeInterval(long timeBucket) { + return ti(TimeBucket.getTimestamp(timeBucket), "ms"); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxModelConstants.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxModelConstants.java new file mode 100644 index 0000000000000000000000000000000000000000..ec54fec83b0d11318669c7b91d4f9205f265f197 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxModelConstants.java @@ -0,0 +1,26 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb; + +public interface InfluxModelConstants { + /** + * Override column because the 'duration' is a identifier of InfluxDB. + */ + String DURATION = "dur"; +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageConfig.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageConfig.java new file mode 100644 index 0000000000000000000000000000000000000000..fe5e847dc9418d21c8a640c266f548a4aaf811b9 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageConfig.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb; + +import java.util.Properties; +import lombok.Getter; +import lombok.Setter; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; + +@Setter +@Getter +public class InfluxStorageConfig extends ModuleConfig { + private String metabaseType; + private Properties h2Props; + private Properties mysqlProps; + + private int metadataQueryMaxSize = 5000; + + private String url; + private String user; + private String password; + private String database; + + private int actions; + private int duration; + + private int fetchTaskLogMaxSize = 5000; +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java new file mode 100644 index 0000000000000000000000000000000000000000..1a95ee3057db4a20a2194301bde696f616de5f92 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/InfluxStorageProvider.java @@ -0,0 +1,174 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb; + +import java.util.Properties; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.storage.IBatchDAO; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.IRegisterLockDAO; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.StorageModule; +import org.apache.skywalking.oap.server.core.storage.cache.IEndpointInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.cache.INetworkAddressInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.cache.IServiceInstanceInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.cache.IServiceInventoryCacheDAO; +import org.apache.skywalking.oap.server.core.storage.model.ModelInstaller; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IMetadataQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; +import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; +import org.apache.skywalking.oap.server.core.storage.ttl.GeneralStorageTTL; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.library.module.ModuleConfig; +import org.apache.skywalking.oap.server.library.module.ModuleDefine; +import org.apache.skywalking.oap.server.library.module.ModuleProvider; +import org.apache.skywalking.oap.server.library.module.ModuleStartException; +import org.apache.skywalking.oap.server.library.module.ServiceNotProvidedException; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.BatchDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.HistoryDeleteDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.InfluxStorageDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.installer.H2Installer; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.installer.MySQLInstaller; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.AggregationQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.AlarmQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.LogQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.MetricsQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.ProfileTaskLogQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.ProfileTaskQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.ProfileThreadSnapshotQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.TopNRecordsQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.TopologyQuery; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.query.TraceQuery; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2EndpointInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2MetadataQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2NetworkAddressInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2RegisterLockDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2RegisterLockInstaller; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ServiceInstanceInventoryCacheDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2ServiceInventoryCacheDAO; + +@Slf4j +public class InfluxStorageProvider extends ModuleProvider { + private InfluxStorageConfig config; + private JDBCHikariCPClient client; + private InfluxClient influxClient; + private H2RegisterLockDAO lockDAO; + + public InfluxStorageProvider() { + config = new InfluxStorageConfig(); + } + + @Override + public String name() { + return "influx"; + } + + @Override + public Class module() { + return StorageModule.class; + } + + @Override + public ModuleConfig createConfigBeanIfAbsent() { + return config; + } + + @Override + public void prepare() throws ServiceNotProvidedException { + + Properties settings; + if ("mysql".equalsIgnoreCase(config.getMetabaseType())) { + settings = config.getMysqlProps(); + } else { + settings = config.getH2Props(); + } + client = new JDBCHikariCPClient(settings); + influxClient = new InfluxClient(config); + + this.registerServiceImplementation(IBatchDAO.class, new BatchDAO(influxClient)); + this.registerServiceImplementation(StorageDAO.class, new InfluxStorageDAO(client, influxClient)); + + this.lockDAO = new H2RegisterLockDAO(client); + this.registerServiceImplementation(IRegisterLockDAO.class, new H2RegisterLockDAO(client)); + this.registerServiceImplementation(IServiceInventoryCacheDAO.class, new H2ServiceInventoryCacheDAO(client)); + this.registerServiceImplementation( + IServiceInstanceInventoryCacheDAO.class, new H2ServiceInstanceInventoryCacheDAO(client)); + this.registerServiceImplementation(IEndpointInventoryCacheDAO.class, new H2EndpointInventoryCacheDAO(client)); + this.registerServiceImplementation( + INetworkAddressInventoryCacheDAO.class, new H2NetworkAddressInventoryCacheDAO(client)); + this.registerServiceImplementation( + IMetadataQueryDAO.class, new H2MetadataQueryDAO(client, config.getMetadataQueryMaxSize())); + + this.registerServiceImplementation(ITopologyQueryDAO.class, new TopologyQuery(influxClient)); + this.registerServiceImplementation(IMetricsQueryDAO.class, new MetricsQuery(influxClient)); + this.registerServiceImplementation(ITraceQueryDAO.class, new TraceQuery(influxClient)); + this.registerServiceImplementation(IAggregationQueryDAO.class, new AggregationQuery(influxClient)); + this.registerServiceImplementation(IAlarmQueryDAO.class, new AlarmQuery(influxClient)); + this.registerServiceImplementation(ITopNRecordsQueryDAO.class, new TopNRecordsQuery(influxClient)); + this.registerServiceImplementation(ILogQueryDAO.class, new LogQuery(influxClient)); + + this.registerServiceImplementation(IProfileTaskQueryDAO.class, new ProfileTaskQuery(influxClient)); + this.registerServiceImplementation( + IProfileThreadSnapshotQueryDAO.class, new ProfileThreadSnapshotQuery(influxClient)); + this.registerServiceImplementation( + IProfileTaskLogQueryDAO.class, new ProfileTaskLogQuery(influxClient, config.getFetchTaskLogMaxSize())); + + this.registerServiceImplementation( + IHistoryDeleteDAO.class, new HistoryDeleteDAO(getManager(), influxClient, new GeneralStorageTTL())); + } + + @Override + public void start() throws ServiceNotProvidedException, ModuleStartException { + try { + client.connect(); + influxClient.connect(); + + ModelInstaller installer; + if (config.getMetabaseType().equalsIgnoreCase("h2")) { + installer = new H2Installer(getManager()); + } else { + installer = new MySQLInstaller(getManager()); + } + installer.install(client); + new H2RegisterLockInstaller().install(client, lockDAO); + } catch (StorageException e) { + throw new ModuleStartException(e.getMessage(), e); + } + } + + @Override + public void notifyAfterCompleted() throws ServiceNotProvidedException, ModuleStartException { + + } + + @Override + public String[] requiredModules() { + return new String[] {CoreModule.NAME}; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/BatchDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/BatchDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..713a0b10cb1159efce8cbfbbbdf34151646802db --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/BatchDAO.java @@ -0,0 +1,60 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.storage.IBatchDAO; +import org.apache.skywalking.oap.server.library.client.request.InsertRequest; +import org.apache.skywalking.oap.server.library.client.request.PrepareRequest; +import org.apache.skywalking.oap.server.library.util.CollectionUtils; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.BatchPoints; + +@Slf4j +public class BatchDAO implements IBatchDAO { + private final InfluxClient client; + + public BatchDAO(InfluxClient client) { + this.client = client; + } + + @Override + public void asynchronous(InsertRequest insertRequest) { + client.write(((InfluxInsertRequest) insertRequest).getPoint()); + } + + @Override + public void synchronous(List prepareRequests) { + if (CollectionUtils.isEmpty(prepareRequests)) { + return; + } + + if (log.isDebugEnabled()) { + log.debug("batch sql statements execute, data size: {}", prepareRequests.size()); + } + + final BatchPoints.Builder builder = BatchPoints.builder(); + prepareRequests.forEach(e -> { + builder.point(((InfluxInsertRequest) e).getPoint()); + }); + + client.write(builder.build()); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/HistoryDeleteDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/HistoryDeleteDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..758a9f57bd6d6e6e8bbb0e2a3123adab5385003a --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/HistoryDeleteDAO.java @@ -0,0 +1,71 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import java.io.IOException; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.CoreModule; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.config.ConfigService; +import org.apache.skywalking.oap.server.core.storage.IHistoryDeleteDAO; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.ttl.StorageTTL; +import org.apache.skywalking.oap.server.core.storage.ttl.TTLCalculator; +import org.apache.skywalking.oap.server.library.module.ModuleDefineHolder; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.joda.time.DateTime; + +@Slf4j +public class HistoryDeleteDAO implements IHistoryDeleteDAO { + private final ModuleDefineHolder moduleDefineHolder; + private final InfluxClient client; + private final StorageTTL storageTTL; + + public HistoryDeleteDAO(ModuleDefineHolder moduleDefineHolder, InfluxClient client, StorageTTL storageTTL) { + this.moduleDefineHolder = moduleDefineHolder; + this.storageTTL = storageTTL; + this.client = client; + } + + @Override + public void deleteHistory(Model model, String timeBucketColumnName) throws IOException { + if (log.isDebugEnabled()) { + log.debug("TTL execution log, model: {}", model.getName()); + } + try { + ConfigService configService = moduleDefineHolder.find(CoreModule.NAME) + .provider() + .getService(ConfigService.class); + + TTLCalculator ttlCalculator; + if (model.isRecord()) { + ttlCalculator = storageTTL.recordCalculator(); + } else { + ttlCalculator = storageTTL.metricsCalculator(model.getDownsampling()); + } + + client.deleteByQuery( + model.getName(), + TimeBucket.getTimestamp(ttlCalculator.timeBefore(DateTime.now(), configService.getDataTTLConfig()) + 1) + ); + } catch (Exception e) { + log.error("TTL execution log, model: {}, errMsg: {}", model.getName(), e.getMessage()); + } + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java new file mode 100644 index 0000000000000000000000000000000000000000..82e0f12ad9d51b92b017de872a1a3aee9c859d69 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxInsertRequest.java @@ -0,0 +1,80 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import com.google.common.collect.Maps; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.storage.StorageBuilder; +import org.apache.skywalking.oap.server.core.storage.StorageData; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; +import org.apache.skywalking.oap.server.core.storage.type.StorageDataType; +import org.apache.skywalking.oap.server.library.client.request.InsertRequest; +import org.apache.skywalking.oap.server.library.client.request.UpdateRequest; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.Point; + +/** + * InfluxDB Point wrapper. + */ +public class InfluxInsertRequest implements InsertRequest, UpdateRequest { + public static final String ID = "id"; + + private Point.Builder builder; + private Map fields = Maps.newHashMap(); + + public InfluxInsertRequest(Model model, StorageData storageData, StorageBuilder storageBuilder) { + Map objectMap = storageBuilder.data2Map(storageData); + + for (ModelColumn column : model.getColumns()) { + Object value = objectMap.get(column.getColumnName().getName()); + + if (value instanceof StorageDataType) { + fields.put( + column.getColumnName().getStorageName(), + ((StorageDataType) value).toStorageData() + ); + } else { + fields.put(column.getColumnName().getStorageName(), value); + } + } + builder = Point.measurement(model.getName()) + .addField(ID, storageData.id()) + .fields(fields) + .tag(InfluxClient.TAG_TIME_BUCKET, String.valueOf(fields.get(Metrics.TIME_BUCKET))); + } + + public InfluxInsertRequest time(long time, TimeUnit unit) { + builder.time(time, unit); + return this; + } + + public InfluxInsertRequest addFieldAsTag(String fieldName, String tagName) { + if (fields.containsKey(fieldName)) { + builder.tag(tagName, String.valueOf(fields.get(fieldName))); + } + return this; + } + + public Point getPoint() { + return builder.build(); + } +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxStorageDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxStorageDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..3ea6df48d27fa28ed255e3d4417fd35cc607b582 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/InfluxStorageDAO.java @@ -0,0 +1,63 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.register.RegisterSource; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.INoneStreamDAO; +import org.apache.skywalking.oap.server.core.storage.IRecordDAO; +import org.apache.skywalking.oap.server.core.storage.IRegisterDAO; +import org.apache.skywalking.oap.server.core.storage.StorageBuilder; +import org.apache.skywalking.oap.server.core.storage.StorageDAO; +import org.apache.skywalking.oap.server.library.client.jdbc.hikaricp.JDBCHikariCPClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2RegisterDAO; + +public class InfluxStorageDAO implements StorageDAO { + private final InfluxClient influxClient; + private final JDBCHikariCPClient client; + + public InfluxStorageDAO(JDBCHikariCPClient client, InfluxClient influxdbClient) { + this.client = client; + this.influxClient = influxdbClient; + } + + @Override + public IMetricsDAO newMetricsDao(StorageBuilder storageBuilder) { + return new MetricsDAO(influxClient, storageBuilder); + } + + @Override + public IRegisterDAO newRegisterDao(StorageBuilder storageBuilder) { + return new H2RegisterDAO(client, storageBuilder); + } + + @Override + public IRecordDAO newRecordDao(StorageBuilder storageBuilder) { + return new RecordDAO(influxClient, storageBuilder); + } + + @Override + public INoneStreamDAO newNoneStreamDao(StorageBuilder storageBuilder) { + return new NoneStreamDAO(influxClient, storageBuilder); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..b44638cb1159113f75da244d7cba5d6afea9d83c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/MetricsDAO.java @@ -0,0 +1,106 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.storage.IMetricsDAO; +import org.apache.skywalking.oap.server.core.storage.StorageBuilder; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; +import org.apache.skywalking.oap.server.core.storage.type.StorageDataType; +import org.apache.skywalking.oap.server.library.client.request.InsertRequest; +import org.apache.skywalking.oap.server.library.client.request.UpdateRequest; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +public class MetricsDAO implements IMetricsDAO { + public static final String TAG_ENTITY_ID = "_entity_id"; + + private final StorageBuilder storageBuilder; + private final InfluxClient client; + + public MetricsDAO(InfluxClient client, StorageBuilder storageBuilder) { + this.client = client; + this.storageBuilder = storageBuilder; + } + + @Override + public List multiGet(Model model, List ids) throws IOException { + WhereQueryImpl query = select() + .regex("*::field") + .from(client.getDatabase(), model.getName()) + .where(contains("id", Joiner.on("|").join(ids))); + QueryResult.Series series = client.queryForSingleSeries(query); + + if (series == null) { + return Collections.emptyList(); + } + + final List metrics = Lists.newArrayList(); + List columns = series.getColumns(); + Map storageAndColumnNames = Maps.newHashMap(); + for (ModelColumn column : model.getColumns()) { + storageAndColumnNames.put(column.getColumnName().getStorageName(), column.getColumnName().getName()); + } + + series.getValues().forEach(values -> { + Map data = Maps.newHashMap(); + + for (int i = 1; i < columns.size(); i++) { + Object value = values.get(i); + if (value instanceof StorageDataType) { + value = ((StorageDataType) value).toStorageData(); + } + + data.put(storageAndColumnNames.get(columns.get(i)), value); + } + metrics.add(storageBuilder.map2Data(data)); + + }); + + return metrics; + } + + @Override + public InsertRequest prepareBatchInsert(Model model, Metrics metrics) throws IOException { + final long timestamp = TimeBucket.getTimestamp(metrics.getTimeBucket(), model.getDownsampling()); + return new InfluxInsertRequest(model, metrics, storageBuilder) + .time(timestamp, TimeUnit.MILLISECONDS) + .addFieldAsTag(Metrics.ENTITY_ID, TAG_ENTITY_ID); + } + + @Override + public UpdateRequest prepareBatchUpdate(Model model, Metrics metrics) throws IOException { + return (UpdateRequest) this.prepareBatchInsert(model, metrics); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..ae00288aa677d93969d4c13b646f72891569933b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/NoneStreamDAO.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.apm.commons.datacarrier.common.AtomicRangeInteger; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.config.NoneStream; +import org.apache.skywalking.oap.server.core.profile.ProfileTaskRecord; +import org.apache.skywalking.oap.server.core.storage.INoneStreamDAO; +import org.apache.skywalking.oap.server.core.storage.StorageBuilder; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.Point; + +public class NoneStreamDAO implements INoneStreamDAO { + public static final String TAG_SERVICE_ID = "_service_id"; + private static final int PADDING_SIZE = 1_000_000; + private static final AtomicRangeInteger SUFFIX = new AtomicRangeInteger(0, PADDING_SIZE); + + private InfluxClient client; + private StorageBuilder storageBuilder; + + public NoneStreamDAO(InfluxClient client, StorageBuilder storageBuilder) { + this.client = client; + this.storageBuilder = storageBuilder; + } + + @Override + public void insert(final Model model, final NoneStream noneStream) throws IOException { + final long timestamp = TimeBucket.getTimestamp( + noneStream.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); + + Point point = new InfluxInsertRequest(model, noneStream, storageBuilder) + .time(timestamp, TimeUnit.NANOSECONDS) + .addFieldAsTag(ProfileTaskRecord.SERVICE_ID, TAG_SERVICE_ID).getPoint(); + + client.write(point); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java new file mode 100644 index 0000000000000000000000000000000000000000..fae2e4109cd8809508f24f112c997f96188ba86e --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/base/RecordDAO.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.base; + +import java.io.IOException; +import java.util.concurrent.TimeUnit; +import org.apache.skywalking.apm.commons.datacarrier.common.AtomicRangeInteger; +import org.apache.skywalking.oap.server.core.analysis.TimeBucket; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.analysis.record.Record; +import org.apache.skywalking.oap.server.core.storage.IRecordDAO; +import org.apache.skywalking.oap.server.core.storage.StorageBuilder; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.request.InsertRequest; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; + +public class RecordDAO implements IRecordDAO { + public static final String TAG_SERVICE_ID = "_service_id"; + private static final int PADDING_SIZE = 1_000_000; + private static final AtomicRangeInteger SUFFIX = new AtomicRangeInteger(0, PADDING_SIZE); + + private InfluxClient client; + private StorageBuilder storageBuilder; + + public RecordDAO(InfluxClient client, StorageBuilder storageBuilder) { + this.client = client; + this.storageBuilder = storageBuilder; + } + + @Override + public InsertRequest prepareBatchInsert(Model model, Record record) throws IOException { + final long timestamp = TimeBucket.getTimestamp( + record.getTimeBucket(), model.getDownsampling()) * PADDING_SIZE + SUFFIX.getAndIncrement(); + + return new InfluxInsertRequest(model, record, storageBuilder) + .time(timestamp, TimeUnit.NANOSECONDS) + .addFieldAsTag(SegmentRecord.SERVICE_ID, TAG_SERVICE_ID); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/H2Installer.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/H2Installer.java new file mode 100644 index 0000000000000000000000000000000000000000..347f16bd71b668e8e3a3a85a183f4576d34e8079 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/H2Installer.java @@ -0,0 +1,44 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.installer; + +import org.apache.skywalking.oap.server.core.profile.ProfileTaskRecord; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.Client; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxModelConstants; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.h2.dao.H2TableInstaller; + +public class H2Installer extends H2TableInstaller { + + public H2Installer(ModuleManager moduleManager) { + super(moduleManager); + overrideColumnName(ProfileTaskRecord.DURATION, InfluxModelConstants.DURATION); + } + + @Override + protected boolean isExists(Client client, Model model) throws StorageException { + if (MetaTableDefine.contains(model)) { + return super.isExists(client, model); + } + return true; + } + +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/MetaTableDefine.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/MetaTableDefine.java new file mode 100644 index 0000000000000000000000000000000000000000..63472c9c3c4aa2383d58f487aef704439061b6e8 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/MetaTableDefine.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.installer; + +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.TableMetaInfo; + +/** + * Here defines which table is stored in metadata database(H2/MySQL). + */ +public class MetaTableDefine { + + /** + * In the InfluxDB implementation, only the metadata is managed by the traditional MySQL/H2 storages. + * Model#isCapableOfTimeSeries represents the metadata. + * + * @param model Model + * @return true if the {@link Model} is stored in H2/MySQL + */ + public static boolean contains(Model model) { + if (!model.isCapableOfTimeSeries()) { + return true; + } + TableMetaInfo.addModel(model); + return false; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/MySQLInstaller.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/MySQLInstaller.java new file mode 100644 index 0000000000000000000000000000000000000000..019d76949995f18c46e1b35080248adefbe528af --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/installer/MySQLInstaller.java @@ -0,0 +1,43 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.installer; + +import org.apache.skywalking.oap.server.core.profile.ProfileTaskRecord; +import org.apache.skywalking.oap.server.core.storage.StorageException; +import org.apache.skywalking.oap.server.core.storage.model.Model; +import org.apache.skywalking.oap.server.library.client.Client; +import org.apache.skywalking.oap.server.library.module.ModuleManager; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxModelConstants; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.mysql.MySQLTableInstaller; + +public class MySQLInstaller extends MySQLTableInstaller { + + public MySQLInstaller(ModuleManager moduleManager) { + super(moduleManager); + overrideColumnName(ProfileTaskRecord.DURATION, InfluxModelConstants.DURATION); + } + + @Override + protected boolean isExists(Client client, Model model) throws StorageException { + if (MetaTableDefine.contains(model)) { + return super.isExists(client, model); + } + return true; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..36436bbf26381fc92b19c7211943779e7cc2929b --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AggregationQuery.java @@ -0,0 +1,154 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.Downsampling; +import org.apache.skywalking.oap.server.core.query.entity.Order; +import org.apache.skywalking.oap.server.core.query.entity.TopNEntity; +import org.apache.skywalking.oap.server.core.register.EndpointInventory; +import org.apache.skywalking.oap.server.core.register.ServiceInstanceInventory; +import org.apache.skywalking.oap.server.core.storage.model.ModelName; +import org.apache.skywalking.oap.server.core.storage.query.IAggregationQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.MetricsDAO; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.SelectSubQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class AggregationQuery implements IAggregationQueryDAO { + private InfluxClient client; + + public AggregationQuery(InfluxClient client) { + this.client = client; + } + + @Override + public List getServiceTopN(String indName, String valueCName, int topN, Downsampling downsampling, + long startTB, long endTB, Order order) throws IOException { + return getTopNEntity(downsampling, indName, subQuery(indName, valueCName, startTB, endTB), order, topN); + } + + @Override + public List getAllServiceInstanceTopN(String indName, String valueCName, int topN, + Downsampling downsampling, + long startTB, long endTB, Order order) throws IOException { + return getTopNEntity(downsampling, indName, subQuery(indName, valueCName, startTB, endTB), order, topN); + } + + @Override + public List getServiceInstanceTopN(int serviceId, String indName, String valueCName, int topN, + Downsampling downsampling, + long startTB, long endTB, Order order) throws IOException { + return getTopNEntity( + downsampling, indName, + subQuery(ServiceInstanceInventory.SERVICE_ID, serviceId, indName, valueCName, startTB, endTB), order, topN + ); + } + + @Override + public List getAllEndpointTopN(String indName, String valueCName, int topN, Downsampling downsampling, + long startTB, long endTB, Order order) throws IOException { + return getTopNEntity(downsampling, indName, subQuery(indName, valueCName, startTB, endTB), order, topN); + } + + @Override + public List getEndpointTopN(int serviceId, String indName, String valueCName, int topN, + Downsampling downsampling, + long startTB, long endTB, Order order) throws IOException { + return getTopNEntity( + downsampling, indName, + subQuery(EndpointInventory.SERVICE_ID, serviceId, indName, valueCName, startTB, endTB), order, topN + ); + } + + private List getTopNEntity(Downsampling downsampling, + String name, + SelectSubQueryImpl subQuery, + Order order, + int topN) throws IOException { + String measurement = ModelName.build(downsampling, name); + // Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`. + Comparator comparator = DESCENDING; + String functionName = "top"; + if (order == Order.ASC) { + functionName = "bottom"; + comparator = ASCENDING; + } + + SelectQueryImpl query = select().function(functionName, "mean", topN).as("value") + .column(MetricsDAO.TAG_ENTITY_ID) + .from(client.getDatabase(), measurement); + query.setSubQuery(subQuery); + + List series = client.queryForSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (series == null || series.isEmpty()) { + return Collections.emptyList(); + } + + List> dataset = series.get(0).getValues(); + List entities = Lists.newArrayListWithCapacity(dataset.size()); + dataset.forEach(values -> { + final TopNEntity entity = new TopNEntity(); + entity.setId((String) values.get(2)); + entity.setValue(((Double) values.get(1)).longValue()); + entities.add(entity); + }); + + Collections.sort(entities, comparator); // re-sort by self, because of the result order by time. + return entities; + } + + private SelectSubQueryImpl subQuery(String serviceColumnName, int serviceId, String name, + String columnName, + long startTB, long endTB) { + return select().fromSubQuery(client.getDatabase()).mean(columnName).from(name) + .where() + .and(eq(serviceColumnName, serviceId)) + .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) + .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))) + .groupBy(MetricsDAO.TAG_ENTITY_ID); + } + + private SelectSubQueryImpl subQuery(String name, String columnName, long startTB, long endTB) { + return select().fromSubQuery(client.getDatabase()).mean(columnName).from(name) + .where() + .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) + .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))) + .groupBy(MetricsDAO.TAG_ENTITY_ID); + } + + private static final Comparator ASCENDING = Comparator.comparingLong(TopNEntity::getValue); + + private static final Comparator DESCENDING = (a, b) -> Long.compare(b.getValue(), a.getValue()); +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AlarmQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AlarmQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..c5fec30214aa6d1d197b86678903894c5e89189c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/AlarmQuery.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.alarm.AlarmRecord; +import org.apache.skywalking.oap.server.core.query.entity.AlarmMessage; +import org.apache.skywalking.oap.server.core.query.entity.Alarms; +import org.apache.skywalking.oap.server.core.query.entity.Scope; +import org.apache.skywalking.oap.server.core.storage.query.IAlarmQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.elasticsearch.common.Strings; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.regex; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class AlarmQuery implements IAlarmQueryDAO { + private final InfluxClient client; + + public AlarmQuery(InfluxClient client) { + this.client = client; + } + + @Override + public Alarms getAlarm(Integer scopeId, String keyword, int limit, int from, long startTB, + long endTB) throws IOException { + + WhereQueryImpl recallQuery = select() + .function("top", AlarmRecord.START_TIME, limit + from).as(AlarmRecord.START_TIME) + .column(AlarmRecord.ID0) + .column(AlarmRecord.ALARM_MESSAGE) + .column(AlarmRecord.SCOPE) + .from(client.getDatabase(), AlarmRecord.INDEX_NAME) + .where(); + if (startTB > 0 && endTB > 0) { + recallQuery.and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) + .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))); + } + if (!Strings.isNullOrEmpty(keyword)) { + recallQuery.and(regex(AlarmRecord.ALARM_MESSAGE, keyword)); + } + if (Objects.nonNull(scopeId)) { + recallQuery.and(eq(AlarmRecord.SCOPE, scopeId)); + } + + WhereQueryImpl countQuery = select().count(AlarmRecord.ID0) + .from(client.getDatabase(), AlarmRecord.INDEX_NAME) + .where(); + recallQuery.getClauses().forEach(clause -> { + countQuery.where(clause); + }); + + Query query = new Query(countQuery.getCommand() + recallQuery.getCommand()); + List results = client.query(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), results); + } + if (results.size() != 2) { + throw new IOException("Expecting to get 2 Results, but it is " + results.size()); + } + List series = results.get(1).getSeries(); + if (series == null || series.isEmpty()) { + return new Alarms(); + } + List counter = results.get(0).getSeries(); + Alarms alarms = new Alarms(); + alarms.setTotal(((Number) counter.get(0).getValues().get(0).get(1)).intValue()); + + series.get(0).getValues() + .stream() + // re-sort by self, because of the result order by time. + .sorted((a, b) -> Long.compare((long) b.get(1), (long) a.get(1))) + .skip(from) + .forEach(values -> { + final int sid = (int) values.get(4); + Scope scope = Scope.Finder.valueOf(sid); + + AlarmMessage message = new AlarmMessage(); + message.setStartTime((long) values.get(1)); + message.setId((String) values.get(2)); + message.setMessage((String) values.get(3)); + message.setScope(scope); + message.setScopeId(sid); + + alarms.getMsgs().add(message); + }); + return alarms; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..6b155929fe5fe33baf5b29a2748ff0be7875bf8c --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/LogQuery.java @@ -0,0 +1,153 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord; +import org.apache.skywalking.oap.server.core.query.entity.ContentType; +import org.apache.skywalking.oap.server.core.query.entity.Log; +import org.apache.skywalking.oap.server.core.query.entity.LogState; +import org.apache.skywalking.oap.server.core.query.entity.Logs; +import org.apache.skywalking.oap.server.core.query.entity.Pagination; +import org.apache.skywalking.oap.server.core.storage.query.ILogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; +import org.elasticsearch.common.Strings; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; +import org.influxdb.querybuilder.clauses.ConjunctionClause; + +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.CONTENT; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.CONTENT_TYPE; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.ENDPOINT_ID; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.IS_ERROR; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.SERVICE_ID; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.SERVICE_INSTANCE_ID; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.STATUS_CODE; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.TIMESTAMP; +import static org.apache.skywalking.oap.server.core.analysis.manual.log.AbstractLogRecord.TRACE_ID; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class LogQuery implements ILogQueryDAO { + private final InfluxClient client; + + public LogQuery(InfluxClient client) { + this.client = client; + } + + @Override + public Logs queryLogs(String metricName, int serviceId, int serviceInstanceId, int endpointId, String traceId, + LogState state, String stateCode, Pagination paging, int from, int limit, + long startTB, long endTB) throws IOException { + WhereQueryImpl recallQuery = select().regex("*::field") + .from(client.getDatabase(), metricName) + .where(); + if (serviceId != Const.NONE) { + recallQuery.and(eq(RecordDAO.TAG_SERVICE_ID, String.valueOf(serviceId))); + } + if (serviceInstanceId != Const.NONE) { + recallQuery.and(eq(SERVICE_INSTANCE_ID, serviceInstanceId)); + } + if (endpointId != Const.NONE) { + recallQuery.and(eq(ENDPOINT_ID, endpointId)); + } + if (!Strings.isNullOrEmpty(traceId)) { + recallQuery.and(eq(TRACE_ID, traceId)); + } + switch (state) { + case ERROR: { + recallQuery.and(eq(IS_ERROR, true)); + break; + } + case SUCCESS: { + recallQuery.and(eq(IS_ERROR, false)); + break; + } + } + if (!Strings.isNullOrEmpty(stateCode)) { + recallQuery.and(eq(STATUS_CODE, stateCode)); + } + recallQuery.and(gte(AbstractLogRecord.TIME_BUCKET, startTB)) + .and(lte(AbstractLogRecord.TIME_BUCKET, endTB)); + + if (from > Const.NONE) { + recallQuery.limit(limit, from); + } else { + recallQuery.limit(limit); + } + + SelectQueryImpl countQuery = select().count(ENDPOINT_ID).from(client.getDatabase(), metricName); + for (ConjunctionClause clause : recallQuery.getClauses()) { + countQuery.where(clause); + } + + Query query = new Query(countQuery.getCommand() + recallQuery.getCommand()); + List results = client.query(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} \nresult set: {}", query.getCommand(), results); + } + if (results.size() != 2) { + throw new IOException("Expecting to get 2 Results, but it is " + results.size()); + } + + final Logs logs = new Logs(); + QueryResult.Result counter = results.get(0); + QueryResult.Result seriesList = results.get(1); + + logs.setTotal(((Number) counter.getSeries().get(0).getValues().get(0).get(1)).intValue()); + seriesList.getSeries().forEach(series -> { + final List columns = series.getColumns(); + + series.getValues().forEach(values -> { + Map data = Maps.newHashMap(); + Log log = new Log(); + + for (int i = 0; i < columns.size(); i++) { + data.put(columns.get(i), values.get(i)); + } + log.setContent((String) data.get(CONTENT)); + log.setContentType(ContentType.instanceOf((int) data.get(CONTENT_TYPE))); + + log.setEndpointId((int) data.get(ENDPOINT_ID)); + log.setTraceId((String) data.get(TRACE_ID)); + log.setTimestamp((String) data.get(TIMESTAMP)); + + log.setStatusCode((String) data.get(STATUS_CODE)); + + log.setServiceId((int) data.get(SERVICE_ID)); + log.setServiceInstanceId((int) data.get(SERVICE_INSTANCE_ID)); + + logs.getLogs().add(log); + }); + }); + + return logs; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..9336139e274b920b62c0b964340c2c5ddba44048 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/MetricsQuery.java @@ -0,0 +1,304 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.base.Joiner; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.Downsampling; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValue; +import org.apache.skywalking.oap.server.core.analysis.metrics.IntKeyLongValueHashMap; +import org.apache.skywalking.oap.server.core.analysis.metrics.ThermodynamicMetrics; +import org.apache.skywalking.oap.server.core.query.entity.IntValues; +import org.apache.skywalking.oap.server.core.query.entity.KVInt; +import org.apache.skywalking.oap.server.core.query.entity.Thermodynamic; +import org.apache.skywalking.oap.server.core.query.sql.Function; +import org.apache.skywalking.oap.server.core.query.sql.KeyValues; +import org.apache.skywalking.oap.server.core.query.sql.Where; +import org.apache.skywalking.oap.server.core.storage.model.ModelColumn; +import org.apache.skywalking.oap.server.core.storage.model.ModelName; +import org.apache.skywalking.oap.server.core.storage.query.IMetricsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.MetricsDAO; +import org.apache.skywalking.oap.server.storage.plugin.jdbc.TableMetaInfo; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.SelectionQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class MetricsQuery implements IMetricsQueryDAO { + private final InfluxClient client; + + public MetricsQuery(InfluxClient client) { + this.client = client; + } + + @Override + public IntValues getValues(String indName, Downsampling downsampling, long startTB, long endTB, + Where where, String valueCName, Function function) throws IOException { + String measurement = ModelName.build(downsampling, indName); + + SelectionQueryImpl query = select(); + switch (function) { + case Avg: + query.mean(valueCName); + break; + default: + query.sum(valueCName); + } + WhereQueryImpl queryWhereQuery = query.from(client.getDatabase(), measurement).where(); + + Map> columnTypes = Maps.newHashMap(); + for (ModelColumn column : TableMetaInfo.get(measurement).getColumns()) { + columnTypes.put(column.getColumnName().getStorageName(), column.getType()); + } + + List ids = new ArrayList<>(20); + List whereKeyValues = where.getKeyValues(); + if (!whereKeyValues.isEmpty()) { + StringBuilder clauseBuilder = new StringBuilder(); + for (KeyValues kv : whereKeyValues) { + final List values = kv.getValues(); + + Class type = columnTypes.get(kv.getKey()); + if (values.size() == 1) { + String value = kv.getValues().get(0); + if (type == String.class) { + value = "'" + value + "'"; + } + clauseBuilder.append(kv.getKey()).append("=").append(value).append(" OR "); + } else { + ids.addAll(values); + if (type == String.class) { + clauseBuilder.append(kv.getKey()) + .append(" =~ /") + .append(Joiner.on("|").join(values)) + .append("/ OR "); + continue; + } + for (String value : values) { + clauseBuilder.append(kv.getKey()).append(" = '").append(value).append("' OR "); + } + } + } + queryWhereQuery.where(clauseBuilder.substring(0, clauseBuilder.length() - 4)); + } + queryWhereQuery + .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB, downsampling))) + .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB, downsampling))) + .groupBy(MetricsDAO.TAG_ENTITY_ID); + + IntValues intValues = new IntValues(); + List seriesList = client.queryForSeries(queryWhereQuery); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", queryWhereQuery.getCommand(), seriesList); + } + if (!(seriesList == null || seriesList.isEmpty())) { + for (QueryResult.Series series : seriesList) { + KVInt kv = new KVInt(); + kv.setId(series.getTags().get(MetricsDAO.TAG_ENTITY_ID)); + Number value = (Number) series.getValues().get(0).get(1); + kv.setValue(value.longValue()); + + intValues.addKVInt(kv); + } + } + + return orderWithDefault0(intValues, ids); + } + + @Override + public IntValues getLinearIntValues(String indName, Downsampling downsampling, List ids, String valueCName) + throws IOException { + String measurement = ModelName.build(downsampling, indName); + + WhereQueryImpl query = select() + .column("id") + .column(valueCName) + .from(client.getDatabase(), measurement) + .where(); + + if (ids != null && !ids.isEmpty()) { + if (ids.size() == 1) { + query.where(eq("id", ids.get(0))); + } else { + query.where(contains("id", Joiner.on("|").join(ids))); + } + } + List seriesList = client.queryForSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), seriesList); + } + + IntValues intValues = new IntValues(); + if (!(seriesList == null || seriesList.isEmpty())) { + seriesList.get(0).getValues().forEach(values -> { + KVInt kv = new KVInt(); + kv.setValue(((Number) values.get(2)).longValue()); + kv.setId((String) values.get(1)); + intValues.addKVInt(kv); + }); + } + return orderWithDefault0(intValues, ids); + } + + /** + * Make sure the order is same as the expected order, and keep default value as 0. + * + * @param origin IntValues + * @param expectedOrder List + * @return + */ + private IntValues orderWithDefault0(IntValues origin, List expectedOrder) { + IntValues intValues = new IntValues(); + + expectedOrder.forEach(id -> { + KVInt e = new KVInt(); + e.setId(id); + e.setValue(origin.findValue(id, 0)); + intValues.addKVInt(e); + }); + + return intValues; + } + + @Override + public IntValues[] getMultipleLinearIntValues(String indName, Downsampling downsampling, List ids, + List linearIndex, String valueCName) throws IOException { + String measurement = ModelName.build(downsampling, indName); + + WhereQueryImpl query = select() + .column("id") + .column(valueCName) + .from(client.getDatabase(), measurement) + .where(); + + if (ids != null && !ids.isEmpty()) { + if (ids.size() == 1) { + query.where(eq("id", ids.get(0))); + } else { + query.where(contains("id", Joiner.on("|").join(ids))); + } + } + List series = client.queryForSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + IntValues[] intValues = new IntValues[linearIndex.size()]; + for (int i = 0; i < intValues.length; i++) { + intValues[i] = new IntValues(); + } + if (series == null || series.isEmpty()) { + return intValues; + } + series.get(0).getValues().forEach(values -> { + IntKeyLongValueHashMap multipleValues = new IntKeyLongValueHashMap(5); + multipleValues.toObject((String) values.get(2)); + + final String id = (String) values.get(1); + for (int i = 0; i < intValues.length; i++) { + Integer index = linearIndex.get(i); + KVInt kv = new KVInt(); + kv.setId(id); + kv.setValue(multipleValues.get(index).getValue()); + intValues[i].addKVInt(kv); + } + }); + return orderWithDefault0(intValues, ids); + } + + /** + * Make sure the order is same as the expected order, and keep default value as 0. + * + * @param origin IntValues[] + * @param expectedOrder List + * @return + */ + private IntValues[] orderWithDefault0(IntValues[] origin, List expectedOrder) { + for (int i = 0; i < origin.length; i++) { + origin[i] = orderWithDefault0(origin[i], expectedOrder); + } + return origin; + } + + @Override + public Thermodynamic getThermodynamic(String indName, Downsampling downsampling, List ids, + String valueCName) + throws IOException { + String measurement = ModelName.build(downsampling, indName); + WhereQueryImpl query = select() + .column(ThermodynamicMetrics.STEP) + .column(ThermodynamicMetrics.NUM_OF_STEPS) + .column(ThermodynamicMetrics.DETAIL_GROUP) + .column("id") + .from(client.getDatabase(), measurement) + .where(contains("id", Joiner.on("|").join(ids))); + Map> thermodynamicValueMatrix = new HashMap<>(); + + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (series == null) { + return new Thermodynamic(); + } + + int numOfSteps = 0, axisYStep = 0; + List> thermodynamicValueCollection = new ArrayList<>(); + Thermodynamic thermodynamic = new Thermodynamic(); + for (List values : series.getValues()) { + numOfSteps = (int) values.get(2) + 1; + axisYStep = (int) values.get(1); + IntKeyLongValueHashMap intKeyLongValues = new IntKeyLongValueHashMap(5); + intKeyLongValues.toObject((String) values.get(3)); + List axisYValues = new ArrayList<>(numOfSteps); + for (int i = 0; i < numOfSteps; i++) { + axisYValues.add(0L); + } + for (IntKeyLongValue intKeyLongValue : intKeyLongValues.values()) { + axisYValues.set(intKeyLongValue.getKey(), intKeyLongValue.getValue()); + } + thermodynamicValueMatrix.put((String) values.get(4), axisYValues); + } + // try to add default values when there is no data in that time bucket. + ids.forEach(id -> { + if (thermodynamicValueMatrix.containsKey(id)) { + thermodynamicValueCollection.add(thermodynamicValueMatrix.get(id)); + } else { + thermodynamicValueCollection.add(new ArrayList<>()); + } + }); + thermodynamic.fromMatrixData(thermodynamicValueCollection, numOfSteps); + thermodynamic.setAxisYStep(axisYStep); + + return thermodynamic; + } +} \ No newline at end of file diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..06ff3f4ef901bdbbc92bd7abaafd98cb5b8e0339 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskLogQuery.java @@ -0,0 +1,98 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.collect.Lists; +import com.google.common.collect.Maps; +import java.io.IOException; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.profile.ProfileTaskLogRecord; +import org.apache.skywalking.oap.server.core.query.entity.ProfileTaskLog; +import org.apache.skywalking.oap.server.core.query.entity.ProfileTaskLogOperationType; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskLogQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class ProfileTaskLogQuery implements IProfileTaskLogQueryDAO { + private InfluxClient client; + private int fetchTaskLogMaxSize; + + public ProfileTaskLogQuery(InfluxClient client, int fetchTaskLogMaxSize) { + this.client = client; + this.fetchTaskLogMaxSize = fetchTaskLogMaxSize; + } + + @Override + public List getTaskLogList(String taskId) throws IOException { + WhereQueryImpl query = select() + .function("top", ProfileTaskLogRecord.OPERATION_TIME, fetchTaskLogMaxSize) + .column("id") + .column(ProfileTaskLogRecord.TASK_ID) + .column(ProfileTaskLogRecord.INSTANCE_ID) + .column(ProfileTaskLogRecord.OPERATION_TIME) + .column(ProfileTaskLogRecord.OPERATION_TYPE) + .from(client.getDatabase(), ProfileTaskLogRecord.INDEX_NAME) + .where(); + + if (StringUtil.isNotEmpty(taskId)) { + query.and(eq(ProfileTaskLogRecord.TASK_ID, taskId)); + } + + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (series == null) { + return Collections.emptyList(); + } + List columns = series.getColumns(); + Map columnsMap = Maps.newHashMap(); + for (int i = 0; i < columns.size(); i++) { + columnsMap.put(columns.get(i), i); + } + + List taskLogs = Lists.newArrayList(); + series.getValues().stream() + // re-sort by self, because of the result order by time. + .sorted((a, b) -> Long.compare(((Number) b.get(1)).longValue(), ((Number) a.get(1)).longValue())) + .forEach(values -> { + taskLogs.add(ProfileTaskLog.builder() + .id((String) values.get(columnsMap.get("id"))) + .taskId((String) values.get(columnsMap.get(ProfileTaskLogRecord.TASK_ID))) + .instanceId( + (int) values.get(columnsMap.get(ProfileTaskLogRecord.INSTANCE_ID))) + .operationTime( + (Long) values.get(columnsMap.get(ProfileTaskLogRecord.OPERATION_TIME))) + .operationType(ProfileTaskLogOperationType.parse( + (int) values.get(columnsMap.get(ProfileTaskLogRecord.OPERATION_TYPE)))) + .build()); + }); + return taskLogs; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..555352880ef7381cf48b01854acef8c79dc54615 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileTaskQuery.java @@ -0,0 +1,131 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.List; +import java.util.Objects; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.profile.ProfileTaskRecord; +import org.apache.skywalking.oap.server.core.query.entity.ProfileTask; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileTaskQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxModelConstants; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.NoneStreamDAO; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +public class ProfileTaskQuery implements IProfileTaskQueryDAO { + private InfluxClient client; + + public ProfileTaskQuery(InfluxClient client) { + this.client = client; + } + + @Override + public List getTaskList(final Integer serviceId, + final String endpointName, + final Long startTimeBucket, + final Long endTimeBucket, + final Integer limit) throws IOException { + WhereQueryImpl query = + select("id", ProfileTaskRecord.SERVICE_ID, + ProfileTaskRecord.ENDPOINT_NAME, ProfileTaskRecord.START_TIME, + ProfileTaskRecord.CREATE_TIME, + InfluxModelConstants.DURATION, + ProfileTaskRecord.MIN_DURATION_THRESHOLD, + ProfileTaskRecord.DUMP_PERIOD, + ProfileTaskRecord.MAX_SAMPLING_COUNT + ) + .from(client.getDatabase(), ProfileTaskRecord.INDEX_NAME) + .where(); + + if (Objects.nonNull(serviceId)) { + query.and(eq(NoneStreamDAO.TAG_SERVICE_ID, String.valueOf(serviceId))); + } + if (StringUtil.isNotEmpty(endpointName)) { + query.and(eq(ProfileTaskRecord.ENDPOINT_NAME, endpointName)); + } + if (Objects.nonNull(startTimeBucket)) { + query.and(gte(ProfileTaskRecord.TIME_BUCKET, startTimeBucket)); + } + if (Objects.nonNull(endTimeBucket)) { + query.and(lte(ProfileTaskRecord.TIME_BUCKET, endTimeBucket)); + } + if (Objects.nonNull(limit)) { + query.limit(limit); + } + + List tasks = Lists.newArrayList(); + QueryResult.Series series = client.queryForSingleSeries(query); + if (series != null) { + series.getValues().forEach(values -> { + tasks.add(profileTaskBuilder(values)); + }); + } + return tasks; + } + + @Override + public ProfileTask getById(final String id) throws IOException { + if (StringUtil.isEmpty(id)) { + return null; + } + SelectQueryImpl query = select("id", ProfileTaskRecord.SERVICE_ID, + ProfileTaskRecord.ENDPOINT_NAME, ProfileTaskRecord.START_TIME, + ProfileTaskRecord.CREATE_TIME, + InfluxModelConstants.DURATION, + ProfileTaskRecord.MIN_DURATION_THRESHOLD, + ProfileTaskRecord.DUMP_PERIOD, + ProfileTaskRecord.MAX_SAMPLING_COUNT + ) + .from(client.getDatabase(), ProfileTaskRecord.INDEX_NAME) + .where() + .and(eq("id", id)) + .limit(1); + + QueryResult.Series series = client.queryForSingleSeries(query); + if (Objects.nonNull(series)) { + return profileTaskBuilder(series.getValues().get(0)); + } + return null; + } + + private static final ProfileTask profileTaskBuilder(List values) { + return ProfileTask.builder() + .id((String) values.get(1)) + .serviceId(((Number) values.get(2)).intValue()) + .endpointName((String) values.get(3)) + .startTime(((Number) values.get(4)).longValue()) + .createTime(((Number) values.get(5)).longValue()) + .duration((int) values.get(6)) + .minDurationThreshold((int) values.get(7)) + .dumpPeriod((int) values.get(8)) + .maxSamplingCount((int) values.get(9)) + .build(); + } + +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..753ef582e9e6c8b8f253dd7dca91d846b5f6f3ca --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/ProfileThreadSnapshotQuery.java @@ -0,0 +1,165 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.base.Joiner; +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Base64; +import java.util.Collections; +import java.util.LinkedList; +import java.util.List; +import org.apache.skywalking.apm.util.StringUtil; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.profile.ProfileThreadSnapshotRecord; +import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; +import org.apache.skywalking.oap.server.core.storage.profile.IProfileThreadSnapshotQueryDAO; +import org.apache.skywalking.oap.server.library.util.BooleanUtils; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.contains; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +public class ProfileThreadSnapshotQuery implements IProfileThreadSnapshotQueryDAO { + private final InfluxClient client; + + public ProfileThreadSnapshotQuery(InfluxClient client) { + this.client = client; + } + + @Override + public List queryProfiledSegments(String taskId) throws IOException { + WhereQueryImpl query = select(ProfileThreadSnapshotRecord.SEGMENT_ID) + .from(client.getDatabase(), ProfileThreadSnapshotRecord.INDEX_NAME) + .where() + .and(eq(ProfileThreadSnapshotRecord.TASK_ID, taskId)) + .and(eq(ProfileThreadSnapshotRecord.SEQUENCE, 0)); + + final LinkedList segments = new LinkedList<>(); + QueryResult.Series series = client.queryForSingleSeries(query); + if (series == null) { + return Collections.emptyList(); + } + series.getValues().forEach(values -> { + segments.add((String) values.get(1)); + }); + + if (segments.isEmpty()) { + return Collections.emptyList(); + } + + query = select() + .function("bottom", SegmentRecord.START_TIME, segments.size()) + .column(SegmentRecord.SEGMENT_ID) + .column(SegmentRecord.START_TIME) + .column(SegmentRecord.ENDPOINT_NAME) + .column(SegmentRecord.LATENCY) + .column(SegmentRecord.IS_ERROR) + .column(SegmentRecord.TRACE_ID) + .from(client.getDatabase(), SegmentRecord.INDEX_NAME) + .where() + .and(contains(SegmentRecord.SEGMENT_ID, Joiner.on("|").join(segments))); + + ArrayList result = Lists.newArrayListWithCapacity(segments.size()); + client.queryForSingleSeries(query) + .getValues() + .stream() + .sorted((a, b) -> Long.compare(((Number) b.get(1)).longValue(), ((Number) a.get(1)).longValue())) + .forEach(values -> { + BasicTrace basicTrace = new BasicTrace(); + + basicTrace.setSegmentId((String) values.get(2)); + basicTrace.setStart(String.valueOf(values.get(3))); + basicTrace.getEndpointNames().add((String) values.get(4)); + basicTrace.setDuration((int) values.get(5)); + basicTrace.setError(BooleanUtils.valueToBoolean((int) values.get(6))); + String traceIds = (String) values.get(7); + basicTrace.getTraceIds().add(traceIds); + + result.add(basicTrace); + }); + + return result; + } + + @Override + public int queryMinSequence(String segmentId, long start, long end) throws IOException { + return querySequenceWithAgg("min", segmentId, start, end); + } + + @Override + public int queryMaxSequence(String segmentId, long start, long end) throws IOException { + return querySequenceWithAgg("max", segmentId, start, end); + } + + @Override + public List queryRecords(String segmentId, int minSequence, + int maxSequence) throws IOException { + WhereQueryImpl query = select( + ProfileThreadSnapshotRecord.TASK_ID, + ProfileThreadSnapshotRecord.SEGMENT_ID, + ProfileThreadSnapshotRecord.DUMP_TIME, + ProfileThreadSnapshotRecord.SEQUENCE, + ProfileThreadSnapshotRecord.STACK_BINARY + ) + .from(client.getDatabase(), ProfileThreadSnapshotRecord.INDEX_NAME) + .where(eq(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId)) + .and(gte(ProfileThreadSnapshotRecord.SEQUENCE, minSequence)) + .and(lte(ProfileThreadSnapshotRecord.SEQUENCE, maxSequence)); + + ArrayList result = new ArrayList<>(maxSequence - minSequence); + client.queryForSingleSeries(query).getValues().forEach(values -> { + ProfileThreadSnapshotRecord record = new ProfileThreadSnapshotRecord(); + + record.setTaskId((String) values.get(1)); + record.setSegmentId((String) values.get(2)); + record.setDumpTime(((Number) values.get(3)).longValue()); + record.setSequence((int) values.get(4)); + String dataBinaryBase64 = String.valueOf(values.get(5)); + if (StringUtil.isNotEmpty(dataBinaryBase64)) { + record.setStackBinary(Base64.getDecoder().decode(dataBinaryBase64)); + } + + result.add(record); + }); + + return result; + } + + private int querySequenceWithAgg(String function, String segmentId, long start, long end) throws IOException { + WhereQueryImpl query = select() + .function(function, ProfileThreadSnapshotRecord.SEQUENCE) + .from(client.getDatabase(), ProfileThreadSnapshotRecord.INDEX_NAME) + .where() + .and(eq(ProfileThreadSnapshotRecord.SEGMENT_ID, segmentId)) + .and(gte(ProfileThreadSnapshotRecord.DUMP_TIME, start)) + .and(lte(ProfileThreadSnapshotRecord.DUMP_TIME, end)); + QueryResult.Series series = client.queryForSingleSeries(query); + if (series == null) { + return -1; + } + return ((Number) series.getValues().get(0).get(1)).intValue(); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..285eb7755ea66946252b9151a32e5fe34559e62e --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopNRecordsQuery.java @@ -0,0 +1,94 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.Const; +import org.apache.skywalking.oap.server.core.analysis.topn.TopN; +import org.apache.skywalking.oap.server.core.query.entity.Order; +import org.apache.skywalking.oap.server.core.query.entity.TopNRecord; +import org.apache.skywalking.oap.server.core.storage.query.ITopNRecordsQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class TopNRecordsQuery implements ITopNRecordsQueryDAO { + private final InfluxClient client; + + public TopNRecordsQuery(InfluxClient client) { + this.client = client; + } + + @Override + public List getTopNRecords(long startSecondTB, long endSecondTB, String metricName, + int serviceId, int topN, Order order) throws IOException { + String function = "bottom"; + // Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`. + Comparator comparator = Comparator.comparingLong(TopNRecord::getLatency); + if (order.equals(Order.DES)) { + function = "top"; + comparator = (a, b) -> Long.compare(b.getLatency(), a.getLatency()); + } + + WhereQueryImpl query = select() + .function(function, TopN.LATENCY, topN) + .column(TopN.STATEMENT) + .column(TopN.TRACE_ID) + .from(client.getDatabase(), metricName) + .where() + .and(gte(TopN.TIME_BUCKET, startSecondTB)) + .and(lte(TopN.TIME_BUCKET, endSecondTB)); + + if (serviceId != Const.NONE) { + query.and(eq(RecordDAO.TAG_SERVICE_ID, String.valueOf(serviceId))); + } + + QueryResult.Series series = client.queryForSingleSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (series == null) { + return Collections.emptyList(); + } + + final List records = new ArrayList<>(); + series.getValues().forEach(values -> { + TopNRecord record = new TopNRecord(); + record.setLatency((long) values.get(1)); + record.setTraceId((String) values.get(3)); + record.setStatement((String) values.get(2)); + records.add(record); + }); + + Collections.sort(records, comparator); // re-sort by self, because of the result order by time. + return records; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..34b81c8fb759f34cbc8e18da3f1114ee6aac5fc2 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TopologyQuery.java @@ -0,0 +1,257 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.Downsampling; +import org.apache.skywalking.oap.server.core.analysis.manual.RelationDefineUtil; +import org.apache.skywalking.oap.server.core.analysis.manual.endpointrelation.EndpointRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.instance.ServiceInstanceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationClientSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.manual.relation.service.ServiceRelationServerSideMetrics; +import org.apache.skywalking.oap.server.core.analysis.metrics.Metrics; +import org.apache.skywalking.oap.server.core.query.entity.Call; +import org.apache.skywalking.oap.server.core.source.DetectPoint; +import org.apache.skywalking.oap.server.core.storage.model.ModelName; +import org.apache.skywalking.oap.server.core.storage.query.ITopologyQueryDAO; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.WhereNested; +import org.influxdb.querybuilder.WhereQueryImpl; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class TopologyQuery implements ITopologyQueryDAO { + private final InfluxClient client; + + public TopologyQuery(InfluxClient client) { + this.client = client; + } + + @Override + public List loadSpecifiedServerSideServiceRelations(Downsampling downsampling, long startTB, + long endTB, + List serviceIds) throws IOException { + String measurement = ModelName.build(downsampling, ServiceRelationServerSideMetrics.INDEX_NAME); + WhereQueryImpl query = buildServiceCallsQuery( + measurement, + startTB, + endTB, + ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, + ServiceRelationServerSideMetrics.DEST_SERVICE_ID, + serviceIds + ); + return buildCalls(query, DetectPoint.SERVER); + } + + @Override + public List loadSpecifiedClientSideServiceRelations(Downsampling downsampling, long startTB, + long endTB, + List serviceIds) throws IOException { + String measurement = ModelName.build(downsampling, ServiceRelationClientSideMetrics.INDEX_NAME); + WhereQueryImpl query = buildServiceCallsQuery( + measurement, + startTB, + endTB, + ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, + ServiceRelationServerSideMetrics.DEST_SERVICE_ID, + serviceIds + ); + return buildCalls(query, DetectPoint.CLIENT); + } + + @Override + public List loadServerSideServiceRelations(Downsampling downsampling, long startTB, + long endTB) throws IOException { + String measurement = ModelName.build(downsampling, ServiceRelationServerSideMetrics.INDEX_NAME); + WhereQueryImpl query = buildServiceCallsQuery( + measurement, + startTB, + endTB, + ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, + ServiceRelationServerSideMetrics.DEST_SERVICE_ID, + new ArrayList<>(0) + ); + return buildCalls(query, DetectPoint.SERVER); + } + + @Override + public List loadClientSideServiceRelations(Downsampling downsampling, long startTB, + long endTB) throws IOException { + String tableName = ModelName.build(downsampling, ServiceRelationClientSideMetrics.INDEX_NAME); + WhereQueryImpl query = buildServiceCallsQuery( + tableName, + startTB, + endTB, + ServiceRelationServerSideMetrics.SOURCE_SERVICE_ID, + ServiceRelationServerSideMetrics.DEST_SERVICE_ID, + new ArrayList<>(0) + ); + return buildCalls(query, DetectPoint.CLIENT); + } + + @Override + public List loadServerSideServiceInstanceRelations(int clientServiceId, + int serverServiceId, + Downsampling downsampling, + long startTB, + long endTB) throws IOException { + String measurement = ModelName.build(downsampling, ServiceInstanceRelationServerSideMetrics.INDEX_NAME); + WhereQueryImpl query = buildServiceInstanceCallsQuery(measurement, + startTB, + endTB, + ServiceInstanceRelationServerSideMetrics.SOURCE_SERVICE_ID, + ServiceInstanceRelationServerSideMetrics.DEST_SERVICE_ID, + clientServiceId, serverServiceId + ); + return buildCalls(query, DetectPoint.SERVER); + } + + @Override + public List loadClientSideServiceInstanceRelations(int clientServiceId, + int serverServiceId, + Downsampling downsampling, + long startTB, + long endTB) throws IOException { + String measurement = ModelName.build(downsampling, ServiceInstanceRelationClientSideMetrics.INDEX_NAME); + WhereQueryImpl query = buildServiceInstanceCallsQuery(measurement, + startTB, + endTB, + ServiceInstanceRelationClientSideMetrics.SOURCE_SERVICE_ID, + ServiceInstanceRelationClientSideMetrics.DEST_SERVICE_ID, + clientServiceId, serverServiceId + ); + return buildCalls(query, DetectPoint.CLIENT); + } + + @Override + public List loadSpecifiedDestOfServerSideEndpointRelations(Downsampling downsampling, + long startTB, + long endTB, + int destEndpointId) throws IOException { + String measurement = ModelName.build(downsampling, EndpointRelationServerSideMetrics.INDEX_NAME); + + WhereQueryImpl query = buildServiceCallsQuery( + measurement, + startTB, + endTB, + EndpointRelationServerSideMetrics.SOURCE_ENDPOINT_ID, + EndpointRelationServerSideMetrics.DEST_ENDPOINT_ID, + Collections.emptyList() + ); + query.and(eq(EndpointRelationServerSideMetrics.DEST_ENDPOINT_ID, destEndpointId)); + + WhereQueryImpl query2 = buildServiceCallsQuery( + measurement, + startTB, + endTB, + EndpointRelationServerSideMetrics.SOURCE_ENDPOINT_ID, + EndpointRelationServerSideMetrics.DEST_ENDPOINT_ID, + Collections.emptyList() + ); + query2.and(eq(EndpointRelationServerSideMetrics.SOURCE_ENDPOINT_ID, destEndpointId)); + + List calls = buildCalls(query, DetectPoint.SERVER); + calls.addAll(buildCalls(query2, DetectPoint.CLIENT)); + return calls; + } + + private WhereQueryImpl buildServiceCallsQuery(String measurement, long startTB, long endTB, String sourceCName, + String destCName, List serviceIds) { + WhereQueryImpl query = select() + .function("distinct", Metrics.ENTITY_ID) + .from(client.getDatabase(), measurement) + .where() + .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) + .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))); + + if (!serviceIds.isEmpty()) { + WhereNested whereNested = query.andNested(); + for (Integer id : serviceIds) { + whereNested.or(eq(sourceCName, id)) + .or(eq(destCName, id)); + } + whereNested.close(); + } + return query; + } + + private WhereQueryImpl buildServiceInstanceCallsQuery(String measurement, + long startTB, + long endTB, + String sourceCName, + String destCName, + int sourceServiceId, + int destServiceId) { + WhereQueryImpl query = select() + .function("distinct", Metrics.ENTITY_ID) + .from(client.getDatabase(), measurement) + .where() + .and(gte(InfluxClient.TIME, InfluxClient.timeInterval(startTB))) + .and(lte(InfluxClient.TIME, InfluxClient.timeInterval(endTB))); + + StringBuilder builder = new StringBuilder("(("); + builder.append(sourceCName).append("=").append(sourceServiceId) + .append(" and ") + .append(destCName).append("=").append(destServiceId) + .append(") or (") + .append(sourceCName).append("=").append(destServiceId) + .append(") and (") + .append(destCName).append("=").append(sourceServiceId) + .append("))"); + query.where(builder.toString()); + return query; + } + + private List buildCalls(WhereQueryImpl query, + DetectPoint detectPoint) throws IOException { + QueryResult.Series series = client.queryForSingleSeries(query); + + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (series == null) { + return Collections.emptyList(); + } + + List calls = new ArrayList<>(); + series.getValues().forEach(values -> { + Call.CallDetail call = new Call.CallDetail(); + String entityId = (String) values.get(1); + RelationDefineUtil.RelationDefine relationDefine = RelationDefineUtil.splitEntityId(entityId); + + call.setSource(relationDefine.getSource()); + call.setTarget(relationDefine.getDest()); + call.setComponentId(relationDefine.getComponentId()); + call.setDetectPoint(detectPoint); + call.generateID(); + calls.add(call); + }); + return calls; + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java new file mode 100644 index 0000000000000000000000000000000000000000..6ad679edb0f18ad77021dce2153abc354a255996 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/influxdb/query/TraceQuery.java @@ -0,0 +1,218 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.oap.server.storage.plugin.influxdb.query; + +import com.google.common.collect.Lists; +import java.io.IOException; +import java.util.Base64; +import java.util.Collections; +import java.util.List; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.oap.server.core.analysis.manual.segment.SegmentRecord; +import org.apache.skywalking.oap.server.core.query.entity.BasicTrace; +import org.apache.skywalking.oap.server.core.query.entity.QueryOrder; +import org.apache.skywalking.oap.server.core.query.entity.Span; +import org.apache.skywalking.oap.server.core.query.entity.TraceBrief; +import org.apache.skywalking.oap.server.core.query.entity.TraceState; +import org.apache.skywalking.oap.server.core.storage.query.ITraceQueryDAO; +import org.apache.skywalking.oap.server.library.util.BooleanUtils; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxClient; +import org.apache.skywalking.oap.server.storage.plugin.influxdb.base.RecordDAO; +import org.elasticsearch.common.Strings; +import org.influxdb.dto.Query; +import org.influxdb.dto.QueryResult; +import org.influxdb.querybuilder.SelectQueryImpl; +import org.influxdb.querybuilder.WhereQueryImpl; +import org.influxdb.querybuilder.clauses.Clause; + +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.eq; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.gte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.lte; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.regex; +import static org.influxdb.querybuilder.BuiltQuery.QueryBuilder.select; + +@Slf4j +public class TraceQuery implements ITraceQueryDAO { + private final InfluxClient client; + + public TraceQuery(InfluxClient client) { + this.client = client; + } + + @Override + public TraceBrief queryBasicTraces(long startSecondTB, + long endSecondTB, + long minDuration, + long maxDuration, + String endpointName, + int serviceId, + int serviceInstanceId, + int endpointId, + String traceId, + int limit, + int from, + TraceState traceState, + QueryOrder queryOrder) + throws IOException { + + String orderBy = SegmentRecord.START_TIME; + if (queryOrder == QueryOrder.BY_DURATION) { + orderBy = SegmentRecord.LATENCY; + } + + WhereQueryImpl recallQuery = select() + .function("top", orderBy, limit + from) + .column(SegmentRecord.SEGMENT_ID) + .column(SegmentRecord.START_TIME) + .column(SegmentRecord.ENDPOINT_NAME) + .column(SegmentRecord.LATENCY) + .column(SegmentRecord.IS_ERROR) + .column(SegmentRecord.TRACE_ID) + .from(client.getDatabase(), SegmentRecord.INDEX_NAME) + .where(); + + if (startSecondTB != 0 && endSecondTB != 0) { + recallQuery.and(gte(SegmentRecord.TIME_BUCKET, startSecondTB)) + .and(lte(SegmentRecord.TIME_BUCKET, endSecondTB)); + } + if (minDuration != 0) { + recallQuery.and(gte(SegmentRecord.LATENCY, minDuration)); + } + if (maxDuration != 0) { + recallQuery.and(lte(SegmentRecord.LATENCY, maxDuration)); + } + if (!Strings.isNullOrEmpty(endpointName)) { + recallQuery.and(regex(SegmentRecord.ENDPOINT_NAME, "/" + endpointName.replaceAll("/", "\\\\/") + "/")); + } + if (serviceId != 0) { + recallQuery.and(eq(RecordDAO.TAG_SERVICE_ID, String.valueOf(serviceId))); + } + if (serviceInstanceId != 0) { + recallQuery.and(eq(SegmentRecord.SERVICE_INSTANCE_ID, serviceInstanceId)); + } + if (endpointId != 0) { + recallQuery.and(eq(SegmentRecord.ENDPOINT_ID, endpointId)); + } + if (!Strings.isNullOrEmpty(traceId)) { + recallQuery.and(eq(SegmentRecord.TRACE_ID, traceId)); + } + switch (traceState) { + case ERROR: + recallQuery.and(eq(SegmentRecord.IS_ERROR, BooleanUtils.TRUE)); + break; + case SUCCESS: + recallQuery.and(eq(SegmentRecord.IS_ERROR, BooleanUtils.FALSE)); + break; + } + + WhereQueryImpl countQuery = select() + .count(SegmentRecord.ENDPOINT_ID) + .from(client.getDatabase(), SegmentRecord.INDEX_NAME) + .where(); + for (Clause clause : recallQuery.getClauses()) { + countQuery.where(clause); + } + Query query = new Query(countQuery.getCommand() + recallQuery.getCommand()); + + List results = client.query(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), results); + } + if (results.size() != 2) { + throw new IOException("Expecting to get 2 Results, but it is " + results.size()); + } + List counter = results.get(0).getSeries(); + List result = results.get(1).getSeries(); + if (result == null || result.isEmpty()) { + return new TraceBrief(); + } + + TraceBrief traceBrief = new TraceBrief(); + traceBrief.setTotal(((Number) counter.get(0).getValues().get(0).get(1)).intValue()); + + result.get(0).getValues().stream().sorted((a, b) -> { + // Have to re-sort here. Because the function, top()/bottom(), get the result ordered by the `time`. + return Long.compare(((Number) b.get(1)).longValue(), ((Number) a.get(1)).longValue()); + }).skip(from).forEach(values -> { + BasicTrace basicTrace = new BasicTrace(); + + basicTrace.setSegmentId((String) values.get(2)); + basicTrace.setStart(String.valueOf((long) values.get(3))); + basicTrace.getEndpointNames().add((String) values.get(4)); + basicTrace.setDuration((int) values.get(5)); + basicTrace.setError(BooleanUtils.valueToBoolean((int) values.get(6))); + basicTrace.getTraceIds().add((String) values.get(7)); + + traceBrief.getTraces().add(basicTrace); + }); + return traceBrief; + } + + @Override + public List queryByTraceId(String traceId) throws IOException { + WhereQueryImpl query = select().column(SegmentRecord.SEGMENT_ID) + .column(SegmentRecord.TRACE_ID) + .column(SegmentRecord.SERVICE_ID) + .column(SegmentRecord.ENDPOINT_NAME) + .column(SegmentRecord.START_TIME) + .column(SegmentRecord.END_TIME) + .column(SegmentRecord.LATENCY) + .column(SegmentRecord.IS_ERROR) + .column(SegmentRecord.DATA_BINARY) + .column(SegmentRecord.VERSION) + .from(client.getDatabase(), SegmentRecord.INDEX_NAME) + .where() + .and(eq(SegmentRecord.TRACE_ID, traceId)); + List series = client.queryForSeries(query); + if (log.isDebugEnabled()) { + log.debug("SQL: {} result set: {}", query.getCommand(), series); + } + if (series == null || series.isEmpty()) { + return Collections.emptyList(); + } + List segmentRecords = Lists.newArrayList(); + series.get(0).getValues().forEach(values -> { + SegmentRecord segmentRecord = new SegmentRecord(); + + segmentRecord.setSegmentId((String) values.get(1)); + segmentRecord.setTraceId((String) values.get(2)); + segmentRecord.setServiceId((int) values.get(3)); + segmentRecord.setEndpointName((String) values.get(4)); + segmentRecord.setStartTime((long) values.get(5)); + segmentRecord.setEndTime((long) values.get(6)); + segmentRecord.setLatency((int) values.get(7)); + segmentRecord.setIsError((int) values.get(8)); + segmentRecord.setVersion((int) values.get(10)); + + String base64 = (String) values.get(9); + if (!Strings.isNullOrEmpty(base64)) { + segmentRecord.setDataBinary(Base64.getDecoder().decode(base64)); + } + + segmentRecords.add(segmentRecord); + }); + + return segmentRecords; + } + + @Override + public List doFlexibleTraceQuery(String traceId) throws IOException { + return Collections.emptyList(); + } +} diff --git a/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider new file mode 100644 index 0000000000000000000000000000000000000000..b53b0b0e93c5b13c16f5e17872e142c82a7b2599 --- /dev/null +++ b/oap-server/server-storage-plugin/storage-influxdb-plugin/src/main/resources/META-INF/services/org.apache.skywalking.oap.server.library.module.ModuleProvider @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# + +org.apache.skywalking.oap.server.storage.plugin.influxdb.InfluxStorageProvider \ No newline at end of file diff --git a/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/docker/rc.d/rc0-prepare.sh b/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/docker/rc.d/rc0-prepare.sh index 06c5a8e995f8be9495ec8ec4f3f578a7870a9a53..92759b5c0d50f3bf643503e7d64cfc66c4fd6542 100755 --- a/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/docker/rc.d/rc0-prepare.sh +++ b/test/e2e/e2e-cluster-with-gateway/e2e-cluster-with-gateway-test-runner/src/docker/rc.d/rc0-prepare.sh @@ -15,8 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -apt-get update && apt-get install -y gawk - if test "${MODE}" = "cluster"; then original_wd=$(pwd) diff --git a/test/e2e/e2e-cluster/e2e-cluster-test-runner/pom.xml b/test/e2e/e2e-cluster/e2e-cluster-test-runner/pom.xml index 485ac8402bc6abd663d938dd05569b20689e1320..45810a6a2cec22c5e3848d0ec0a31fb03fa15602 100755 --- a/test/e2e/e2e-cluster/e2e-cluster-test-runner/pom.xml +++ b/test/e2e/e2e-cluster/e2e-cluster-test-runner/pom.xml @@ -59,128 +59,282 @@ skywalking-e2e-container-${build.id}-cluster + + + elasticsearch + + + + io.fabric8 + docker-maven-plugin + + %a-%t-%i + Always + + + elastic/elasticsearch:${elasticsearch.version} + ${e2e.container.name.prefix}-elasticsearch + + + es.port:9200 + + + + http://localhost:${es.port} + GET + 200 + + + + + single-node + + + + + zookeeper:${zookeeper.image.version} + ${e2e.container.name.prefix}-zookeeper + + + zk.port:2181 + + + binding to port + + + + + + skyapm/e2e-container:${e2e.container.version} + ${e2e.container.name.prefix} + + + cluster + elasticsearch + ${elasticsearch.version} + + ${e2e.container.name.prefix}-elasticsearch:9200 + + + ${e2e.container.name.prefix}-zookeeper:2181 + + + + ${provider.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800 + -DSW_AGENT_NAME=${provider.name} + -Dserver.port=9090 + + + + ${provider.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801 + -DSW_AGENT_NAME=${provider.name} + -Dserver.port=9091 + + + + ${consumer.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801 + -DSW_AGENT_NAME=${consumer.name} + -Dserver.port=9092 + + + + ${e2e.container.name.prefix}-elasticsearch + ${e2e.container.name.prefix}-zookeeper + + + +webapp.host:webapp.port:8081 + +service.host:service.port:9092 + + + ${e2e.container.name.prefix}-elasticsearch + ${e2e.container.name.prefix}-zookeeper + + + + ${sw.home}:/sw + + ../${provider.name}/target/${provider.name}-${project.version}.jar:/home/${provider.name}-${project.version}.jar + + + ../${consumer.name}/target/${consumer.name}-${project.version}.jar:/home/${consumer.name}-${project.version}.jar + + + ${project.basedir}/src/docker/rc.d:/rc.d:ro + + + ${project.basedir}/src/docker/clusterize.awk:/clusterize.awk + + + + + SkyWalking e2e container is ready for tests + + + + + + + + + + + + influxdb + + + + io.fabric8 + docker-maven-plugin + + %a-%t-%i + + + mysql/mysql-server:${mysql.version} + ${e2e.container.name.prefix}-mysql + + + Socket: '/var/run/mysqld/mysqlx.sock' bind-address: '::' port: 3306 + + + + root@1234 + swtest + % + + + mysql.port:3306 + + + + + influxdb:${influxdb.version} + ${e2e.container.name.prefix}-influxdb + + + influxdb.port:8086 + + + + http://${docker.host.address}:${influxdb.port}/ping + GET + + + + + + + zookeeper:${zookeeper.image.version} + ${e2e.container.name.prefix}-zookeeper + + + zk.port:2181 + + + binding to port + + + + + + skyapm/e2e-container:${e2e.container.version} + ${e2e.container.name.prefix} + + + cluster + influxdb + mysql + + jdbc:mysql://${e2e.container.name.prefix}-mysql:3306/swtest + + + http://${e2e.container.name.prefix}-influxdb:8086 + + + ${e2e.container.name.prefix}-zookeeper:2181 + + + + ${provider.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800 + -DSW_AGENT_NAME=${provider.name} + -Dserver.port=9090 + + + + ${provider.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801 + -DSW_AGENT_NAME=${provider.name} + -Dserver.port=9091 + + + + ${consumer.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801 + -DSW_AGENT_NAME=${consumer.name} + -Dserver.port=9092 + + + + ${e2e.container.name.prefix}-influxdb + ${e2e.container.name.prefix}-zookeeper + ${e2e.container.name.prefix}-mysql + + + +webapp.host:webapp.port:8081 + +service.host:service.port:9092 + + + ${e2e.container.name.prefix}-influxdb + ${e2e.container.name.prefix}-zookeeper + ${e2e.container.name.prefix}-mysql + + + + ${sw.home}:/sw + + ../${provider.name}/target/${provider.name}-${project.version}.jar:/home/${provider.name}-${project.version}.jar + + + ../${consumer.name}/target/${consumer.name}-${project.version}.jar:/home/${consumer.name}-${project.version}.jar + + + ${project.basedir}/src/docker/rc.d:/rc.d:ro + + + ${project.basedir}/src/docker/clusterize.awk:/clusterize.awk + + + + + SkyWalking e2e container is ready for tests + + + + + + + + + + + - - io.fabric8 - docker-maven-plugin - - %a-%t-%i - Always - - - elastic/elasticsearch:${elasticsearch.version} - ${e2e.container.name.prefix}-elasticsearch - - - es.port:9200 - - - - http://localhost:${es.port} - GET - 200 - - - - - single-node - - - - - zookeeper:${zookeeper.image.version} - ${e2e.container.name.prefix}-zookeeper - - - zk.port:2181 - - - binding to port - - - - - - skyapm/e2e-container:${e2e.container.version} - ${e2e.container.name.prefix} - - - cluster - ${elasticsearch.version} - - ${e2e.container.name.prefix}-elasticsearch:9200 - - - ${e2e.container.name.prefix}-zookeeper:2181 - - - - ${provider.name}-${project.version}.jar - - - -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800 - -DSW_AGENT_NAME=${provider.name} - -Dserver.port=9090 - - - - ${provider.name}-${project.version}.jar - - - -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801 - -DSW_AGENT_NAME=${provider.name} - -Dserver.port=9091 - - - - ${consumer.name}-${project.version}.jar - - - -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11801 - -DSW_AGENT_NAME=${consumer.name} - -Dserver.port=9092 - - - - ${e2e.container.name.prefix}-elasticsearch - ${e2e.container.name.prefix}-zookeeper - - - +webapp.host:webapp.port:8081 - +service.host:service.port:9092 - - - ${e2e.container.name.prefix}-elasticsearch - ${e2e.container.name.prefix}-zookeeper - - - - ${sw.home}:/sw - - ../${provider.name}/target/${provider.name}-${project.version}.jar:/home/${provider.name}-${project.version}.jar - - - ../${consumer.name}/target/${consumer.name}-${project.version}.jar:/home/${consumer.name}-${project.version}.jar - - - ${project.basedir}/src/docker/rc.d:/rc.d:ro - - - ${project.basedir}/src/docker/clusterize.awk:/clusterize.awk - - - - - SkyWalking e2e container is ready for tests - - - - - - - - org.apache.maven.plugins diff --git a/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/clusterize.awk b/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/clusterize.awk index 830ede77e2c6147b3bfbaf744dafaf5e451b618f..135b33fbfc7983429d8af71d991746ef725cb818 100644 --- a/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/clusterize.awk +++ b/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/clusterize.awk @@ -21,8 +21,8 @@ BEGIN { in_cluster_zk_section=0; in_storage_section=0; - in_storage_es_section=0; in_storage_h2_section=0; + in_storage_selected=0; } { @@ -60,23 +60,29 @@ BEGIN { } else if (in_storage_section == 1) { # in the storage: section now # disable h2 module - if (in_storage_es_section == 0) { - if (ENVIRON["ES_VERSION"] ~ /^6.+/) { - in_storage_es_section=$0 ~ /^#?\s+elasticsearch:$/ - } else if (ENVIRON["ES_VERSION"] ~ /^7.+/) { - in_storage_es_section=$0 ~ /^#?\s+elasticsearch7:$/ + if (in_storage_selected == 0) { + if (ENVIRON["STORAGE"] == "elasticsearch") { + if (ENVIRON["ES_VERSION"] ~ /^6.+/) { + in_storage_selected=$0 ~ /^#?\s+elasticsearch:$/ + } else if (ENVIRON["ES_VERSION"] ~ /^7.+/) { + in_storage_selected=$0 ~ /^#?\s+elasticsearch7:$/ + } + } else if (ENVIRON["STORAGE"] == "influxdb") { + in_storage_selected=$0 ~ /^#?\s+influx:$/ } } else { - in_storage_es_section=$0 ~ /^#?\s{4}/ + in_storage_selected=$0 ~ /^#?\s{4}/ } + if (in_storage_h2_section == 0) { in_storage_h2_section=$0 ~ /^#?\s+h2:$/ } else { in_storage_h2_section=$0 ~ /^#?\s{4}/ } - if (in_storage_es_section == 1) { - # in the storage.elasticsearch section now - # uncomment es config + + if (in_storage_selected == 1) { + # enable selected storage + # uncomment es/influx config gsub("^#", "", $0) print } else if (in_storage_h2_section == 1) { @@ -92,5 +98,4 @@ BEGIN { } else { print } -} - +} \ No newline at end of file diff --git a/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/rc.d/rc0-prepare.sh b/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/rc.d/rc0-prepare.sh index f50baca0212b137dd5050cc64e2451564173eda4..11739337f2cf152714dff3375b9103169aca5bde 100755 --- a/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/rc.d/rc0-prepare.sh +++ b/test/e2e/e2e-cluster/e2e-cluster-test-runner/src/docker/rc.d/rc0-prepare.sh @@ -15,7 +15,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -apt-get update && apt-get install -y gawk +if test "${SW_STORAGE_METABASE_TYPE}" = "mysql"; then + MYSQL_URL="https://repo.maven.apache.org/maven2/mysql/mysql-connector-java/8.0.13/mysql-connector-java-8.0.13.jar" + MYSQL_DRIVER="mysql-connector-java-8.0.13.jar" + + # Download MySQL connector. + curl -L -o "${SW_HOME}/oap-libs/${MYSQL_DRIVER}" ${MYSQL_URL} + [[ $? -ne 0 ]] && echo "Fail to download ${MYSQL_DRIVER}." && exit 1 +fi if test "${MODE}" = "cluster"; then original_wd=$(pwd) diff --git a/test/e2e/e2e-influxdb/pom.xml b/test/e2e/e2e-influxdb/pom.xml new file mode 100755 index 0000000000000000000000000000000000000000..0f44251318c7468c424d1955e983640431c47b8a --- /dev/null +++ b/test/e2e/e2e-influxdb/pom.xml @@ -0,0 +1,170 @@ + + + + + + apache-skywalking-e2e + org.apache.skywalking + 1.0.0 + + + 4.0.0 + + e2e-influxdb + + + skywalking-e2e-container-${build.id}-single-node-influxdb + + + + + org.springframework.boot + spring-boot-starter-data-jpa + ${spring.boot.version} + + + + com.h2database + h2 + ${h2.version} + + + + org.apache.skywalking + e2e-base + ${project.version} + + + + + + + org.springframework.boot + spring-boot-maven-plugin + ${spring.boot.version} + + true + true + true + + + + + repackage + + + + + + io.fabric8 + docker-maven-plugin + + %a-%t-%i + + + influxdb:${influxdb.version} + ${e2e.container.name.prefix}-influxdb + + + influxdb.port:8086 + + + + http://${docker.host.address}:${influxdb.port}/ping + GET + + + + + + + skyapm/e2e-container:${e2e.container.version} + ${e2e.container.name.prefix} + + + ${project.build.finalName}.jar + http://${e2e.container.name.prefix}-influxdb:8086 + + + ${e2e.container.name.prefix}-influxdb + + + ${e2e.container.name.prefix}-influxdb + + + +webapp.host:webapp.port:8081 + +client.host:client.port:9090 + + + + ${sw.home}:/sw + ${project.build.directory}:/home + ${project.basedir}/src/docker/rc.d:/rc.d:ro + ${project.basedir}/src/docker/application.yml:/application.yml + + + + + + http://${docker.host.address}:${client.port}/e2e/health-check + + GET + 200 + + + + + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + + ${webapp.host} + + + ${webapp.port} + + + ${client.host} + + + ${client.port} + + + + + + + verify + + + + + + + \ No newline at end of file diff --git a/test/e2e/e2e-influxdb/src/docker/application.yml b/test/e2e/e2e-influxdb/src/docker/application.yml new file mode 100644 index 0000000000000000000000000000000000000000..ff7b7cb4a0ed102cb2ed89b53256151ae8e963eb --- /dev/null +++ b/test/e2e/e2e-influxdb/src/docker/application.yml @@ -0,0 +1,232 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +cluster: + standalone: +# Please check your ZooKeeper is 3.5+, However, it is also compatible with ZooKeeper 3.4.x. Replace the ZooKeeper 3.5+ +# library the oap-libs folder with your ZooKeeper 3.4.x library. +# zookeeper: +# nameSpace: ${SW_NAMESPACE:""} +# hostPort: ${SW_CLUSTER_ZK_HOST_PORT:localhost:2181} +# #Retry Policy +# baseSleepTimeMs: ${SW_CLUSTER_ZK_SLEEP_TIME:1000} # initial amount of time to wait between retries +# maxRetries: ${SW_CLUSTER_ZK_MAX_RETRIES:3} # max number of times to retry +# # Enable ACL +# enableACL: ${SW_ZK_ENABLE_ACL:false} # disable ACL in default +# schema: ${SW_ZK_SCHEMA:digest} # only support digest schema +# expression: ${SW_ZK_EXPRESSION:skywalking:skywalking} +# kubernetes: +# watchTimeoutSeconds: ${SW_CLUSTER_K8S_WATCH_TIMEOUT:60} +# namespace: ${SW_CLUSTER_K8S_NAMESPACE:default} +# labelSelector: ${SW_CLUSTER_K8S_LABEL:app=collector,release=skywalking} +# uidEnvName: ${SW_CLUSTER_K8S_UID:SKYWALKING_COLLECTOR_UID} +# consul: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# Consul cluster nodes, example: 10.0.0.1:8500,10.0.0.2:8500,10.0.0.3:8500 +# hostPort: ${SW_CLUSTER_CONSUL_HOST_PORT:localhost:8500} +# nacos: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# # Nacos Configuration namespace +# namespace: ${SW_CLUSTER_NACOS_NAMESPACE:"public"} +# hostPort: ${SW_CLUSTER_NACOS_HOST_PORT:localhost:8848} +# etcd: +# serviceName: ${SW_SERVICE_NAME:"SkyWalking_OAP_Cluster"} +# etcd cluster nodes, example: 10.0.0.1:2379,10.0.0.2:2379,10.0.0.3:2379 +# hostPort: ${SW_CLUSTER_ETCD_HOST_PORT:localhost:2379} +core: + default: + # Mixed: Receive agent data, Level 1 aggregate, Level 2 aggregate + # Receiver: Receive agent data, Level 1 aggregate + # Aggregator: Level 2 aggregate + role: ${SW_CORE_ROLE:Mixed} # Mixed/Receiver/Aggregator + restHost: ${SW_CORE_REST_HOST:0.0.0.0} + restPort: ${SW_CORE_REST_PORT:12800} + restContextPath: ${SW_CORE_REST_CONTEXT_PATH:/} + gRPCHost: ${SW_CORE_GRPC_HOST:0.0.0.0} + gRPCPort: ${SW_CORE_GRPC_PORT:11800} + downsampling: + - Hour + - Day + - Month + # Set a timeout on metrics data. After the timeout has expired, the metrics data will automatically be deleted. + enableDataKeeperExecutor: ${SW_CORE_ENABLE_DATA_KEEPER_EXECUTOR:true} # Turn it off then automatically metrics data delete will be close. + dataKeeperExecutePeriod: ${SW_CORE_DATA_KEEPER_EXECUTE_PERIOD:5} # How often the data keeper executor runs periodically, unit is minute + recordDataTTL: ${SW_CORE_RECORD_DATA_TTL:90} # Unit is minute + minuteMetricsDataTTL: ${SW_CORE_MINUTE_METRIC_DATA_TTL:90} # Unit is minute + hourMetricsDataTTL: ${SW_CORE_HOUR_METRIC_DATA_TTL:36} # Unit is hour + dayMetricsDataTTL: ${SW_CORE_DAY_METRIC_DATA_TTL:45} # Unit is day + monthMetricsDataTTL: ${SW_CORE_MONTH_METRIC_DATA_TTL:18} # Unit is month + # Cache metric data for 1 minute to reduce database queries, and if the OAP cluster changes within that minute, + # the metrics may not be accurate within that minute. + enableDatabaseSession: ${SW_CORE_ENABLE_DATABASE_SESSION:true} +storage: +# elasticsearch: +# nameSpace: ${SW_NAMESPACE:""} +# clusterNodes: ${SW_STORAGE_ES_CLUSTER_NODES:localhost:9200} +# protocol: ${SW_STORAGE_ES_HTTP_PROTOCOL:"http"} +# #trustStorePath: ${SW_SW_STORAGE_ES_SSL_JKS_PATH:"../es_keystore.jks"} +# #trustStorePass: ${SW_SW_STORAGE_ES_SSL_JKS_PASS:""} +# user: ${SW_ES_USER:""} +# password: ${SW_ES_PASSWORD:""} +# indexShardsNumber: ${SW_STORAGE_ES_INDEX_SHARDS_NUMBER:2} +# indexReplicasNumber: ${SW_STORAGE_ES_INDEX_REPLICAS_NUMBER:0} +# # Those data TTL settings will override the same settings in core module. +# recordDataTTL: ${SW_STORAGE_ES_RECORD_DATA_TTL:7} # Unit is day +# otherMetricsDataTTL: ${SW_STORAGE_ES_OTHER_METRIC_DATA_TTL:45} # Unit is day +# monthMetricsDataTTL: ${SW_STORAGE_ES_MONTH_METRIC_DATA_TTL:18} # Unit is month +# # Batch process setting, refer to https://www.elastic.co/guide/en/elasticsearch/client/java-api/5.5/java-docs-bulk-processor.html +# bulkActions: ${SW_STORAGE_ES_BULK_ACTIONS:1000} # Execute the bulk every 1000 requests +# flushInterval: ${SW_STORAGE_ES_FLUSH_INTERVAL:10} # flush the bulk every 10 seconds whatever the number of requests +# concurrentRequests: ${SW_STORAGE_ES_CONCURRENT_REQUESTS:2} # the number of concurrent requests +# metadataQueryMaxSize: ${SW_STORAGE_ES_QUERY_MAX_SIZE:5000} +# segmentQueryMaxSize: ${SW_STORAGE_ES_QUERY_SEGMENT_SIZE:200} +# profileTaskQueryMaxSize: ${SW_STORAGE_ES_QUERY_PROFILE_TASK_SIZE:200} +# h2: +# driver: ${SW_STORAGE_H2_DRIVER:org.h2.jdbcx.JdbcDataSource} +# url: ${SW_STORAGE_H2_URL:jdbc:h2:mem:skywalking-oap-db} +# user: ${SW_STORAGE_H2_USER:sa} +# metadataQueryMaxSize: ${SW_STORAGE_H2_QUERY_MAX_SIZE:5000} +# mysql: +# properties: +# jdbcUrl: ${SW_JDBC_URL:"jdbc:mysql://localhost:3306/swtest"} +# dataSource.user: ${SW_DATA_SOURCE_USER:root} +# dataSource.password: ${SW_DATA_SOURCE_PASSWORD:root@1234} +# dataSource.cachePrepStmts: ${SW_DATA_SOURCE_CACHE_PREP_STMTS:true} +# dataSource.prepStmtCacheSize: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_SIZE:250} +# dataSource.prepStmtCacheSqlLimit: ${SW_DATA_SOURCE_PREP_STMT_CACHE_SQL_LIMIT:2048} +# dataSource.useServerPrepStmts: ${SW_DATA_SOURCE_USE_SERVER_PREP_STMTS:true} +# metadataQueryMaxSize: ${SW_STORAGE_MYSQL_QUERY_MAX_SIZE:5000} + influx: + # Metadata storage provider configuration + metabaseType: ${SW_STORAGE_METABASE_TYPE:H2} # There are 2 options as Metabase provider, H2 or MySQL. + h2Props: + dataSourceClassName: ${SW_STORAGE_METABASE_DRIVER:org.h2.jdbcx.JdbcDataSource} + dataSource.url: ${SW_STORAGE_METABASE_URL:jdbc:h2:mem:skywalking-oap-db} + dataSource.user: ${SW_STORAGE_METABASE_USER:sa} + dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:} + mysqlProps: + jdbcUrl: ${SW_STORAGE_METABASE_URL:"jdbc:mysql://localhost:3306/swtest"} + dataSource.user: ${SW_STORAGE_METABASE_USER:root} + dataSource.password: ${SW_STORAGE_METABASE_PASSWORD:root@1234} + dataSource.cachePrepStmts: ${SW_STORAGE_METABASE_CACHE_PREP_STMTS:true} + dataSource.prepStmtCacheSize: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_SIZE:250} + dataSource.prepStmtCacheSqlLimit: ${SW_STORAGE_METABASE_PREP_STMT_CACHE_SQL_LIMIT:2048} + dataSource.useServerPrepStmts: ${SW_STORAGE_METABASE_USE_SERVER_PREP_STMTS:true} + metadataQueryMaxSize: ${SW_STORAGE_METABASE_QUERY_MAX_SIZE:5000} + # InfluxDB configuration + url: ${SW_STORAGE_INFLUXDB_URL:http://localhost:8086} + user: ${SW_STORAGE_INFLUXDB_USER:root} + password: ${SW_STORAGE_INFLUXDB_PASSWORD:} + database: ${SW_STORAGE_INFLUXDB_DATABASE:skywalking} + actions: ${SW_STORAGE_INFLUXDB_ACTIONS:1000} # the number of actions to collect + duration: ${SW_STORAGE_INFLUXDB_DURATION:1000} # the time to wait at most (milliseconds) + fetchTaskLogMaxSize: ${SW_STORAGE_INFLUXDB_FETCH_TASK_LOG_MAX_SIZE:5000} # the max number of fetch task log in a request +receiver-sharing-server: + default: +receiver-register: + default: +receiver-trace: + default: + bufferPath: ${SW_RECEIVER_BUFFER_PATH:../trace-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_RECEIVER_BUFFER_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_RECEIVER_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_RECEIVER_BUFFER_FILE_CLEAN_WHEN_RESTART:false} + sampleRate: ${SW_TRACE_SAMPLE_RATE:10000} # The sample rate precision is 1/10000. 10000 means 100% sample in default. + slowDBAccessThreshold: ${SW_SLOW_DB_THRESHOLD:default:200,mongodb:100} # The slow database access thresholds. Unit ms. +receiver-jvm: + default: +receiver-clr: + default: +#receiver-so11y: +# default: +receiver-profile: + default: +service-mesh: + default: + bufferPath: ${SW_SERVICE_MESH_BUFFER_PATH:../mesh-buffer/} # Path to trace buffer files, suggest to use absolute path + bufferOffsetMaxFileSize: ${SW_SERVICE_MESH_OFFSET_MAX_FILE_SIZE:100} # Unit is MB + bufferDataMaxFileSize: ${SW_SERVICE_MESH_BUFFER_DATA_MAX_FILE_SIZE:500} # Unit is MB + bufferFileCleanWhenRestart: ${SW_SERVICE_MESH_BUFFER_FILE_CLEAN_WHEN_RESTART:false} +istio-telemetry: + default: +envoy-metric: + default: +# alsHTTPAnalysis: ${SW_ENVOY_METRIC_ALS_HTTP_ANALYSIS:k8s-mesh} +#receiver_zipkin: +# default: +# host: ${SW_RECEIVER_ZIPKIN_HOST:0.0.0.0} +# port: ${SW_RECEIVER_ZIPKIN_PORT:9411} +# contextPath: ${SW_RECEIVER_ZIPKIN_CONTEXT_PATH:/} +#receiver_jaeger: +# default: +# gRPCHost: ${SW_RECEIVER_JAEGER_HOST:0.0.0.0} +# gRPCPort: ${SW_RECEIVER_JAEGER_PORT:14250} +query: + graphql: + path: ${SW_QUERY_GRAPHQL_PATH:/graphql} +alarm: + default: +telemetry: + none: +# prometheus: +# host: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0} +# port: ${SW_TELEMETRY_PROMETHEUS_PORT:1234} +# so11y: +# prometheusExporterEnabled: ${SW_TELEMETRY_SO11Y_PROMETHEUS_ENABLED:true} +# prometheusExporterHost: ${SW_TELEMETRY_PROMETHEUS_HOST:0.0.0.0} +# prometheusExporterPort: ${SW_TELEMETRY_PROMETHEUS_PORT:1234} +configuration: + none: +# apollo: +# apolloMeta: http://106.12.25.204:8080 +# apolloCluster: default +# # apolloEnv: # defaults to null +# appId: skywalking +# period: 5 +# nacos: +# # Nacos Server Host +# serverAddr: 127.0.0.1 +# # Nacos Server Port +# port: 8848 +# # Nacos Configuration Group +# group: 'skywalking' +# # Nacos Configuration namespace +# namespace: '' +# # Unit seconds, sync period. Default fetch every 60 seconds. +# period : 5 +# # the name of current cluster, set the name if you want to upstream system known. +# clusterName: "default" +# zookeeper: +# period : 60 # Unit seconds, sync period. Default fetch every 60 seconds. +# nameSpace: /default +# hostPort: localhost:2181 +# #Retry Policy +# baseSleepTimeMs: 1000 # initial amount of time to wait between retries +# maxRetries: 3 # max number of times to retry +# etcd: +# period : 60 # Unit seconds, sync period. Default fetch every 60 seconds. +# group : 'skywalking' +# serverAddr: localhost:2379 +# clusterName: "default" +# consul: +# # Consul host and ports, separated by comma, e.g. 1.2.3.4:8500,2.3.4.5:8500 +# hostAndPorts: ${consul.address} +# # Sync period in seconds. Defaults to 60 seconds. +# period: 1 + +#exporter: +# grpc: +# targetHost: ${SW_EXPORTER_GRPC_HOST:127.0.0.1} +# targetPort: ${SW_EXPORTER_GRPC_PORT:9870} diff --git a/test/e2e/e2e-influxdb/src/docker/rc.d/rc0-prepare.sh b/test/e2e/e2e-influxdb/src/docker/rc.d/rc0-prepare.sh new file mode 100755 index 0000000000000000000000000000000000000000..a926869733e8f271a5a84721c5b7bf1c68977177 --- /dev/null +++ b/test/e2e/e2e-influxdb/src/docker/rc.d/rc0-prepare.sh @@ -0,0 +1,21 @@ +#!/usr/bin/env bash +# Licensed to the SkyAPM under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo "InfluxDB with H2 database is storage provider..." + +# Modify application.yml to set InfluxDB as storage provider. +cat /application.yml > "${SW_HOME}/config/application.yml" diff --git a/test/e2e/e2e-influxdb/src/docker/rc.d/rc1-startup.sh b/test/e2e/e2e-influxdb/src/docker/rc.d/rc1-startup.sh new file mode 100755 index 0000000000000000000000000000000000000000..f9382e4343c619048596875fddf933d8d5cfd38b --- /dev/null +++ b/test/e2e/e2e-influxdb/src/docker/rc.d/rc1-startup.sh @@ -0,0 +1,40 @@ +#!/usr/bin/env bash +# Licensed to the SkyAPM under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo 'starting OAP server...' && start_oap 'init' + +echo 'starting Web app...' && start_webapp '0.0.0.0' 8081 + +echo 'starting instrumented services...' && start_instrumented_services + +check_tcp 127.0.0.1 \ + 9090 \ + 60 \ + 10 \ + "waiting for the instrumented service to be ready" + +if [[ $? -ne 0 ]]; then + echo "instrumented service 0 failed to start in 30 * 10 seconds: " + cat ${SERVICE_LOG}/* + exit 1 +fi + +echo "SkyWalking e2e container is ready for tests" + +tail -f ${OAP_LOG_DIR}/* \ + ${WEBAPP_LOG_DIR}/* \ + ${SERVICE_LOG}/* diff --git a/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/SampleClientApplication.java b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/SampleClientApplication.java new file mode 100644 index 0000000000000000000000000000000000000000..67ba54eedfd4d20b8d1a5a1ac04b34a0b40237b7 --- /dev/null +++ b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/SampleClientApplication.java @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.sample.client; + +import org.springframework.boot.SpringApplication; +import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.data.jpa.repository.config.EnableJpaRepositories; + +@EnableJpaRepositories +@SpringBootApplication +public class SampleClientApplication { + public static void main(String[] args) { + SpringApplication.run(SampleClientApplication.class, args); + } +} diff --git a/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/TestController.java b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/TestController.java new file mode 100644 index 0000000000000000000000000000000000000000..1a344ab45c45e49cc59b955044cb753561cedd6d --- /dev/null +++ b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/TestController.java @@ -0,0 +1,46 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.sample.client; + +import org.springframework.web.bind.annotation.GetMapping; +import org.springframework.web.bind.annotation.PostMapping; +import org.springframework.web.bind.annotation.RequestBody; +import org.springframework.web.bind.annotation.RequestMapping; +import org.springframework.web.bind.annotation.RestController; + +@RestController +@RequestMapping("/e2e") +public class TestController { + private final UserRepo userRepo; + + public TestController(final UserRepo userRepo) { + this.userRepo = userRepo; + } + + @GetMapping("/health-check") + public String hello() { + return "healthy"; + } + + @PostMapping("/users") + public User createAuthor(@RequestBody final User user) throws InterruptedException { + Thread.sleep(1000L); + return userRepo.save(user); + } +} diff --git a/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/User.java b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/User.java new file mode 100644 index 0000000000000000000000000000000000000000..3b6c317440dee380c2b9b7a1104d0892ae7c00de --- /dev/null +++ b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/User.java @@ -0,0 +1,53 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.sample.client; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.GeneratedValue; +import javax.persistence.Id; + +@Entity +public class User { + public User() { + } + + @Id + @GeneratedValue + private Long id; + + @Column + private String name; + + public Long getId() { + return id; + } + + public void setId(final Long id) { + this.id = id; + } + + public String getName() { + return name; + } + + public void setName(final String name) { + this.name = name; + } +} diff --git a/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/UserRepo.java b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/UserRepo.java new file mode 100644 index 0000000000000000000000000000000000000000..2ee38a60dd3cdf15d96d4668c61591d66528679d --- /dev/null +++ b/test/e2e/e2e-influxdb/src/main/java/org/apache/skywalking/e2e/sample/client/UserRepo.java @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e.sample.client; + +import org.springframework.data.jpa.repository.JpaRepository; + +public interface UserRepo extends JpaRepository { +} diff --git a/test/e2e/e2e-influxdb/src/main/resources/application.yml b/test/e2e/e2e-influxdb/src/main/resources/application.yml new file mode 100644 index 0000000000000000000000000000000000000000..ef3ed01cd41127298e112d5fad0dae7714aadc3d --- /dev/null +++ b/test/e2e/e2e-influxdb/src/main/resources/application.yml @@ -0,0 +1,35 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server: + port: 9090 + +spring: + main: + banner-mode: 'off' + datasource: + url: jdbc:h2:mem:testdb + driver-class-name: org.h2.Driver + data-username: sa + password: sa + platform: org.hibernate.dialect.H2Dialect + jpa: + generate-ddl: true + hibernate: + ddl-auto: create-drop + properties: + hibernate.format_sql: true + show-sql: true diff --git a/test/e2e/e2e-influxdb/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java b/test/e2e/e2e-influxdb/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java new file mode 100644 index 0000000000000000000000000000000000000000..81944b7313f3bb99daafd8d33c507df9644c545e --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/java/org/apache/skywalking/e2e/SampleVerificationITCase.java @@ -0,0 +1,369 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + +package org.apache.skywalking.e2e; + +import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.Metrics; +import org.apache.skywalking.e2e.metrics.MetricsQuery; +import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesMatcher; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoint; +import org.apache.skywalking.e2e.service.endpoint.EndpointQuery; +import org.apache.skywalking.e2e.service.endpoint.Endpoints; +import org.apache.skywalking.e2e.service.endpoint.EndpointsMatcher; +import org.apache.skywalking.e2e.service.instance.Instance; +import org.apache.skywalking.e2e.service.instance.Instances; +import org.apache.skywalking.e2e.service.instance.InstancesMatcher; +import org.apache.skywalking.e2e.service.instance.InstancesQuery; +import org.apache.skywalking.e2e.topo.*; +import org.apache.skywalking.e2e.trace.Trace; +import org.apache.skywalking.e2e.trace.TracesMatcher; +import org.apache.skywalking.e2e.trace.TracesQuery; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.core.io.ClassPathResource; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; +import org.springframework.test.annotation.DirtiesContext; +import org.springframework.test.context.junit4.SpringJUnit4ClassRunner; +import org.springframework.web.client.RestTemplate; +import org.yaml.snakeyaml.Yaml; + +import java.io.InputStream; +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import static org.apache.skywalking.e2e.metrics.MetricsMatcher.verifyMetrics; +import static org.apache.skywalking.e2e.metrics.MetricsQuery.*; +import static org.assertj.core.api.Assertions.assertThat; + +@RunWith(SpringJUnit4ClassRunner.class) +public class SampleVerificationITCase { + private static final Logger LOGGER = LoggerFactory.getLogger(SampleVerificationITCase.class); + + private final RestTemplate restTemplate = new RestTemplate(); + private final int retryInterval = 30; + + private SimpleQueryClient queryClient; + private String instrumentedServiceUrl; + + @Before + public void setUp() { + final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1"); + final String swWebappPort = System.getProperty("sw.webapp.port", "32783"); + final String instrumentedServiceHost = System.getProperty("client.host", "127.0.0.1"); + final String instrumentedServicePort = System.getProperty("client.port", "32782"); + queryClient = new SimpleQueryClient(swWebappHost, swWebappPort); + instrumentedServiceUrl = "http://" + instrumentedServiceHost + ":" + instrumentedServicePort; + } + + @Test(timeout = 1200000) + @DirtiesContext + public void verify() throws Exception { + final LocalDateTime minutesAgo = LocalDateTime.now(ZoneOffset.UTC); + + while (true) { + try { + final Map user = new HashMap<>(); + user.put("name", "SkyWalking"); + final ResponseEntity responseEntity = restTemplate.postForEntity( + instrumentedServiceUrl + "/e2e/users", + user, + String.class + ); + LOGGER.info("responseEntity: {}", responseEntity); + assertThat(responseEntity.getStatusCode()).isEqualTo(HttpStatus.OK); + final List traces = queryClient.traces( + new TracesQuery() + .start(minutesAgo) + .end(LocalDateTime.now()) + .orderByDuration() + ); + if (!traces.isEmpty()) { + break; + } + Thread.sleep(10000L); + } catch (Exception ignored) { + } + } + + doRetryableVerification(() -> { + try { + verifyTraces(minutesAgo); + } catch (Exception e) { + LOGGER.warn(e.getMessage(), e); + } + }); + + doRetryableVerification(() -> { + try { + verifyServices(minutesAgo); + } catch (Exception e) { + LOGGER.warn(e.getMessage(), e); + } + }); + + doRetryableVerification(() -> { + try { + verifyTopo(minutesAgo); + } catch (Exception e) { + LOGGER.warn(e.getMessage(), e); + } + }); + + doRetryableVerification(() -> { + try{ + verifyServiceInstanceTopo(minutesAgo); + }catch (Exception e) { + LOGGER.warn(e.getMessage(), e); + } + }); + } + + private void verifyTopo(LocalDateTime minutesAgo) throws Exception { + final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); + + final TopoData topoData = queryClient.topo( + new TopoQuery() + .stepByMinute() + .start(minutesAgo.minusDays(1)) + .end(now) + ); + LOGGER.info("topoData: {}", topoData); + + InputStream expectedInputStream = + new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.topo.yml").getInputStream(); + + final TopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, TopoMatcher.class); + topoMatcher.verify(topoData); + verifyServiceRelationMetrics(topoData.getCalls(), minutesAgo); + } + + private void verifyServiceInstanceTopo(LocalDateTime minutesAgo) throws Exception { + final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); + + final ServiceInstanceTopoData topoData = queryClient.serviceInstanceTopo( + new ServiceInstanceTopoQuery() + .stepByMinute() + .start(minutesAgo.minusDays(1)) + .end(now) + .clientServiceId("1") + .serverServiceId("2") + ); + LOGGER.info("instanceTopoData: {}", topoData); + + InputStream expectedInputStream = + new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml").getInputStream(); + + final ServiceInstanceTopoMatcher topoMatcher = new Yaml().loadAs(expectedInputStream, ServiceInstanceTopoMatcher.class); + topoMatcher.verify(topoData); + verifyServiceInstanceRelationMetrics(topoData.getCalls(), minutesAgo); + } + + private void verifyServices(LocalDateTime minutesAgo) throws Exception { + final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); + + final List services = queryClient.services( + new ServicesQuery() + .start(minutesAgo) + .end(now) + ); + LOGGER.info("services: {}", services); + + InputStream expectedInputStream = + new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.services.yml").getInputStream(); + + final ServicesMatcher servicesMatcher = new Yaml().loadAs(expectedInputStream, ServicesMatcher.class); + servicesMatcher.verify(services); + + for (Service service : services) { + LOGGER.info("verifying service instances: {}", service); + + verifyServiceMetrics(service); + + Instances instances = verifyServiceInstances(minutesAgo, now, service); + + verifyInstancesMetrics(instances); + + Endpoints endpoints = verifyServiceEndpoints(minutesAgo, now, service); + + verifyEndpointsMetrics(endpoints); + } + } + + private Instances verifyServiceInstances(LocalDateTime minutesAgo, LocalDateTime now, + Service service) throws Exception { + InputStream expectedInputStream; + Instances instances = queryClient.instances( + new InstancesQuery() + .serviceId(service.getKey()) + .start(minutesAgo) + .end(now) + ); + LOGGER.info("instances: {}", instances); + expectedInputStream = + new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.instances.yml").getInputStream(); + final InstancesMatcher instancesMatcher = new Yaml().loadAs(expectedInputStream, InstancesMatcher.class); + instancesMatcher.verify(instances); + return instances; + } + + private Endpoints verifyServiceEndpoints(LocalDateTime minutesAgo, LocalDateTime now, + Service service) throws Exception { + Endpoints instances = queryClient.endpoints( + new EndpointQuery().serviceId(service.getKey()) + ); + LOGGER.info("instances: {}", instances); + InputStream expectedInputStream = + new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.endpoints.yml").getInputStream(); + final EndpointsMatcher endpointsMatcher = new Yaml().loadAs(expectedInputStream, EndpointsMatcher.class); + endpointsMatcher.verify(instances); + return instances; + } + + private void verifyInstancesMetrics(Instances instances) throws Exception { + for (Instance instance : instances.getInstances()) { + for (String metricsName : ALL_INSTANCE_METRICS) { + LOGGER.info("verifying service instance response time: {}", instance); + final Metrics instanceMetrics = queryClient.metrics( + new MetricsQuery() + .stepByMinute() + .metricsName(metricsName) + .id(instance.getKey()) + ); + LOGGER.info("instanceMetrics: {}", instanceMetrics); + AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(instanceMetrics); + LOGGER.info("{}: {}", metricsName, instanceMetrics); + } + } + } + + private void verifyEndpointsMetrics(Endpoints endpoints) throws Exception { + for (Endpoint endpoint : endpoints.getEndpoints()) { + if (!endpoint.getLabel().equals("/e2e/users")) { + continue; + } + for (String metricName : ALL_ENDPOINT_METRICS) { + LOGGER.info("verifying endpoint {}, metrics: {}", endpoint, metricName); + final Metrics metrics = queryClient.metrics( + new MetricsQuery() + .stepByMinute() + .metricsName(metricName) + .id(endpoint.getKey()) + ); + LOGGER.info("metrics: {}", metrics); + AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(metrics); + LOGGER.info("{}: {}", metricName, metrics); + } + } + } + + private void verifyServiceMetrics(Service service) throws Exception { + for (String metricName : ALL_SERVICE_METRICS) { + LOGGER.info("verifying service {}, metrics: {}", service, metricName); + final Metrics serviceMetrics = queryClient.metrics( + new MetricsQuery() + .stepByMinute() + .metricsName(metricName) + .id(service.getKey()) + ); + LOGGER.info("serviceMetrics: {}", serviceMetrics); + AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + instanceRespTimeMatcher.verify(serviceMetrics); + LOGGER.info("{}: {}", metricName, serviceMetrics); + } + } + + private void verifyTraces(LocalDateTime minutesAgo) throws Exception { + final LocalDateTime now = LocalDateTime.now(ZoneOffset.UTC); + + final List traces = queryClient.traces( + new TracesQuery() + .start(minutesAgo) + .end(now) + .orderByDuration() + ); + LOGGER.info("traces: {}", traces); + + InputStream expectedInputStream = + new ClassPathResource("expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.traces.yml").getInputStream(); + + final TracesMatcher tracesMatcher = new Yaml().loadAs(expectedInputStream, TracesMatcher.class); + tracesMatcher.verifyLoosely(traces); + } + + private void verifyServiceInstanceRelationMetrics(List calls, final LocalDateTime minutesAgo) throws Exception { + verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_INSTANCE_RELATION_CLIENT_METRICS, ALL_SERVICE_INSTANCE_RELATION_SERVER_METRICS); + } + + private void verifyServiceRelationMetrics(List calls, final LocalDateTime minutesAgo) throws Exception { + verifyRelationMetrics(calls, minutesAgo, ALL_SERVICE_RELATION_CLIENT_METRICS, ALL_SERVICE_RELATION_SERVER_METRICS); + } + + private void verifyRelationMetrics(List calls, final LocalDateTime minutesAgo, String[] relationClientMetrics, String[] relationServerMetrics) throws Exception { + for (Call call : calls) { + for (String detectPoint : call.getDetectPoints()) { + switch (detectPoint) { + case "CLIENT": { + for (String metricName : relationClientMetrics) { + verifyMetrics(queryClient, metricName, call.getId(), minutesAgo); + } + break; + } + case "SERVER": { + for (String metricName : relationServerMetrics) { + verifyMetrics(queryClient, metricName, call.getId(), minutesAgo); + } + break; + } + } + } + } + } + + private void doRetryableVerification(Runnable runnable) throws InterruptedException { + while (true) { + try { + runnable.run(); + break; + } catch (Throwable ignored) { + Thread.sleep(retryInterval); + } + } + } +} diff --git a/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.endpoints.yml b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.endpoints.yml new file mode 100644 index 0000000000000000000000000000000000000000..a1f5b450af380a8b16eda893abf5d39d6adb0cb1 --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.endpoints.yml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 health-check by docker-maven-plugin +# 1 drop table if exists, because we have `ddl-auto: create-drop` +# 1 drop sequence +# 1 create sequence +# 1 create table statement + +endpoints: + - key: not null + label: /e2e/health-check + - key: not null + label: /e2e/users diff --git a/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.instances.yml b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.instances.yml new file mode 100644 index 0000000000000000000000000000000000000000..26bb314d2b5d486a486b158349d80cde573e77a6 --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.instances.yml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 health-check by docker-maven-plugin +# 1 drop table if exists, because we have `ddl-auto: create-drop` +# 1 drop sequence +# 1 create sequence +# 1 create table statement + +instances: + - key: 2 + label: not null + attributes: + - name: os_name + value: not null + - name: host_name + value: not null + - name: process_no + value: gt 0 + - name: ipv4s + value: not null diff --git a/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml new file mode 100644 index 0000000000000000000000000000000000000000..d312bd8a78b0047a070151c846b975917cda844a --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.serviceInstanceTopo.yml @@ -0,0 +1,41 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 health-check by docker-maven-plugin +# 1 drop table if exists, because we have `ddl-auto: create-drop` +# 1 drop sequence +# 1 create sequence +# 1 create table statement + +nodes: + - id: 1 + name: User + type: USER + serviceId: 1 + serviceName: User + isReal: false + - id: 2 + name: not null + serviceId: 2 + serviceName: Your_ApplicationName + type: Tomcat + isReal: true +calls: + - id: 1_2 + source: 1 + detectPoints: + - SERVER + target: 2 diff --git a/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.services.yml b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.services.yml new file mode 100644 index 0000000000000000000000000000000000000000..07ff835878d8a6f111bce16ec743236184814220 --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.services.yml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 health-check by docker-maven-plugin +# 1 drop table if exists, because we have `ddl-auto: create-drop` +# 1 drop sequence +# 1 create sequence +# 1 create table statement + +services: + - key: 2 + label: "Your_ApplicationName" diff --git a/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.topo.yml b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.topo.yml new file mode 100644 index 0000000000000000000000000000000000000000..3a455f5b60c8f29b41989fb8f2b5e5016fb9d249 --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.topo.yml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 health-check by docker-maven-plugin +# 1 drop table if exists, because we have `ddl-auto: create-drop` +# 1 drop sequence +# 1 create sequence +# 1 create table statement + +nodes: + - id: 1 + name: User + type: USER + isReal: false + - id: 2 + name: Your_ApplicationName + type: Tomcat + isReal: true + - id: 3 + name: "localhost:-1" + type: H2 + isReal: false +calls: + - id: 2_3 + source: 2 + detectPoints: + - CLIENT + target: 3 + - id: 1_2 + source: 1 + detectPoints: + - SERVER + target: 2 diff --git a/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.traces.yml b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.traces.yml new file mode 100644 index 0000000000000000000000000000000000000000..2052aad3acd3776b972a62b321a77064cc278e4e --- /dev/null +++ b/test/e2e/e2e-influxdb/src/test/resources/expected-data/org.apache.skywalking.e2e.SampleVerificationITCase.traces.yml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# 1 health-check by docker-maven-plugin +# 1 drop table if exists, because we have `ddl-auto: create-drop` +# 1 drop sequence +# 1 create sequence +# 1 create table statement + +traces: + - key: not null + endpointNames: + - /e2e/users + duration: ge 0 + start: gt 0 + isError: false + traceIds: + - not null diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/pom.xml b/test/e2e/e2e-profile/e2e-profile-test-runner/pom.xml index 18dea8011e52c46956e65b56def205eb659e004a..daa5ed469a42d2a29b351981650af354cdedf619 100644 --- a/test/e2e/e2e-profile/e2e-profile-test-runner/pom.xml +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/pom.xml @@ -290,6 +290,82 @@ + + + + influxdb + + + + io.fabric8 + docker-maven-plugin + + %a-%t-%i + + + influxdb:${influxdb.version} + ${e2e.container.name.prefix}-influxdb + + + influxdb.port:8086 + + + + http://${docker.host.address}:${influxdb.port}/ping + GET + + + + + + + skyapm/e2e-container:${e2e.container.version} + ${e2e.container.name.prefix} + + + influxdb + http://${e2e.container.name.prefix}-influxdb:8086 + + ${provider.name}-${project.version}.jar + + + -DSW_AGENT_COLLECTOR_BACKEND_SERVICES=127.0.0.1:11800 + -DSW_AGENT_PROFILE_ACTIVE=true + -DSW_AGENT_NAME=${provider.name} + -Dserver.port=9090 + + + + ${e2e.container.name.prefix}-influxdb + + + +webapp.host:webapp.port:8081 + +service.host:service.port:9090 + + + ${e2e.container.name.prefix}-influxdb + + + + ${sw.home}:/sw + ../${provider.name}/target/${provider.name}-${project.version}.jar:/home/${provider.name}-${project.version}.jar + ${project.basedir}/src/docker/rc.d:/rc.d:ro + ${project.basedir}/src/docker/adapt_storage.awk:/adapt_storage.awk + ${project.basedir}/src/docker/profile_official_analysis.oal:/profile_official_analysis.oal + + + + SkyWalking e2e container is ready for tests + + + + + + + + + + @@ -318,5 +394,4 @@ - \ No newline at end of file diff --git a/test/e2e/e2e-profile/e2e-profile-test-runner/src/docker/adapt_storage.awk b/test/e2e/e2e-profile/e2e-profile-test-runner/src/docker/adapt_storage.awk index bf05c3010091d84bc1556d82fef447e882fc9b4a..f9c2b0e1f7e1e7ad07aa849ad54eba9112993180 100644 --- a/test/e2e/e2e-profile/e2e-profile-test-runner/src/docker/adapt_storage.awk +++ b/test/e2e/e2e-profile/e2e-profile-test-runner/src/docker/adapt_storage.awk @@ -36,9 +36,11 @@ BEGIN { } else if (ENVIRON["ES_VERSION"] ~ /^7.+/) { in_storage_type_section=$0 ~ /^#?\s+elasticsearch7:$/ } else if (ENVIRON["STORAGE"] ~ /^mysql.*$/) { - in_storage_type_section=$0 ~ /^#?\s+mysql/ + in_storage_type_section=$0 ~ /^#?\s+mysql:/ } else if (ENVIRON["STORAGE"] ~ /^h2.*$/) { in_storage_type_section=$0 ~ /^#?\s+h2:$/ + } else if (ENVIRON["STORAGE"] ~ /^influx.*$/) { + in_storage_type_section=$0 ~ /^#?\s+influx:$/ } } else { in_storage_type_section=$0 ~ /^#?\s{4}/ diff --git a/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh index 1ded6dd1b878d53fe815bac9f42c898279fccd7f..2bf08622df1384a353a259bf48f5237c17857bd1 100755 --- a/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh +++ b/test/e2e/e2e-ttl/e2e-ttl-es/src/docker/rc.d/rc0-prepare.sh @@ -15,8 +15,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -apt-get update && apt-get install -y gawk - original_wd=$(pwd) # substitute application.yml to be capable of es mode diff --git a/test/e2e/e2e-ttl/e2e-ttl-influxdb/pom.xml b/test/e2e/e2e-ttl/e2e-ttl-influxdb/pom.xml new file mode 100644 index 0000000000000000000000000000000000000000..d407446d010a488e4182548ebe5a7c86df22e188 --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-influxdb/pom.xml @@ -0,0 +1,150 @@ + + + + + + e2e-ttl + org.apache.skywalking + 1.0.0 + + 4.0.0 + + e2e-ttl-influxdb + + + dist-for-ttl-influxdb + 1.2 + skywalking-e2e-container-${build.id}-ttl-influxdb + 1.14.0 + 1.7.9 + 1.5.0.Final + 0.5.0 + + + + + org.apache.skywalking + e2e-base + ${project.version} + + + org.apache.skywalking + e2e-protocol + ${project.version} + + + + + + + kr.motd.maven + os-maven-plugin + ${os-maven-plugin.version} + + + initialize + + detect + + + + + + io.fabric8 + docker-maven-plugin + + %a-%t-%i + + + influxdb:${influxdb.version} + ${e2e.container.name.prefix}-influxdb + + + influxdb.port:8086 + + + + http://${docker.host.address}:${influxdb.port}/ping + GET + + + + + + + skyapm/e2e-container:${e2e.container.version} + ${e2e.container.name.prefix} + + + standalone + http://${e2e.container.name.prefix}-influxdb:8086 + + + ${e2e.container.name.prefix}-influxdb + + + +webapp.host:webapp.port:8081 + +oap.host:oap.port:11800 + + + ${e2e.container.name.prefix}-influxdb + + + + ${sw.home}:/sw + ${project.basedir}/src/docker/rc.d:/rc.d:ro + ${project.basedir}/src/docker/influx_storage.awk:/influx_storage.awk:ro + ${project.basedir}/src/docker/ttl_official_analysis.oal:/ttl_official_analysis.oal + + + + SkyWalking e2e container is ready for tests + + + + + + + + + + + org.apache.maven.plugins + maven-failsafe-plugin + + + ${webapp.host} + ${webapp.port} + ${oap.host} + ${oap.port} + + + + + + verify + + + + + + + + \ No newline at end of file diff --git a/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/influx_storage.awk b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/influx_storage.awk new file mode 100644 index 0000000000000000000000000000000000000000..28ece11827f269c70d2b7de87e959d940c54bc41 --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/influx_storage.awk @@ -0,0 +1,64 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +#!/usr/bin/awk -f + +BEGIN { + in_storage_section=0; + in_storage_influx_section=0; + in_storage_h2_section=0; +} + +{ + if (in_storage_section == 0) { + in_storage_section=$0 ~ /^storage:$/ + } else { + in_storage_section=$0 ~ /^(#|\s{2})/ + } + + if (in_storage_section == 1) { + # in the storage: section now + # disable h2 module + if (in_storage_influx_section == 0) { + in_storage_influx_section=$0 ~ /^#?\s+influx:$/ + } else { + in_storage_influx_section=$0 ~ /^#?\s{4}/ + } + if (in_storage_h2_section == 0) { + in_storage_h2_section=$0 ~ /^#?\s+h2:$/ + } else { + in_storage_h2_section=$0 ~ /^#?\s{4}/ + } + if (in_storage_influx_section == 1) { + # in the storage.influxdb section now + # uncomment influx config + gsub("^#", "", $0) + print + } else if (in_storage_h2_section == 1) { + # comment out h2 config + if ($0 !~ /^#/) { + print "#" $0 + } else { + print + } + } else { + print + } + } else { + print + } +} + diff --git a/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/rc.d/rc0-prepare.sh b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/rc.d/rc0-prepare.sh new file mode 100755 index 0000000000000000000000000000000000000000..e79e59b0054a4b15c0d51cbcbe247345168b1f5e --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/rc.d/rc0-prepare.sh @@ -0,0 +1,28 @@ +#!/usr/bin/env bash +# Licensed to the SkyAPM under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +original_wd=$(pwd) + +# substitute application.yml to be capable of es mode +cd ${SW_HOME}/config \ + && gawk -f /influx_storage.awk application.yml > influx_storage_app.yml \ + && mv influx_storage_app.yml application.yml \ + && cp /ttl_official_analysis.oal official_analysis.oal \ + && sed '//a' log4j2.xml > log4j2debuggable.xml \ + && mv log4j2debuggable.xml log4j2.xml + +cd ${original_wd} \ No newline at end of file diff --git a/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/rc.d/rc1-startup.sh b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/rc.d/rc1-startup.sh new file mode 100755 index 0000000000000000000000000000000000000000..4a9fd4cd06630e6dd34362a865ba2234d4607b7e --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/rc.d/rc1-startup.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +# Licensed to the SkyAPM under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +echo 'starting OAP server...' \ + && SW_CORE_DATA_KEEPER_EXECUTE_PERIOD=1 \ + SW_CORE_RECORD_DATA_TTL=7 \ + SW_CORE_MINUTE_METRIC_DATA_TTL=6000 \ + SW_CORE_HOUR_METRIC_DATA_TTL=100 \ + SW_CORE_DAY_METRIC_DATA_TTL=5 \ + SW_CORE_MONTH_METRIC_DATA_TTL=4 \ + SW_RECEIVER_BUFFER_PATH=/tmp/oap/trace_buffer1 \ + SW_SERVICE_MESH_BUFFER_PATH=/tmp/oap/mesh_buffer1 \ + start_oap 'init' + +echo 'starting Web app...' \ + && start_webapp '0.0.0.0' 8081 + +echo "SkyWalking e2e container is ready for tests" + +tail -f ${OAP_LOG_DIR}/* \ + ${WEBAPP_LOG_DIR}/* diff --git a/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/ttl_official_analysis.oal b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/ttl_official_analysis.oal new file mode 100755 index 0000000000000000000000000000000000000000..9353579058a8ee10bde6603c90ca29d321cde2e1 --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/docker/ttl_official_analysis.oal @@ -0,0 +1,23 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + * + */ + + +// In TTL ES7 testing, using full oal config makes the test very unstable, so only service_p99 is reserved for testing. + +// Service scope metrics +service_resp_time = from(Service.latency).longAvg(); \ No newline at end of file diff --git a/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java new file mode 100644 index 0000000000000000000000000000000000000000..4901ccc7f1ce5d586bc0ffd2ce62a3804c63423c --- /dev/null +++ b/test/e2e/e2e-ttl/e2e-ttl-influxdb/src/test/java/org/apache/skywalking/e2e/StorageTTLITCase.java @@ -0,0 +1,285 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.skywalking.e2e; + +import io.grpc.ManagedChannel; +import io.grpc.ManagedChannelBuilder; +import io.grpc.internal.DnsNameResolverProvider; +import io.grpc.netty.NettyChannelBuilder; +import io.grpc.stub.StreamObserver; +import lombok.extern.slf4j.Slf4j; +import org.apache.skywalking.apm.network.common.DetectPoint; +import org.apache.skywalking.apm.network.servicemesh.MeshProbeDownstream; +import org.apache.skywalking.apm.network.servicemesh.Protocol; +import org.apache.skywalking.apm.network.servicemesh.ServiceMeshMetric; +import org.apache.skywalking.apm.network.servicemesh.ServiceMeshMetricServiceGrpc; +import org.apache.skywalking.e2e.metrics.AllOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.AtLeastOneOfMetricsMatcher; +import org.apache.skywalking.e2e.metrics.Metrics; +import org.apache.skywalking.e2e.metrics.MetricsQuery; +import org.apache.skywalking.e2e.metrics.MetricsValueMatcher; +import org.apache.skywalking.e2e.service.Service; +import org.apache.skywalking.e2e.service.ServicesQuery; +import org.junit.Before; +import org.junit.Test; + +import java.time.LocalDateTime; +import java.time.ZoneOffset; +import java.util.List; +import java.util.concurrent.CountDownLatch; + +import static org.apache.skywalking.e2e.metrics.MetricsQuery.SERVICE_RESP_TIME; +import static org.assertj.core.api.Assertions.assertThat; + +/** + * @author kezhenxu94 + */ +@Slf4j +public class StorageTTLITCase { + + //TODO Make TTL ES7 test stable. Ref https://github.com/apache/skywalking/pull/3978, https://github.com/apache/skywalking/issues/4018 + + private static final int SW_STORAGE_ES_MONTH_METRIC_DATA_TTL = 4; + private static final int SW_STORAGE_ES_OTHER_METRIC_DATA_TTL = 5; + + private static final int MAX_INBOUND_MESSAGE_SIZE = 1024 * 1024 * 50; + private static final boolean USE_PLAIN_TEXT = true; + private static final boolean SUCCESS = true; + + private SimpleQueryClient queryClient; + + private ServiceMeshMetricServiceGrpc.ServiceMeshMetricServiceStub grpcStub; + + @Before + public void setUp() { + + final String swWebappHost = System.getProperty("sw.webapp.host", "127.0.0.1"); + final String swWebappPort = System.getProperty("sw.webapp.port", "32789"); + final String oapPort = System.getProperty("oap.port", "32788"); + queryClient = new SimpleQueryClient(swWebappHost, swWebappPort); + + final ManagedChannelBuilder builder = + NettyChannelBuilder.forAddress("127.0.0.1", Integer.parseInt(oapPort)) + .nameResolverFactory(new DnsNameResolverProvider()) + .maxInboundMessageSize(MAX_INBOUND_MESSAGE_SIZE) + .usePlaintext(USE_PLAIN_TEXT); + + final ManagedChannel channel = builder.build(); + + grpcStub = ServiceMeshMetricServiceGrpc.newStub(channel); + } + + @Test(timeout = 360000) + public void dayMetricsDataShouldBeRemovedAfterTTL() throws Exception { + + final ServiceMeshMetric.Builder builder = ServiceMeshMetric + .newBuilder() + .setSourceServiceName("e2e-test-source-service") + .setSourceServiceInstance("e2e-test-source-service-instance") + .setDestServiceName("e2e-test-dest-service") + .setDestServiceInstance("e2e-test-dest-service-instance") + .setEndpoint("e2e/test") + .setLatency(2000) + .setResponseCode(200) + .setStatus(SUCCESS) + .setProtocol(Protocol.HTTP) + .setDetectPoint(DetectPoint.server); + + final LocalDateTime now = LocalDateTime.now(); + final LocalDateTime startTime = now.minusDays(SW_STORAGE_ES_OTHER_METRIC_DATA_TTL + 1); + final LocalDateTime endTime = startTime.plusMinutes(1); + + final LocalDateTime queryStart = startTime; + final LocalDateTime queryEnd = now.minusDays(SW_STORAGE_ES_OTHER_METRIC_DATA_TTL); + + ensureSendingMetricsWorks( + builder, + startTime.toEpochSecond(ZoneOffset.UTC) * 1000, + endTime.toEpochSecond(ZoneOffset.UTC) * 1000, + queryStart, + queryEnd, + "DAY" + ); + + shouldBeEmptyBetweenTimeRange(queryStart, queryEnd, "DAY"); + } + + @Test(timeout = 360000) + public void monthMetricsDataShouldBeRemovedAfterTTL() throws Exception { + + final ServiceMeshMetric.Builder builder = ServiceMeshMetric + .newBuilder() + .setSourceServiceName("e2e-test-source-service") + .setSourceServiceInstance("e2e-test-source-service-instance") + .setDestServiceName("e2e-test-dest-service") + .setDestServiceInstance("e2e-test-dest-service-instance") + .setEndpoint("e2e/test") + .setLatency(2000) + .setResponseCode(200) + .setStatus(SUCCESS) + .setProtocol(Protocol.HTTP) + .setDetectPoint(DetectPoint.server); + + final LocalDateTime now = LocalDateTime.now(); + final LocalDateTime startTime = now.minusMonths(SW_STORAGE_ES_MONTH_METRIC_DATA_TTL + 1); + final LocalDateTime endTime = startTime.plusMinutes(1); + + final LocalDateTime queryStart = startTime; + final LocalDateTime queryEnd = now.minusMonths(SW_STORAGE_ES_MONTH_METRIC_DATA_TTL); + + ensureSendingMetricsWorks( + builder, + startTime.toEpochSecond(ZoneOffset.UTC) * 1000, + endTime.toEpochSecond(ZoneOffset.UTC) * 1000, + queryStart, + queryEnd, + "MONTH" + ); + + shouldBeEmptyBetweenTimeRange(queryStart, queryEnd, "MONTH"); + } + + private void shouldBeEmptyBetweenTimeRange( + final LocalDateTime queryStart, + final LocalDateTime queryEnd, + final String step + ) throws InterruptedException { + + boolean valid = false; + for (int i = 0; i < 10 && !valid; i++) { + try { + final Metrics serviceMetrics = queryMetrics(queryStart, queryEnd, step); + + log.info("ServiceMetrics: {}", serviceMetrics); + + AllOfMetricsMatcher instanceRespTimeMatcher = new AllOfMetricsMatcher(); + MetricsValueMatcher equalsZero = new MetricsValueMatcher(); + equalsZero.setValue("eq 0"); + instanceRespTimeMatcher.setValue(equalsZero); + try { + instanceRespTimeMatcher.verify(serviceMetrics); + valid = true; + } catch (Throwable t) { + log.warn("History metrics data are not deleted yet, {}", t.getMessage()); + Thread.sleep(60000); + } + } catch (Throwable t) { + log.warn("History metrics data are not deleted yet", t); + Thread.sleep(60000); + } + } + } + + private void ensureSendingMetricsWorks( + final ServiceMeshMetric.Builder builder, + final long startTime, + final long endTime, + final LocalDateTime queryStart, + final LocalDateTime queryEnd, + final String step + ) throws Exception { + boolean prepared = false; + while (!prepared) { + sendMetrics( + builder + .setStartTime(startTime) + .setEndTime(endTime) + .build() + ); + final Metrics serviceMetrics = queryMetrics(queryStart, queryEnd, step); + final AtLeastOneOfMetricsMatcher instanceRespTimeMatcher = new AtLeastOneOfMetricsMatcher(); + final MetricsValueMatcher greaterThanZero = new MetricsValueMatcher(); + greaterThanZero.setValue("gt 0"); + instanceRespTimeMatcher.setValue(greaterThanZero); + try { + instanceRespTimeMatcher.verify(serviceMetrics); + prepared = true; + } catch (Throwable ignored) { + sendMetrics( + builder + .setStartTime(startTime) + .setEndTime(endTime) + .build() + ); + Thread.sleep(10000); + } + } + } + + private Metrics queryMetrics( + final LocalDateTime queryStart, + final LocalDateTime queryEnd, + final String step + ) throws Exception { + for (int i = 0; i < 10; i++) { + try { + final List services = queryClient.services( + new ServicesQuery() + .start(LocalDateTime.now().minusDays(SW_STORAGE_ES_OTHER_METRIC_DATA_TTL)) + .end(LocalDateTime.now()) + ); + + log.info("Services: {}", services); + + assertThat(services).isNotEmpty(); + + String serviceId = services.stream().filter(it -> "e2e-test-dest-service".equals(it.getLabel())).findFirst().get().getKey(); + + return queryClient.metrics( + new MetricsQuery() + .id(serviceId) + .metricsName(SERVICE_RESP_TIME) + .step(step) + .start(queryStart) + .end(queryEnd) + ); + } catch (Throwable ignored) { + Thread.sleep(10000); + } + } + return null; + } + + private void sendMetrics(final ServiceMeshMetric metrics) throws InterruptedException { + final CountDownLatch latch = new CountDownLatch(1); + + StreamObserver collect = grpcStub.collect(new StreamObserver() { + @Override + public void onNext(final MeshProbeDownstream meshProbeDownstream) { + + } + + @Override + public void onError(final Throwable throwable) { + throwable.printStackTrace(); + latch.countDown(); + } + + @Override + public void onCompleted() { + latch.countDown(); + } + }); + + collect.onNext(metrics); + + collect.onCompleted(); + + latch.await(); + } +} diff --git a/test/e2e/e2e-ttl/pom.xml b/test/e2e/e2e-ttl/pom.xml index eab8c608e1246524f35e350dbb7bb54439045462..0d422025f5e07c9fb6c928e7ca8ba9cd863d08c7 100644 --- a/test/e2e/e2e-ttl/pom.xml +++ b/test/e2e/e2e-ttl/pom.xml @@ -31,6 +31,7 @@ pom e2e-ttl-es + e2e-ttl-influxdb \ No newline at end of file diff --git a/test/e2e/pom.xml b/test/e2e/pom.xml index 45c775757d0d4ea7ecd309f977289a84afccb35e..0a6c8b520cde11b607d72a70c151a8a55e0023e3 100644 --- a/test/e2e/pom.xml +++ b/test/e2e/pom.xml @@ -34,6 +34,7 @@ e2e-base e2e-single-service + e2e-influxdb e2e-mysql e2e-cluster e2e-cluster-with-gateway @@ -62,6 +63,7 @@ 2.8.5 1.4.199 8.0.13 + 1.7.9 6.3.2 3.5 1.18.10 diff --git a/tools/dependencies/known-oap-backend-dependencies-es7.txt b/tools/dependencies/known-oap-backend-dependencies-es7.txt index 2de9f943ee3ac83252f1c2dfeef52ac8a8aba926..59c3f8df449e4d328607d66cf2ce4f6a5d263a49 100755 --- a/tools/dependencies/known-oap-backend-dependencies-es7.txt +++ b/tools/dependencies/known-oap-backend-dependencies-es7.txt @@ -159,3 +159,8 @@ sundr-core-0.9.2.jar swagger-annotations-1.5.12.jar t-digest-3.2.jar zookeeper-3.4.10.jar +converter-moshi-2.5.0.jar +influxdb-java-2.15.jar +logging-interceptor-3.13.1.jar +moshi-1.5.0.jar +msgpack-core-0.8.16.jar diff --git a/tools/dependencies/known-oap-backend-dependencies.txt b/tools/dependencies/known-oap-backend-dependencies.txt index efa3d786e376313a1d83ba5f6f0d8231518ff675..836cced4b090d2d36526d9d4c9cd45a027d26c5e 100755 --- a/tools/dependencies/known-oap-backend-dependencies.txt +++ b/tools/dependencies/known-oap-backend-dependencies.txt @@ -158,3 +158,8 @@ swagger-annotations-1.5.12.jar t-digest-3.2.jar zipkin-2.9.1.jar zookeeper-3.4.10.jar +converter-moshi-2.5.0.jar +influxdb-java-2.15.jar +logging-interceptor-3.13.1.jar +moshi-1.5.0.jar +msgpack-core-0.8.16.jar