From e89fe259135d3185e52c3e3306dc83cc8d39b612 Mon Sep 17 00:00:00 2001 From: Sail <37333508+SailVR@users.noreply.github.com> Date: Tue, 21 Apr 2020 20:56:49 +0800 Subject: [PATCH] [IOTDB-528]modify grafana group by (#1073) --- .../7-Ecosystem Integration/1-Grafana.md | 70 ++++++++++---- .../7-Ecosystem Integration/1-Grafana.md | 65 ++++++++++--- grafana/readme.md | 74 +++++++++++---- grafana/readme_zh.md | 95 +++++++++++++------ .../web/grafana/dao/impl/BasicDaoImpl.java | 70 ++++++++------ .../main/resources}/application.properties | 10 +- 6 files changed, 277 insertions(+), 107 deletions(-) rename grafana/{conf => src/main/resources}/application.properties (79%) diff --git a/docs/UserGuide/7-Ecosystem Integration/1-Grafana.md b/docs/UserGuide/7-Ecosystem Integration/1-Grafana.md index 7f3cb07bdf..5ad4e9b421 100644 --- a/docs/UserGuide/7-Ecosystem Integration/1-Grafana.md +++ b/docs/UserGuide/7-Ecosystem Integration/1-Grafana.md @@ -18,7 +18,7 @@ under the License. --> -# Grafana +# IoTDB-Grafana ## Outline @@ -59,7 +59,7 @@ grafana-cli plugins install grafana-simple-json-datasource Alternatively, you can manually download the .zip file and unpack it into your grafana plugins directory. -* `{grafana-install-directory}\data\plugin\` (Windows) +* `{grafana-install-directory}\data\plugins\` (Windows) * `/var/lib/grafana/plugins` (Linux) * `/usr/local/var/lib/grafana/plugins`(Mac) @@ -80,27 +80,22 @@ See https://github.com/apache/incubator-iotdb ```shell git clone https://github.com/apache/incubator-iotdb.git -mvn clean package -pl grafana -am -Dmaven.test.skip=true -cd grafana ``` -Copy `application.properties` from `conf/` directory to `target` directory. (Or just make sure that `application.properties` and `iotdb-grafana-{version}.war` are in the same directory.) +### Start IoTDB-Grafana -Edit `application.properties` +#### Option one -``` -# ip and port of IoTDB -spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/ -spring.datasource.username = root -spring.datasource.password = root -spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver -server.port = 8888 -``` +Import the entire project, after the maven dependency is installed, directly run`incubatoriotdb/grafana/rc/main/java/org/apache/iotdb/web/grafana`directory` TsfileWebDemoApplication.java`, this grafana connector is developed by springboot -### Start IoTDB-Grafana +#### Option two + +In `/grafana/target/`directory ```shell -cd grafana/target/ +cd incubator-iotdb +mvn clean package -pl grafana -am -Dmaven.test.skip=true +cd grafana/target java -jar iotdb-grafana-{version}.war ``` @@ -119,6 +114,8 @@ $ java -jar iotdb-grafana-{version}.war ... ``` +If you need to configure properties, move the `grafana/src/main/resources/application.properties` to the same directory as the war package (`grafana/target`) + ## Explore in Grafana The default port of Grafana is 3000, see http://localhost:3000 @@ -136,4 +133,43 @@ Select `Data Sources` and then `Add data source`, select `SimpleJson` in `Type` Add diagrams in dashboard and customize your query. See http://docs.grafana.org/guides/getting_started/ - \ No newline at end of file + + +## config grafana + +``` +# ip and port of IoTDB +spring.datasource.url=jdbc:iotdb://127.0.0.1:6667/ +spring.datasource.username=root +spring.datasource.password=root +spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver +server.port=8888 +# Use this value to set timestamp precision as "ms", "us" or "ns", which must to be same with the timestamp +# precision of Apache IoTDB engine. +timestamp_precision=ms + +# Use this value to set down sampling true/false +isDownSampling=true +# defaut sampling intervals +interval=1m +# aggregation function to use to downsampling the data +# COUNT, FIRST_VALUE, LAST_VALUE, MAX_TIME, MAX_VALUE, AVG, MIN_TIME, MIN_VALUE, NOW, SUM +function=avg +``` + +The specific configuration information of interval is as follows + +<1h: no sampling + +1h~1d : intervals = 1m + +1d~30d:intervals = 1h + +\>30d：intervals = 1d + +After configuration, please re-run war package + +``` +java -jar iotdb-grafana-{version}.war +``` + diff --git a/docs/zh/UserGuide/7-Ecosystem Integration/1-Grafana.md b/docs/zh/UserGuide/7-Ecosystem Integration/1-Grafana.md index bf7134aee6..bf65ca952b 100644 --- a/docs/zh/UserGuide/7-Ecosystem Integration/1-Grafana.md +++ b/docs/zh/UserGuide/7-Ecosystem Integration/1-Grafana.md @@ -51,7 +51,7 @@ Grafana是开源的指标量监测和可视化工具，可用于展示时序数 * 插件名称: simple-json-datasource * 下载地址: https://github.com/grafana/simple-json-datasource -具体下载方法是：到Grafana的插件目录中：`{Grafana文件目录}\data\plugin\`（Windows系统，启动Grafana后会自动创建`data\plugin`目录）或`/var/lib/grafana/plugins` （Linux系统，plugins目录需要手动创建）或`/usr/local/var/lib/grafana/plugins`（MacOS系统，具体位置参看使用`brew install`安装Grafana后命令行给出的位置提示。 +具体下载方法是：到Grafana的插件目录中：`{Grafana文件目录}\data\plugins\`（Windows系统，启动Grafana后会自动创建`data\plugins`目录）或`/var/lib/grafana/plugins` （Linux系统，plugins目录需要手动创建）或`/usr/local/var/lib/grafana/plugins`（MacOS系统，具体位置参看使用`brew install`安装Grafana后命令行给出的位置提示。 执行下面的命令： @@ -83,25 +83,20 @@ Shell > grafana-server --config=/usr/local/etc/grafana/grafana.ini --homepath /u ```shell git clone https://github.com/apache/incubator-iotdb.git -mvn clean package -pl grafana -am -Dmaven.test.skip=true -cd grafana ``` -编译成功后，您需将`application.properties`文件从`conf/`目录复制到`target/`目录下，并在该文件中插入以下（编辑属性值）： +### 启动IoTDB-Grafana -``` -# ip and port of IoTDB -spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/ -spring.datasource.username = root -spring.datasource.password = root -spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver -server.port = 8888 -``` +#### 方案一（适合开发者） -### 启动IoTDB-Grafana +导入整个项目，maven依赖安装完后，直接运行`incubatoriotdb/grafana/rc/main/java/org/apache/iotdb/web/grafana`目录下`TsfileWebDemoApplication.java`，这个grafana连接器采用springboot开发 + +#### 方案二（适合使用者） ```shell -cd grafana/target/ +cd incubator-iotdb +mvn clean package -pl grafana -am -Dmaven.test.skip=true +cd grafana/target java -jar iotdb-grafana-{version}.war . ____ _ __ _ _ /\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \ @@ -113,10 +108,14 @@ java -jar iotdb-grafana-{version}.war ... ``` +如果您需要配置属性，将`grafana/src/main/resources/application.properties`移动到war包同级目录下（`grafana/target`） + ## 使用Grafana Grafana以网页的dashboard形式为您展示数据，在使用时请您打开浏览器，访问http://\:\ +默认地址为http://localhost:3000/ + 注：IP为您的Grafana所在的服务器IP，Port为Grafana的运行端口（默认3000）。默认登录的用户名和密码都是“admin”。 ### 添加IoTDB数据源 @@ -133,3 +132,41 @@ Grafana以网页的dashboard形式为您展示数据，在使用时请您打开 +## 配置grafana + +``` +# IoTDB的IP和端口 +spring.datasource.url=jdbc:iotdb://127.0.0.1:6667/ +spring.datasource.username=root +spring.datasource.password=root +spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver +server.port=8888 +# Use this value to set timestamp precision as "ms", "us" or "ns", which must to be same with the timestamp +# precision of Apache IoTDB engine. +timestamp_precision=ms + +# 是否开启降采样 +isDownSampling=true +# 默认采样interval +interval=1m +# 用于对数据进行降采样的聚合函数 +# COUNT, FIRST_VALUE, LAST_VALUE, MAX_TIME, MAX_VALUE, AVG, MIN_TIME, MIN_VALUE, NOW, SUM +function=avg +``` + +其中interval具体配置信息如下 + +<1h: no sampling + +1h~1d : intervals = 1m + +1d~30d:intervals = 1h + +\>30d：intervals = 1d + +配置完后，请重新运行war包 + +``` +java -jar iotdb-grafana-{version}.war +``` + diff --git a/grafana/readme.md b/grafana/readme.md index 00ed84201c..c52a1de98d 100644 --- a/grafana/readme.md +++ b/grafana/readme.md @@ -7,9 +7,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,7 +18,7 @@ under the License. --> -# Grafana +# IoTDB-Grafana ## Outline @@ -59,7 +59,7 @@ grafana-cli plugins install grafana-simple-json-datasource Alternatively, you can manually download the .zip file and unpack it into your grafana plugins directory. -* `{grafana-install-directory}\data\plugin\` (Windows) +* `{grafana-install-directory}\data\plugins\` (Windows) * `/var/lib/grafana/plugins` (Linux) * `/usr/local/var/lib/grafana/plugins`(Mac) @@ -80,27 +80,22 @@ See https://github.com/apache/incubator-iotdb ```shell git clone https://github.com/apache/incubator-iotdb.git -mvn clean package -pl grafana -am -Dmaven.test.skip=true -cd grafana ``` -Copy `application.properties` from `conf/` directory to `target` directory. (Or just make sure that `application.properties` and `iotdb-grafana-{version}.war` are in the same directory.) +### Start IoTDB-Grafana -Edit `application.properties` +#### Option one -``` -# ip and port of IoTDB -spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/ -spring.datasource.username = root -spring.datasource.password = root -spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver -server.port = 8888 -``` +Import the entire project, after the maven dependency is installed, directly run`incubatoriotdb/grafana/rc/main/java/org/apache/iotdb/web/grafana`directory` TsfileWebDemoApplication.java`, this grafana connector is developed by springboot -### Start IoTDB-Grafana +#### Option two + +In `/grafana/target/`directory ```shell -cd grafana/target/ +cd incubator-iotdb +mvn clean package -pl grafana -am -Dmaven.test.skip=true +cd grafana/target java -jar iotdb-grafana-{version}.war ``` @@ -119,6 +114,8 @@ $ java -jar iotdb-grafana-{version}.war ... ``` +If you need to configure properties, move the ` grafana/src/main/resources/application.properties` to the same directory as the war package (`grafana/target`) + ## Explore in Grafana The default port of Grafana is 3000, see http://localhost:3000 @@ -136,4 +133,43 @@ Select `Data Sources` and then `Add data source`, select `SimpleJson` in `Type` Add diagrams in dashboard and customize your query. See http://docs.grafana.org/guides/getting_started/ - \ No newline at end of file + + +## config grafana + +``` +# ip and port of IoTDB +spring.datasource.url=jdbc:iotdb://127.0.0.1:6667/ +spring.datasource.username=root +spring.datasource.password=root +spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver +server.port=8888 +# Use this value to set timestamp precision as "ms", "us" or "ns", which must to be same with the timestamp +# precision of Apache IoTDB engine. +timestamp_precision=ms + +# Use this value to set down sampling true/false +isDownSampling=true +# defaut sampling intervals +interval=1m +# aggregation function to use to downsampling the data +# COUNT, FIRST_VALUE, LAST_VALUE, MAX_TIME, MAX_VALUE, AVG, MIN_TIME, MIN_VALUE, NOW, SUM +function=avg +``` + +The specific configuration information of interval is as follows + +<1h: no sampling + +1h~1d : intervals = 1m + +1d~30d:intervals = 1h + +\>30d：intervals = 1d + +After configuration, please re-run war package + +``` +java -jar iotdb-grafana-{version}.war +``` + diff --git a/grafana/readme_zh.md b/grafana/readme_zh.md index e9ba959acc..bf65ca952b 100644 --- a/grafana/readme_zh.md +++ b/grafana/readme_zh.md @@ -7,9 +7,9 @@ to you under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at - + http://www.apache.org/licenses/LICENSE-2.0 - + Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY @@ -18,25 +18,25 @@ under the License. --> - + +# IoTDB-Grafana + ## 概览 - IoTDB-Grafana - - Grafana的安装与部署 - - 安装 - - simple-json-datasource数据源插件安装 - - 启动Grafana - - IoTDB安装 - - IoTDB-Grafana连接器安装 - - 启动IoTDB-Grafana - - 使用Grafana - - 添加IoTDB数据源 - - 操作Grafana + - Grafana的安装与部署 + - 安装 + - simple-json-datasource数据源插件安装 + - 启动Grafana + - IoTDB安装 + - IoTDB-Grafana连接器安装 + - 启动IoTDB-Grafana + - 使用Grafana + - 添加IoTDB数据源 + - 操作Grafana -# IoTDB-Grafana - Grafana是开源的指标量监测和可视化工具，可用于展示时序数据和应用程序运行分析。Grafana支持Graphite，InfluxDB等国际主流时序时序数据库作为数据源。在IoTDB项目中，我们开发了Grafana展现IoTDB中时序数据的连接器IoTDB-Grafana，为您提供使用Grafana展示IoTDB数据库中的时序数据的可视化方法。 ## Grafana的安装与部署 @@ -51,7 +51,7 @@ Grafana是开源的指标量监测和可视化工具，可用于展示时序数 * 插件名称: simple-json-datasource * 下载地址: https://github.com/grafana/simple-json-datasource -具体下载方法是：到Grafana的插件目录中：`{Grafana文件目录}\data\plugin\`（Windows系统，启动Grafana后会自动创建`data\plugin`目录）或`/var/lib/grafana/plugins` （Linux系统，plugins目录需要手动创建）或`/usr/local/var/lib/grafana/plugins`（MacOS系统，具体位置参看使用`brew install`安装Grafana后命令行给出的位置提示。 +具体下载方法是：到Grafana的插件目录中：`{Grafana文件目录}\data\plugins\`（Windows系统，启动Grafana后会自动创建`data\plugins`目录）或`/var/lib/grafana/plugins` （Linux系统，plugins目录需要手动创建）或`/usr/local/var/lib/grafana/plugins`（MacOS系统，具体位置参看使用`brew install`安装Grafana后命令行给出的位置提示。 执行下面的命令： @@ -83,25 +83,20 @@ Shell > grafana-server --config=/usr/local/etc/grafana/grafana.ini --homepath /u ```shell git clone https://github.com/apache/incubator-iotdb.git -mvn clean package -pl grafana -am -Dmaven.test.skip=true -cd grafana ``` -编译成功后，您需将`application.properties`文件从`conf/`目录复制到`target/`目录下，并在该文件中插入以下（编辑属性值）： +### 启动IoTDB-Grafana -``` -# ip and port of IoTDB -spring.datasource.url = jdbc:iotdb://127.0.0.1:6667/ -spring.datasource.username = root -spring.datasource.password = root -spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver -server.port = 8888 -``` +#### 方案一（适合开发者） -### 启动IoTDB-Grafana +导入整个项目，maven依赖安装完后，直接运行`incubatoriotdb/grafana/rc/main/java/org/apache/iotdb/web/grafana`目录下`TsfileWebDemoApplication.java`，这个grafana连接器采用springboot开发 + +#### 方案二（适合使用者） ```shell -cd grafana/target/ +cd incubator-iotdb +mvn clean package -pl grafana -am -Dmaven.test.skip=true +cd grafana/target java -jar iotdb-grafana-{version}.war . ____ _ __ _ _ /\\ / ___'_ __ _ _(_)_ __ __ _ \ \ \ \ @@ -113,10 +108,14 @@ java -jar iotdb-grafana-{version}.war ... ``` +如果您需要配置属性，将`grafana/src/main/resources/application.properties`移动到war包同级目录下（`grafana/target`） + ## 使用Grafana Grafana以网页的dashboard形式为您展示数据，在使用时请您打开浏览器，访问http://\:\ +默认地址为http://localhost:3000/ + 注：IP为您的Grafana所在的服务器IP，Port为Grafana的运行端口（默认3000）。默认登录的用户名和密码都是“admin”。 ### 添加IoTDB数据源 @@ -133,3 +132,41 @@ Grafana以网页的dashboard形式为您展示数据，在使用时请您打开 +## 配置grafana + +``` +# IoTDB的IP和端口 +spring.datasource.url=jdbc:iotdb://127.0.0.1:6667/ +spring.datasource.username=root +spring.datasource.password=root +spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver +server.port=8888 +# Use this value to set timestamp precision as "ms", "us" or "ns", which must to be same with the timestamp +# precision of Apache IoTDB engine. +timestamp_precision=ms + +# 是否开启降采样 +isDownSampling=true +# 默认采样interval +interval=1m +# 用于对数据进行降采样的聚合函数 +# COUNT, FIRST_VALUE, LAST_VALUE, MAX_TIME, MAX_VALUE, AVG, MIN_TIME, MIN_VALUE, NOW, SUM +function=avg +``` + +其中interval具体配置信息如下 + +<1h: no sampling + +1h~1d : intervals = 1m + +1d~30d:intervals = 1h + +\>30d：intervals = 1d + +配置完后，请重新运行war包 + +``` +java -jar iotdb-grafana-{version}.war +``` + diff --git a/grafana/src/main/java/org/apache/iotdb/web/grafana/dao/impl/BasicDaoImpl.java b/grafana/src/main/java/org/apache/iotdb/web/grafana/dao/impl/BasicDaoImpl.java index eef7689e40..cc2fc374bf 100644 --- a/grafana/src/main/java/org/apache/iotdb/web/grafana/dao/impl/BasicDaoImpl.java +++ b/grafana/src/main/java/org/apache/iotdb/web/grafana/dao/impl/BasicDaoImpl.java @@ -18,21 +18,20 @@ */ package org.apache.iotdb.web.grafana.dao.impl; +import java.time.Duration; import org.apache.iotdb.tsfile.utils.Pair; import org.apache.iotdb.web.grafana.bean.TimeValues; import org.apache.iotdb.web.grafana.dao.BasicDao; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; +import org.springframework.context.annotation.PropertySource; import org.springframework.jdbc.core.ConnectionCallback; import org.springframework.jdbc.core.JdbcTemplate; import org.springframework.jdbc.core.RowMapper; import org.springframework.stereotype.Repository; -import java.io.File; -import java.io.FileInputStream; -import java.io.IOException; -import java.io.InputStream; import java.sql.Connection; import java.sql.ResultSet; import java.sql.SQLException; @@ -46,38 +45,41 @@ import java.util.Properties; * Created by dell on 2017/7/17. */ @Repository +@PropertySource("classpath:application.properties") public class BasicDaoImpl implements BasicDao { private static final Logger logger = LoggerFactory.getLogger(BasicDaoImpl.class); - private static final String CONFIG_PROPERTY_FILE = "application.properties"; - private final JdbcTemplate jdbcTemplate; private static long TIMESTAMP_RADIX = 1L; + @Value("${isDownSampling}") + private boolean isDownSampling; + + @Value("${function}") + private String function; + + @Value("${interval}") + private String interval; + + @Autowired public BasicDaoImpl(JdbcTemplate jdbcTemplate) { this.jdbcTemplate = jdbcTemplate; - try (InputStream inputStream = new FileInputStream(new File(CONFIG_PROPERTY_FILE))) { - Properties properties = new Properties(); - properties.load(inputStream); - String tsPrecision = properties.getProperty("timestamp_precision", "ms"); - switch (tsPrecision) { - case "us": - TIMESTAMP_RADIX = 1000; - break; - case "ns": - TIMESTAMP_RADIX = 1000_000; - break; - default: - TIMESTAMP_RADIX = 1; - } - logger.info("Use timestamp precision {}", tsPrecision); - } catch (IOException e) { - logger.error("Can not find properties [timestamp_precision], use default value [ms]"); - TIMESTAMP_RADIX = 1; + Properties properties = new Properties(); + String tsPrecision = properties.getProperty("timestamp_precision", "ms"); + switch (tsPrecision) { + case "us": + TIMESTAMP_RADIX = 1000; + break; + case "ns": + TIMESTAMP_RADIX = 1000_000; + break; + default: + TIMESTAMP_RADIX = 1; } + logger.info("Use timestamp precision {}", tsPrecision); } @Override @@ -103,14 +105,28 @@ public class BasicDaoImpl implements BasicDao { public List querySeries(String s, Pair timeRange) { Long from = zonedCovertToLong(timeRange.left); Long to = zonedCovertToLong(timeRange.right); - // How many rows will the result have? + final long hours = Duration.between(timeRange.left, timeRange.right).toHours(); + List rows = null; String sql = String.format("SELECT %s FROM root.%s WHERE time > %d and time < %d", s.substring(s.lastIndexOf('.') + 1), s.substring(0, s.lastIndexOf('.')), from * TIMESTAMP_RADIX, to * TIMESTAMP_RADIX); + String columnName = "root." + s; + if (isDownSampling && (hours > 1)) { + if (hours < 30 * 24 && hours > 24) { + interval = "1h"; + } else if (hours > 30 * 24) { + interval = "1d"; + } + sql = String.format( + "SELECT " + function + + "(%s) FROM root.%s WHERE time > %d and time < %d group by ([%d, %d),%s)", + s.substring(s.lastIndexOf('.') + 1), s.substring(0, s.lastIndexOf('.')), from, to, from, + to, interval); + columnName = function + "(root." + s + ")"; + } logger.info(sql); - List rows = null; try { - rows = jdbcTemplate.query(sql, new TimeValuesRowMapper("root." + s)); + rows = jdbcTemplate.query(sql, new TimeValuesRowMapper(columnName)); } catch (Exception e) { logger.error(e.getMessage()); } diff --git a/grafana/conf/application.properties b/grafana/src/main/resources/application.properties similarity index 79% rename from grafana/conf/application.properties rename to grafana/src/main/resources/application.properties index b2b982fb06..7847a8ab77 100644 --- a/grafana/conf/application.properties +++ b/grafana/src/main/resources/application.properties @@ -24,4 +24,12 @@ spring.datasource.driver-class-name=org.apache.iotdb.jdbc.IoTDBDriver server.port=8888 # Use this value to set timestamp precision as "ms", "us" or "ns", which must to be same with the timestamp # precision of Apache IoTDB engine. -timestamp_precision=ms \ No newline at end of file +timestamp_precision=ms + +# Use this value to set down sampling true/false +isDownSampling=true +# defaut sampling intervals +interval=1m +# aggregation function to use to downsampling the data +# COUNT, FIRST_VALUE, LAST_VALUE, MAX_TIME, MAX_VALUE, AVG, MIN_TIME, MIN_VALUE, NOW, SUM +function=avg \ No newline at end of file -- GitLab