diff --git a/Jenkinsfile b/Jenkinsfile index f5b32d55d89d95d5df13f45a9e5a6fa480d90fda..e9e9b52c70aa9b9f2cefad4096dc678651a04210 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -41,68 +41,54 @@ def pre_test(){ killall -9 taosd ||echo "no taosd running" killall -9 gdb || echo "no gdb running" killall -9 python3.8 || echo "no python program running" - cd ${WKC} - git reset --hard HEAD~10 >/dev/null + cd ${WS} + rm -rf ${WK} ''' script { if (env.CHANGE_TARGET == 'master') { sh ''' - cd ${WKC} - git checkout master + cd ${WS} + cd TDinternal_master + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_master TDinternal ''' } else if(env.CHANGE_TARGET == '2.0'){ sh ''' - cd ${WKC} - git checkout 2.0 + cd ${WS} + cd TDinternal_2.0 + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_2.0 TDinternal ''' } else{ sh ''' - cd ${WKC} - git checkout develop + cd ${WS} + cd TDinternal_develop + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_develop TDinternal ''' } } sh''' cd ${WKC} - [ -f src/connector/grafanaplugin/README.md ] && rm -f src/connector/grafanaplugin/README.md > /dev/null || echo "failed to remove grafanaplugin README.md" - git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - git ls-files --stage | grep 160000 | awk '{print $4}' | xargs git rm --cached git submodule update --init --recursive - cd ${WK} - git reset --hard HEAD~10 ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WK} - git checkout 2.0 - ''' - } - else{ - sh ''' - cd ${WK} - git checkout develop - ''' - } - } sh ''' - cd ${WK} - git pull >/dev/null - + cd ${WK} export TZ=Asia/Harbin date - git clean -dfx mkdir debug cd debug cmake .. > /dev/null @@ -116,143 +102,117 @@ def pre_test(){ def pre_test_noinstall(){ sh'hostname' sh''' - cd ${WKC} - git reset --hard HEAD~10 >/dev/null + cd ${WS} + rm -rf ${WK} ''' script { if (env.CHANGE_TARGET == 'master') { sh ''' - cd ${WKC} - git checkout master + cd ${WS} + cd TDinternal_master + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_master TDinternal ''' } else if(env.CHANGE_TARGET == '2.0'){ sh ''' - cd ${WKC} - git checkout 2.0 + cd ${WS} + cd TDinternal_20 + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_20 TDinternal ''' } else{ sh ''' - cd ${WKC} - git checkout develop + cd ${WS} + cd TDinternal_develop + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_develop TDinternal ''' } } sh''' cd ${WKC} - [ -f src/connector/grafanaplugin/README.md ] && rm -f src/connector/grafanaplugin/README.md > /dev/null || echo "failed to remove grafanaplugin README.md" - git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - git ls-files --stage | grep 160000 | awk '{print $4}' | xargs git rm --cached git submodule update --init --recursive - cd ${WK} - git reset --hard HEAD~10 ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WK} - git checkout 2.0 - ''' - } - else{ - sh ''' - cd ${WK} - git checkout develop - ''' - } - } sh ''' - cd ${WK} - git pull >/dev/null - + cd ${WK} export TZ=Asia/Harbin date - git clean -dfx mkdir debug cd debug cmake .. > /dev/null - make + make ''' return 1 } def pre_test_mac(){ sh'hostname' sh''' - cd ${WKC} - git reset --hard HEAD~10 >/dev/null + cd ${WS} + rm -rf ${WK} ''' script { if (env.CHANGE_TARGET == 'master') { sh ''' - cd ${WKC} - git checkout master + cd ${WS} + cd TDinternal_master + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_master TDinternal ''' } else if(env.CHANGE_TARGET == '2.0'){ sh ''' - cd ${WKC} - git checkout 2.0 + cd ${WS} + cd TDinternal_20 + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_20 TDinternal ''' } else{ sh ''' - cd ${WKC} - git checkout develop + cd ${WS} + cd TDinternal_develop + git pull + cd community + git pull + cd ${WS} + cp -fr TDinternal_develop TDinternal ''' } } sh''' cd ${WKC} - [ -f src/connector/grafanaplugin/README.md ] && rm -f src/connector/grafanaplugin/README.md > /dev/null || echo "failed to remove grafanaplugin README.md" - git pull >/dev/null git fetch origin +refs/pull/${CHANGE_ID}/merge git checkout -qf FETCH_HEAD - git clean -dfx - git ls-files --stage | grep 160000 | awk '{print $4}' | xargs git rm --cached git submodule update --init --recursive - cd ${WK} - git reset --hard HEAD~10 ''' - script { - if (env.CHANGE_TARGET == 'master') { - sh ''' - cd ${WK} - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - sh ''' - cd ${WK} - git checkout 2.0 - ''' - } - else{ - sh ''' - cd ${WK} - git checkout develop - ''' - } - } sh ''' - cd ${WK} - git pull >/dev/null - + cd ${WK} export TZ=Asia/Harbin date - git clean -dfx mkdir debug cd debug cmake .. > /dev/null + go env -w GOPROXY=https://goproxy.cn,direct + go env -w GO111MODULE=on cmake --build . ''' return 1 @@ -262,67 +222,47 @@ def pre_test_win(){ taskkill /f /t /im python.exe cd C:\\ rd /s /Q C:\\TDengine - cd C:\\workspace\\TDinternal - rd /s /Q C:\\workspace\\TDinternal\\debug - cd C:\\workspace\\TDinternal\\community - git reset --hard HEAD~10 + cd C:\\workspace + rd /s /Q C:\\workspace\\TDinternal | echo 1 ''' script { if (env.CHANGE_TARGET == 'master') { bat ''' - cd C:\\workspace\\TDinternal\\community - git checkout master + cd C:\\workspace\\TDinternal_master + git pull + cd C:\\workspace\\TDinternal_master\\community + git pull + xcopy /e/y/i/f C:\\workspace\\TDinternal_master C:\\workspace\\TDinternal ''' } else if(env.CHANGE_TARGET == '2.0'){ bat ''' - cd C:\\workspace\\TDinternal\\community - git checkout 2.0 + cd C:\\workspace\\TDinternal_20 + git pull + cd C:\\workspace\\TDinternal_20\\community + git pull + xcopy /e/y/i/f C:\\workspace\\TDinternal_20 C:\\workspace\\TDinternal ''' } else{ bat ''' - cd C:\\workspace\\TDinternal\\community - git checkout develop + cd C:\\workspace\\TDinternal_develop + git pull + cd C:\\workspace\\TDinternal_develop\\community + git pull + xcopy /e/y/i/f C:\\workspace\\TDinternal_develop C:\\workspace\\TDinternal ''' } } bat''' cd C:\\workspace\\TDinternal\\community - git pull git fetch origin +refs/pull/%CHANGE_ID%/merge git checkout -qf FETCH_HEAD - git clean -dfx git submodule update --init --recursive - cd C:\\workspace\\TDinternal - git reset --hard HEAD~10 ''' - script { - if (env.CHANGE_TARGET == 'master') { - bat ''' - cd C:\\workspace\\TDinternal - git checkout master - ''' - } - else if(env.CHANGE_TARGET == '2.0'){ - bat ''' - cd C:\\workspace\\TDinternal - git checkout 2.0 - ''' - } - else{ - bat ''' - cd C:\\workspace\\TDinternal - git checkout develop - ''' - } - } bat ''' cd C:\\workspace\\TDinternal - git pull - date - git clean -dfx mkdir debug cd debug call "C:\\Program Files (x86)\\Microsoft Visual Studio\\2017\\Community\\VC\\Auxiliary\\Build\\vcvarsall.bat" amd64 @@ -342,6 +282,7 @@ pipeline { environment{ WK = '/var/lib/jenkins/workspace/TDinternal' WKC= '/var/lib/jenkins/workspace/TDinternal/community' + WS = '/var/lib/jenkins/workspace' } stages { stage('pre_build'){ diff --git a/README.md b/README.md index a14003397bae69aa74a8a1ff7a55db18ae53a149..fb6eed0267c19259b5e931c8a98c19aff90d8deb 100644 --- a/README.md +++ b/README.md @@ -64,7 +64,7 @@ To build the [taos-tools](https://github.com/taosdata/taos-tools) on Ubuntu/Debi sudo apt install libjansson-dev libsnappy-dev liblzma-dev libz-dev pkg-config ``` -### Centos 7: +### CentOS 7: ```bash sudo yum install epel-release sudo yum update @@ -82,7 +82,7 @@ To install Apache Maven: sudo yum install -y maven ``` -### Centos 8 & Fedora: +### CentOS 8 & Fedora: ```bash sudo dnf install -y gcc gcc-c++ make cmake epel-release git ``` @@ -100,8 +100,9 @@ sudo dnf install -y maven #### Install build dependencies for taos-tools To build the [taos-tools](https://github.com/taosdata/taos-tools) on CentOS, the following packages need to be installed. ```bash -sudo yum install xz-devel snappy-devel jansson-devel pkgconfig libatomic +sudo yum install libz-devel xz-devel snappy-devel jansson-devel pkgconfig libatomic ``` +Note: Since snappy lacks pkg-config support (refer to [link](https://github.com/google/snappy/pull/86)), it lead a cmake prompt libsnappy not found. But snappy will works well. ### Setup golang environment TDengine includes few components developed by Go language. Please refer to golang.org official documentation for golang environment setup. diff --git a/documentation20/cn/00.index/docs.md b/documentation20/cn/00.index/docs.md index bfd870bf6412bf19898f9f5d569e6536bc156b1a..d94a58eebb129e84137a8c55b1ca07be37ec15af 100644 --- a/documentation20/cn/00.index/docs.md +++ b/documentation20/cn/00.index/docs.md @@ -83,6 +83,10 @@ TDengine是一个高效的存储、查询、分析时序大数据的平台,专 * [Windows客户端](https://www.taosdata.com/blog/2019/07/26/514.html):自行编译windows客户端,Windows环境的各种连接器都需要它 * [Rust Connector](/connector/rust): Rust语言下通过libtaos客户端或RESTful接口,连接TDengine服务器。 +## [TDengine 组件与工具](/tools/adapter) + +* [taosAdapter用户手册](/tools/adapter) + ## [与其他工具的连接](/connections) * [Grafana](/connections#grafana):获取并可视化保存在TDengine的数据 diff --git a/documentation20/cn/08.connector/docs.md b/documentation20/cn/08.connector/docs.md index a12cb48df2d402fb45793a5c1dd4da75d65d5522..e56f17a42e17683185057a3aacef249ecfeb0619 100644 --- a/documentation20/cn/08.connector/docs.md +++ b/documentation20/cn/08.connector/docs.md @@ -1052,7 +1052,7 @@ HTTP 请求 URL 采用 `sqlutc` 时,返回结果集的时间戳将采用 UTC 示例程序源码位于 * {client_install_directory}/examples/C# -* [github C# example source code](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C%23) +* [github C# example source code](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C\%23) **注意:** TDengineTest.cs C#示例源程序,包含了数据库连接参数,以及如何执行数据插入、查询等操作。 @@ -1084,7 +1084,7 @@ dotnet add package TDengine.Connector ```c# using TDengineDriver; ``` -* 用户可以参考[TDengineTest.cs](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C%23/TDengineTest)来定义数据库连接参数,以及如何执行数据插入、查询等操作。 +* 用户可以参考[TDengineTest.cs](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C\%23/TDengineTest)来定义数据库连接参数,以及如何执行数据插入、查询等操作。 **注意:** diff --git a/documentation20/cn/14.devops/01.telegraf/docs.md b/documentation20/cn/14.devops/01.telegraf/docs.md index 04765602dab18fbacf7d92d44ca324db660c0ac4..485e7038f0e8aa122b20ba6608a629de66d7dc8c 100644 --- a/documentation20/cn/14.devops/01.telegraf/docs.md +++ b/documentation20/cn/14.devops/01.telegraf/docs.md @@ -25,8 +25,8 @@ IT 运维监测数据通常都是对时间特性比较敏感的数据,例如 ### Grafana 请参考[官方文档](https://grafana.com/grafana/download)。 -### 安装 TDengine -从涛思数据官网[下载](http://taosdata.com/cn/all-downloads/)页面下载最新 TDengine-server 2.3.0.0 或以上版本安装。 +### TDengine +从涛思数据官网[下载](http://taosdata.com/cn/all-downloads/)页面下载最新 TDengine-server 2.3.0.0 或以上版本安装。 ## 数据链路设置 diff --git a/documentation20/cn/14.devops/03.immigrate/docs.md b/documentation20/cn/14.devops/03.immigrate/docs.md index 2f1de29664e7269bf8c675d83fa9f1bee06b6368..4acfecd0cfe903cd993e8c548e9c6b9032dde48a 100644 --- a/documentation20/cn/14.devops/03.immigrate/docs.md +++ b/documentation20/cn/14.devops/03.immigrate/docs.md @@ -8,7 +8,7 @@ - 数据写入和查询的性能远超 OpenTSDB; - 针对时序数据的高效压缩机制,压缩后在磁盘上的存储空间不到 1/5; -- 安装部署非常简单,单一安装包完成安装部署,除了 taosAdapter 需要依赖 Go 运行环境外,不依赖其他的第三方软件,整个安装部署过程秒级搞定; +- 安装部署非常简单,单一安装包完成安装部署,不依赖其他的第三方软件,整个安装部署过程秒级搞定; - 提供的内建函数覆盖 OpenTSDB 支持的全部查询函数,还支持更多的时序数据查询函数、标量函数及聚合函数,支持多种时间窗口聚合、连接查询、表达式运算、多种分组聚合、用户定义排序、以及用户定义函数等高级查询功能。采用类 SQL 的语法规则,更加简单易学,基本上没有学习成本。 - 支持多达 128 个标签,标签总长度可达到 16 KB; - 除 HTTP 之外,还提供 Java、Python、C、Rust、Go 等多种语言的接口,支持 JDBC 等多种企业级标准连接器协议。 @@ -40,7 +40,11 @@ - **调整数据收集器配置** -在 TDengine 2.3 版本中,后台服务 taosd 启动后一个 HTTP 的服务 taosAdapter 也会自动启用*。*利用 taosAdapter 能够兼容 Influxdb 的 Line Protocol 和 OpenTSDB 的 telnet/JSON 写入协议,可以将 collectd 和 StatsD 收集的数据直接推送到TDengine。 +在TDengine 2.3版本中,我们发布了taosAdapter ,taosAdapter 是一个无状态、可快速弹性伸缩的组件,它可以兼容 Influxdb 的 Line Protocol 和 OpenTSDB 的 telnet/JSON 写入协议规范,提供了丰富的数据接入能力,有效的节省用户迁移成本,降低用户应用迁移的难度。 + +用户可以根据需求弹性部署 taosAdapter 实例,结合场景的需要,快速提升数据写入的吞吐量,为不同应用场景下的数据写入提供保障。 + +通过taosAdapter,用户可以将 collectd 和 StatsD 收集的数据直接推送到TDengine ,实现应用场景的无缝迁移,非常的轻松便捷。taosAdapter还支持Telegraf、Icinga、TCollector 、node_exporter的数据接入,使用详情参考[taosAdapter](https://www.taosdata.com/cn/documentation/tools/adapter)。 如果使用 collectd,修改其默认位置 `/etc/collectd/collectd.conf` 的配置文件为指向 taosAdapter 部署的节点 IP 地址和端口。假设 taosAdapter 的 IP 地址为192.168.1.130,端口为 6046,配置如下: @@ -61,24 +65,9 @@ LoadPlugin write_tsdb - **调整看板(Dashborad)系统** -在数据能够正常写入TDengine 后,可以调整适配 Grafana 将写入 TDengine 的数据可视化呈现出来。Grafana 暂时还不能够直接连接 TDengine,在 TDengine 的安装目录下 connector/grafanaplugin 有为 Grafana 提供的连接插件。使用该插件的方式很简单: - -首先将grafanaplugin目录下的dist目录整体拷贝到Grafana的插件目录(默认地址为 `/var/lib/grafana/plugins/`),然后重启 Grafana 即可在 **Add Data Source** 菜单下看见 TDengine 数据源。 - -```shell -sudo cp -r . /var/lib/grafana/plugins/tdengine -sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine -echo -e "[plugins]\nallow_loading_unsigned_plugins = taosdata-tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini - -# start grafana service -sudo service grafana-server restart -# or with systemd -sudo systemctl start grafana-server -``` - +在数据能够正常写入TDengine 后,可以调整适配 Grafana 将写入 TDengine 的数据可视化呈现出来。获取和使用TDengine提供的Grafana插件请参考[与其他工具的连接](https://www.taosdata.com/cn/documentation/connections#grafana)。 - -此外,TDengine 还提供了默认的两套Dashboard 模板,供用户快速查看保存到TDengine库里的信息。你只需要将 Grafana 目录下的模板导入到Grafana中即可激活使用。 +TDengine 提供了默认的两套Dashboard 模板,用户只需要将 Grafana 目录下的模板导入到Grafana中即可激活使用。 ![](../../images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg) @@ -129,8 +118,8 @@ TDengine 当前只支持 Grafana 的可视化看板呈现,所以如果你的 | 序号 | 测量(metric) | 值名称 | 类型 | tag1 | tag2 | tag3 | tag4 | tag5 | | ---- | -------------- | ------ | ------ | ---- | ----------- | -------------------- | --------- | ------ | -| 1 | memory | value | double | host | memory_type | memory_type_instance | source | | -| 2 | swap | value | double | host | swap_type | swap_type_instance | source | | +| 1 | memory | value | double | host | memory_type | memory_type_instance | source | n/a | +| 2 | swap | value | double | host | swap_type | swap_type_instance | source | n/a | | 3 | disk | value | double | host | disk_point | disk_instance | disk_type | source | @@ -191,7 +180,21 @@ TDengine支持标准的JDBC 3.0接口操纵数据库,你也可以使用其他 为了方便历史数据的迁移工作,我们为数据同步工具DataX提供了插件,能够将数据自动写入到TDengine中,需要注意的是DataX的自动化数据迁移只能够支持单值模型的数据迁移过程。 -DataX 具体的使用方式及如何使用DataX将数据写入TDengine请参见其使用帮助手册 [github.com/taosdata/datax](http://github.com/taosdata/datax)。 +DataX 具体的使用方式及如何使用DataX将数据写入TDengine请参见[基于DataX的TDeninge数据迁移工具](https://www.taosdata.com/blog/2021/10/26/3156.html)。 + +在对DataX进行迁移实践后,我们发现通过启动多个进程,同时迁移多个metric 的方式,可以大幅度的提高迁移历史数据的效率,下面是迁移过程中的部分记录,希望这些能为应用迁移工作带来参考。 + +| datax实例个数 (并发进程个数) | 迁移记录速度 (条/秒) | +| ---- | -------------- | +| 1 | 约13.9万 | +| 2 | 约21.8万 | +| 3 | 约24.9万 | +| 5 | 约29.5万 | +| 10 | 约33万 | + + +
(注:测试数据源自 单节点 Intel(R) Core(TM) i7-10700 CPU@2.90GHz 16核64G硬件设备,channel和batchSize 分别为8和1000,每条记录包含10个tag) + ### 2、手动迁移数据 diff --git a/documentation20/en/00.index/docs.md b/documentation20/en/00.index/docs.md index 2042a9c963664e65f960f8b7109511d63dd398f6..15f742e8b2935be70926341499dd357ee957b992 100644 --- a/documentation20/en/00.index/docs.md +++ b/documentation20/en/00.index/docs.md @@ -79,6 +79,10 @@ TDengine is a highly efficient platform to store, query, and analyze time-series - [Windows Client](https://www.taosdata.com/blog/2019/07/26/514.html): compile your own Windows client, which is required by various connectors on the Windows environment - [Rust Connector](/connector/rust): A taosc/RESTful API based TDengine client for Rust +## [Components and Tools](/tools/adapter) + +* [taosAdapter](/tools/adapter) + ## [Connections with Other Tools](/connections) - [Grafana](/connections#grafana): query the data saved in TDengine and provide visualization @@ -128,4 +132,4 @@ TDengine is a highly efficient platform to store, query, and analyze time-series ## FAQ -- [FAQ: Common questions and answers](/faq) \ No newline at end of file +- [FAQ: Common questions and answers](/faq) diff --git a/documentation20/en/08.connector/docs.md b/documentation20/en/08.connector/docs.md index 8fcb91dc71ff1f44d616330572511aea800a1b80..10c0c261a548a36e0c6e735a7981f08fa2c57d03 100644 --- a/documentation20/en/08.connector/docs.md +++ b/documentation20/en/08.connector/docs.md @@ -855,7 +855,7 @@ Only some configuration parameters related to RESTful interface are listed below ### Example Source Code you can find sample code under follow directions: * {client_install_directory}/examples/C# -* [github C# example source code](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C%23) +* [github C# example source code](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C\%23) **Tips:** TDengineTest.cs One of C# connector's sample code that include basic examples like connection,sql executions and so on. @@ -888,7 +888,7 @@ dotnet add package TDengine.Connector ```C# using TDengineDriver; ``` -* user can reference from[TDengineTest.cs](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C%23/TDengineTest) and learn how to define database connection,query,insert and other basic data manipulations. +* user can reference from[TDengineTest.cs](https://github.com/taosdata/TDengine/tree/develop/tests/examples/C\%23/TDengineTest) and learn how to define database connection,query,insert and other basic data manipulations. **Note:** diff --git a/documentation20/en/14.devops/01.telegraf/docs.md b/documentation20/en/14.devops/01.telegraf/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..a8b5db08ccc1131611c12fb53970115a89368376 --- /dev/null +++ b/documentation20/en/14.devops/01.telegraf/docs.md @@ -0,0 +1,75 @@ +# Rapidly build an IT DevOps system with TDengine + Telegraf + Grafana + +## Background +TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. + +There are a lot of time-series data in the IT DevOps scenario, for example: +- Metrics of system resource: CPU, memory, IO and network status, etc. +- Metrics for software system: service status, number of connections, number of requests, number of the timeout, number of errors, response time, service type, and other metrics related to the specific business. + +A mainstream IT DevOps system generally includes a data-collection module, a data persistent module, and a visualization module. Telegraf and Grafana are some of the most popular data-collection and visualization modules. But data persistent module can be varied. OpenTSDB and InfluxDB are some prominent from others. In recent times, TDengine, as emerged time-series data platform provides more advantages including high performance, high reliability, easier management, easier maintenance. + +Here we introduce a way to build an IT DevOps system with TDengine, Telegraf, and Grafana. Even no need one line program code but just modify a few lines of configuration files. + +![IT-DevOps-Solutions-Telegraf.png](../../images/IT-DevOps-Solutions-Telegraf.png) + + +## Installation steps + +### Install Telegraf,Grafana and TDengine +Please refer to each component's official document for Telegraf, Grafana, and TDengine installation. + +### Telegraf +Please refer to the [official document](https://portal.influxdata.com/downloads/). + +### Grafana +Please refer to the [official document](https://grafana.com/grafana/download). + +### TDengine +Please download TDengine 2.3.0.0 or the above version from TAOS Data's [official website](http://taosdata.com/en/all-downloads/). + + +## Setup data chain +### Download TDengine plugin to Grafana plugin's directory + +```bash +1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.1/tdengine-datasource-3.1.1.zip +2. sudo unzip tdengine-datasource-3.1.1.zip -d /var/lib/grafana/plugins/ +3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine +4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini +5. sudo systemctl restart grafana-server.service +``` + +### Modify /etc/telegraf/telegraf.conf +Please add few lines in /etc/telegraf/telegraf.conf as below. Please fill database name for what you desire to save Telegraf's data in TDengine. Please specify the correct value for the hostname of the TDengine server/cluster, username, and password: +``` +[[outputs.http]] + url = "http://:6041/influxdb/v1/write?db=" + method = "POST" + timeout = "5s" + username = "" + password = "" + data_format = "influx" + influx_max_line_bytes = 250 +``` + +Then restart telegraf: +``` +sudo systemctl start telegraf +``` + + +### Import dashboard + +Use your Web browser to access IP:3000 to log in to the Grafana management interface. The default username and password are admin/admin。 + +Click the 'gear' icon from the left bar to select 'Plugins'. You could find the icon of the TDengine data source plugin. + +Click the 'plus' icon from the left bar to select 'Import'. You can download the dashboard JSON file from https://github.com/taosdata/grafanaplugin/blob/master/examples/telegraf/grafana/dashboards/telegraf-dashboard-v0.1.0.json then import it to the Grafana. After that, you should see the interface like: + +![IT-DevOps-Solutions-telegraf-dashboard.png](../../images/IT-DevOps-Solutions-telegraf-dashboard.png) + + +## Summary + +We demonstrated how to build a full-function IT DevOps system with TDengine, Telegraf, and Grafana. TDengine supports schemaless protocol data insertion capability from 2.3.0.0. Based on TDengine's powerful ecosystem software integration capability, the user can build a high efficient and easy-to-maintain IT DevOps system in a few minutes. Please find more detailed documentation about TDengine high-performance data insertion/query functions and more use cases from TAOS Data's official website. diff --git a/documentation20/en/14.devops/02.collectd/docs.md b/documentation20/en/14.devops/02.collectd/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..15a83d7f0c78f9e36122d4c7a0c125daddfa1c6a --- /dev/null +++ b/documentation20/en/14.devops/02.collectd/docs.md @@ -0,0 +1,84 @@ +# Rapidly build a IT DevOps system with TDengine + collectd/StatsD + Grafana + +## Background +TDengine is an open-source big data platform designed and optimized for Internet of Things (IoT), Connected Vehicles, and Industrial IoT. Besides the 10x faster time-series database, it provides caching, stream computing, message queuing and other functionalities to reduce the complexity and costs of development and operations. + +There are a lot of time-series data in the IT DevOps scenario, for example: +- Metrics of system resource: CPU, memory, IO and network status, etc. +- Metrics for software system: service status, number of connections, number of requests, number of the timeout, number of errors, response time, service type, and other metrics related to the specific business. + +A mainstream IT DevOps system generally includes a data-collection module, a data persistent module, and a visualization module. Telegraf and Grafana are some of the most popular data-collection and visualization modules. But data persistent module can be varied. OpenTSDB and InfluxDB are some prominent from others. In recent times, TDengine, as emerged time-series data platform provides more advantages including high performance, high reliability, easier management, easier maintenance. + +Here we introduce a way to build an IT DevOps system with TDengine, collectd/statsD, and Grafana. Even no need one line program code but just modify a few lines of configuration files. + +![IT-DevOps-Solutions-Collectd-StatsD.png](../../images/IT-DevOps-Solutions-Collectd-StatsD.png) + +## Installation steps +Please refer to each component's official document for collectd, StatsD, Grafana, and TDengine installation. + +### collectd +Please refer to the [official document](https://collectd.org/documentation.shtml). + +### StatsD +Please refer to the [official document](https://github.com/statsd/statsd). + +### Grafana +Please refer to the [official document](https://grafana.com/grafana/download). + +### TDengine +Please download TDengine 2.3.0.0 or the above version from TAOS Data's [official website](http://taosdata.com/cn/all-downloads/). + +## Setup data chain +### Download TDengine plugin to Grafana plugin's directory + +```bash +1. wget -c https://github.com/taosdata/grafanaplugin/releases/download/v3.1.1/tdengine-datasource-3.1.1.zip +2. sudo unzip tdengine-datasource-3.1.1.zip -d /var/lib/grafana/plugins/ +3. sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine +4. echo -e "[plugins]\nallow_loading_unsigned_plugins = tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini +5. sudo systemctl restart grafana-server.service +``` + +### To configure collectd +Please add a few lines in /etc/collectd/collectd.conf as below. Please specify the correct value for hostname and the port number: +``` +LoadPlugin network + + Server "" "" + + +sudo systemctl start collectd +``` + +### To configure StatsD +Please add a few lines in the config.js file then restart StatsD. Please use the correct hostname and port number of TDengine and taosAdapter: +``` +fill backends section with "./backends/repeater" +fill repeater section with { host:'', port: } +``` + +### Import dashboard + +Use your Web browser to access IP:3000 to log in to the Grafana management interface. The default username and password are admin/admin。 + +Click the gear icon from the left bar to select 'Plugins'. You could find the icon of the TDengine data source plugin. + +#### Import collectd dashboard + +Please download the dashboard JSON file from https://github.com/taosdata/grafanaplugin/blob/master/examples/collectd/grafana/dashboards/collect-metrics-with-tdengine-v0.1.0.json. + +Click the 'plus' icon from the left bar to select 'Import'. Then you should see the interface like: + +![IT-DevOps-Solutions-collectd-dashboard.png](../../images/IT-DevOps-Solutions-collectd-dashboard.png) + +#### Import StatsD dashboard + +Please download dashboard JSON file from https://github.com/taosdata/grafanaplugin/blob/master/examples/statsd/dashboards/statsd-with-tdengine-v0.1.0.json. + +Click the 'plus' icon from the left bar to select 'Import'. Then you should see the interface like: + +![IT-DevOps-Solutions-statsd-dashboard.png](../../images/IT-DevOps-Solutions-statsd-dashboard.png) + +## Summary + +We demonstrated how to build a full-function IT DevOps system with TDengine, collectd, StatsD, and Grafana. TDengine supports schemaless protocol data insertion capability from 2.3.0.0. Based on TDengine's powerful ecosystem software integration capability, the user can build a high efficient and easy-to-maintain IT DevOps system in few minutes. Please find more detailed documentation about TDengine high-performance data insertion/query functions and more use cases from TAOS Data's official website. diff --git a/documentation20/en/14.devops/03.immigrate/docs.md b/documentation20/en/14.devops/03.immigrate/docs.md new file mode 100644 index 0000000000000000000000000000000000000000..ebba18710f6218ce7043b563a99246ccf62035f9 --- /dev/null +++ b/documentation20/en/14.devops/03.immigrate/docs.md @@ -0,0 +1,436 @@ +# Best practice of immigration from OpenTSDB to TDengine + +As a distributed, scalable, HBase-based distributed temporal database system, OpenTSDB has been introduced and widely used in the field of operation and monitoring by people in DevOps due to its first-mover advantage. However, in recent years, with the rapid development of new technologies such as cloud computing, microservices, and containerization, enterprise-level services have become more and more diverse, and the architecture has become more and more complex, and the application operation infrastructure environment has become more and more diverse, which brings more and more pressure on system and operation monitoring. From this status quo, the use of OpenTSDB as the monitoring backend storage for DevOps is increasingly plagued by performance issues and slow feature upgrades, as well as the resulting increase in application deployment costs and reduced operational efficiency, which are becoming more and more serious as the system scales up. + +In this context, to meet the fast-growing IoT big data market and technical demands, TOS Data has developed an innovative big data processing product TDengine independently after learning the advantages of many traditional relational databases, NoSQL databases, stream computing engines, message queues, etc. TDengine has its unique advantages in time-series big data processing. TDengine can effectively solve the problems currently encountered by OpenTSDB. + +Compared with OpenTSDB, TDengine has the following distinctive features. + +- Performance of data writing and querying far exceeds that of OpenTSDB. +- Efficient compression mechanism for time-series data, which compresses less than 1/5 of the storage space on disk. +- The installation and deployment is very simple, a single installation package to complete the installation and deployment, no other third-party software, the entire installation and deployment process in seconds; +- The built-in functions cover all the query functions supported by OpenTSDB, and also support more time-series data query functions, scalar functions and aggregation functions, and support advanced query functions such as multiple time-window aggregation, join query, expression operation, multiple group aggregation, user-defined sorting, and user-defined functions. Adopting SQL-like syntax rules, it is easier to learn and basically has no learning cost. +- Supports up to 128 tags with a total tag length of up to 16 KB. +- In addition to HTTP, it also provides interfaces to Java, Python, C, Rust, Go, and other languages, and supports a variety of enterprise-class standard connector protocols such as JDBC. + +If we migrate applications originally running on OpenTSDB to TDengine, we can not only effectively reduce the consumption of computing and storage resources and the scale of deployed servers, but also greatly reduce the output of operation and maintenance costs, making operation and maintenance management simpler and easier, and significantly reducing the total cost of ownership. Like OpenTSDB, TDengine has also been open sourced, but the difference is that in addition to the stand-alone version, the latter has also achieved the open source of the cluster version, and the concern of being bound by the vendor has been swept away. + +In the following section we will explain how to migrate OpenTSDB applications to TDengine quickly, securely and reliably without coding, using the most typical and widely used DevOps scenarios. Subsequent chapters will provide more in-depth coverage to facilitate migration for non-DevOps scenarios. + +## Rapid migration of DevOps applications + +### 1. Typical Application Scenarios + +The overall system architecture of a typical DevOps application scenario is shown in the figure below (Figure 1). + +![IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch](../../images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg) +
Figure 1. Typical architecture in a DevOps scenario
+ +In this application scenario, there are Agent tools deployed in the application environment to collect machine metrics, network metrics, and application metrics, data collectors to aggregate information collected by agents, systems for data persistence storage and management, and tools for monitoring data visualization (e.g., Grafana, etc.). + +Among them, Agents deployed in application nodes are responsible for providing operational metrics from different sources to collectd/Statsd, and collectd/StatsD is responsible for pushing the aggregated data to the OpenTSDB cluster system and then visualizing the data using the visualization kanban board Grafana. + +### 2. Migration Service + +- **TDengine installation and deployment** + +First of all, TDengine should be installed. Download the latest stable version of TDengine from the official website, unzip it and run install.sh to install it. For help on using various installation packages, please refer to the blog ["Installation and uninstallation of various TDengine installation packages"](https://www.taosdata.com/blog/2019/08/09/566.html). + +Note that after the installation, do not start the taosd service immediately, but start it after the parameters are correctly configured. + +- **Adjusting the data collector configuration** + +In TDengine version 2.3, an HTTP service taosAdapter is automatically enabled after the backend service taosd is started. The taosAdapter is compatible with Influxdb's Line Protocol and OpenTSDB's telnet/JSON write protocol, allowing data collected by collectd and StatsD to be pushed directly to TDengine. + +If you use collectd, modify the configuration file in its default location /etc/collectd/collectd.conf to point to the IP address and port of the node where taosAdapter is deployed. Assuming the taosAdapter IP address is 192.168.1.130 and the port is 6046, configure it as follows + +```html +LoadPlugin write_tsdb + + + Host "192.168.1.130" + Port "6046" + HostTags "status=production" + StoreRates false + AlwaysAppendDS false + +``` + +This allows collectd to push the data to taosAdapter using the push to OpenTSDB plugin. taosAdapter will call the API to write the data to taosd, thus completing the writing of the data. If you are using StatsD adjust the profile information accordingly. + +- **Adjusting the Dashboard system** + +After the data has been written to TDengine properly, you can adapt Grafana to visualize the data written to TDengine. There is a connection plugin for Grafana in the TDengine installation directory connector/grafanaplugin. The way to use this plugin is simple. + +First copy the entire dist directory under the grafanaplugin directory to Grafana's plugins directory (the default address is /var/lib/grafana/plugins/), and then restart Grafana to see the TDengine data source under the Add Data Source menu. + +```shell +sudo cp -r . /var/lib/grafana/plugins/tdengine +sudo chown grafana:grafana -R /var/lib/grafana/plugins/tdengine +echo -e "[plugins]\nallow_loading_unsigned_plugins = taosdata-tdengine-datasource\n" | sudo tee -a /etc/grafana/grafana.ini + +# start grafana service +sudo service grafana-server restart +# or with systemd +sudo systemctl start grafana-server +``` + + + +In addition, TDengine provides two default Dashboard templates for users to quickly view the information saved to the TDengine repository. You can simply import the templates from the Grafana directory into Grafana to activate their use. + +![](../../images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg) + +
Figure 2. Importing Grafana Templates
+ +After the above steps, you have completed the migration of OpenTSDB to TDengine. You can see that the whole process is very simple, no code needs to be written, and only some configuration files need to be adjusted to complete the migration work. + +### 3. Post-migration architecture + +After the migration is completed, the overall architecture of the system at this time is shown in the figure below (Figure 3), and the acquisition side, data writing side, and monitoring presentation side all remain stable during the whole process, which does not involve any important changes or alterations except for very few configuration adjustments. OpenTSDB to TDengine migration action, using TDengine more powerful processing power and query performance. + +In most DevOps scenarios, if you have a small OpenTSDB cluster (3 nodes or less) as the storage side of DevOps, relying on OpenTSDB to provide data storage and query functions for the system persistence layer, then you can safely replace it with TDengine and save more compute and storage resources. With the same configuration of computing resources, a single TDengine can meet the service capacity provided by 3~5 OpenTSDB nodes. If the scale is relatively large, then a TDengine cluster is required. + +If your application is particularly complex, or the application domain is not a DevOps scenario, you can continue reading subsequent chapters for a more comprehensive and in-depth look at the advanced topics of migrating OpenTSDB applications to TDengine. + +![IT-DevOps-Solutions-Immigrate-TDengine-Arch](../../images/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg) + +
Figure 3. System architecture after the migration is complete
+ +## Migration evaluation and strategy for other scenarios + +### 1. Differences between TDengine and OpenTSDB + +This chapter describes in detail the differences between OpenTSDB and TDengine at the system functionality level. After reading this chapter, you can thoroughly evaluate whether you can migrate certain complex OpenTSDB-based applications to TDengine, and what you should pay attention to after the migration. + +TDengine currently only supports Grafana visual kanban rendering, so if your application uses a front-end kanban other than Grafana (e.g. [TSDash](https://github.com/facebook/tsdash), [Status Wolf](https://github.com/box/StatusWolf), etc.), then the front-end kanban cannot be migrated directly to TDengine and will need to be re-adapted to Grafana before it can function properly. + +As of version 2.3.0.x, TDengine can only support collectd and StatsD as data collection aggregation software, but more data collection aggregation software will be provided in the future. If you use other types of data aggregators on the collection side, your application needs to be adapted to these two data aggregation systems to be able to write data properly. In addition to the two data aggregation end software protocols mentioned above, TDengine also supports writing data directly via InfluxDB's row protocol and OpenTSDB's data writing protocol, JSON format, and you can rewrite the logic on the data push side to write data using the row protocols supported by TDengine. + +In addition, if you use the following features of OpenTSDB in your application, you need to understand the following considerations before migrating your application to TDengine. + +1. `/api/stats`: TDengine provides a new mechanism for handling cluster state monitoring to meet your application's monitoring and maintenance needs of your application. +2. `/api/tree`: TDengine uses a hierarchy of database -> supertable -> sub-table to organize and maintain timelines, with all timelines belonging to the same supertable at the same level in the system. However, it is possible to simulate a logical multi-level structure of the application through the special construction of different tag values. +3. `Rollup And PreAggregates`: The use of Rollup and PreAggregates requires the application to decide where to access the Rollup results and in some scenarios to access the original results, the opaqueness of this structure makes the application processing logic extremely complex and completely non-portable. TDengine does not support automatic downsampling of multiple timelines and preaggregates (for a range of periods) for the time being, but due to its high-performance query processing logic, it can provide high performance even without relying on Rollup and preaggregates. +4. `Rate`: TDengine provides two functions to calculate the rate of change of values, namely Derivative (whose calculation results are consistent with InfluxDB's Derivative behavior) and IRate (whose calculation results are consistent with the IRate function in Prometheus). However, the results of these two functions differ slightly from Rate, but are more powerful overall. In addition,** all the calculation functions provided by OpenTSDB are supported by TDengine with corresponding query functions, and the query functions of TDengine far exceed the query functions supported by OpenTSDB,** which can greatly simplify your application processing logic. + +Through the above introduction, I believe you should be able to understand the changes brought by the migration of OpenTSDB to TDengine, and this information will also help you correctly judge whether it is acceptable to migrate your application to TDengine, and experience the powerful timing data processing capability and convenient user experience provided by TDengine. + +### 2. Migration strategy + +First of all, the OpenTSDB-based system will be migrated involving data schema design, system scale estimation, data write end transformation, data streaming, and application adaptation; after that, the two systems will run in parallel for a period of time, and then the historical data will be migrated to TDengine. Of course, if your application has some functions that strongly depend on the above OpenTSDB features, and at the same time, You can consider keeping the original OpenTSDB system running while starting TDengine to provide the main services. + +## Data model design + +On the one hand, TDengine requires a strict schema definition for its incoming data. On the other hand, the data model of TDengine is richer than that of OpenTSDB, and the multi-valued model is compatible with all single-valued model building requirements. + +Now let's assume a DevOps scenario where we use collectd to collect base metrics of devices, including memory, swap, disk, etc. The schema in OpenTSDB is as follows: + +| No. | metric | value | type | tag1 | tag2 | tag3 | tag4 | tag5 | +| ---- | -------------- | ------ | ------ | ---- | ----------- | -------------------- | --------- | ------ | +| 1 | memory | value | double | host | memory_type | memory_type_instance | source | | +| 2 | swap | value | double | host | swap_type | swap_type_instance | source | | +| 3 | disk | value | double | host | disk_point | disk_instance | disk_type | source | + + + +TDengine requires data stored to have a data schema, i.e., you need to create a supertable and specify the schema of the supertable before writing the data. For data schema creation, you have two ways to do this: 1) Take full advantage of TDengine's native data writing support for OpenTSDB by calling the API provided by TDengine to write the data (in text line or JSON format) to the super table and automate the creation of the single-value model. And automate the creation of single-value models. This approach does not require major adjustments to the data writing application, nor does it require conversion of the written data format. + +At the C level, TDengine provides taos_insert_lines to write data in OpenTSDB format directly (in version 2.3.x this function corresponds to taos_schemaless_insert). For the code reference example, please refer to the sample code schemaless.c in the installation package directory. + + (2) Based on the full understanding of TDengine's data model, establish the mapping relationship between OpenTSDB and TDengine's data model adjustment manually, taking into account that OpenTSDB is a single-value mapping model, it is recommended to use the single-value model in TDengine. TDengine supports both multi-value and single-value models. + +- **Single-valued model**. + +The steps are as follows: the name of the metrics is used as the name of the TDengine super table, which is built with two basic data columns - timestamp and value, and the labels of the super table are equivalent to the label information of the metrics, and the number of labels is equal to the number of labels of the metrics. The sub-tables are named using a fixed rule row naming: `metric + '_' + tags1_value + '_' + tag2_value + '_' + tag3_value ... ` as sub-table names. + +Create 3 super tables in TDengine. + +```sql +create stable memory(ts timestamp, val float) tags(host binary(12),memory_type binary(20), memory_type_instance binary(20), source binary(20)); +create stable swap(ts timestamp, val double) tags(host binary(12), swap_type binary(20), swap_type_binary binary(20), source binary(20)); +create stable disk(ts timestamp, val double) tags(host binary(12), disk_point binary(20), disk_instance binary(20), disk_type binary(20), source binary(20)); +``` + + + +For sub-tables use dynamic table creation as shown below: + +```sql +insert into memory_vm130_memory_bufferred_collectd using memory tags(‘vm130’, ‘memory’, 'buffer', 'collectd') values(1632979445, 3.0656); +``` + +Eventually about 340 sub-tables and 3 super-tables will be created in the system. Note that if the use of concatenated tagged values causes the sub-table names to exceed the system limit (191 bytes), then some encoding (e.g. MD5) needs to be used to convert them to an acceptable length. + +- **Multi-value model** + +If you want to take advantage of TDengine's multi-value modeling capabilities, you need to first meet the requirements that different collection quantities have the same collection frequency and can reach the **data writing side simultaneously via a message queue**, thus ensuring that multiple metrics are written at once using SQL statements. The name of the metric is used as the name of the super table to create a multi-column model of data with the same collection frequency and capable of arriving at the same. The data can be collected with the same frequency and arrive in multiple columns. The names of the sub-tables are named using a fixed rule. Each metric above contains only one measurement value, so it cannot be transformed into a multi-value model. + + + +## Data triage and application adaptation + +Data is subscribed from the message queue and an adapted writer is started to write the data. + +After the data starts to be written for a sustained period, SQL statements can be used to check whether the amount of data written meets the expected write requirements. The following SQL statement is used to count the amount of data. + +```sql +select count(*) from memory +``` + +After completing the query, if the written data does not differ from the expected one, and there are no abnormal error messages from the writing program itself, then you can confirm that the data writing is complete and valid. + +TDengine does not support query or data fetch processing using OpenTSDB query syntax, but it does provide support for each type of OpenTSDB query. You can check Annex 2 for the corresponding query processing adjustments and application usage, or refer to the TDengine user manual for a full understanding of the types of queries supported by TDengine. + +TDengine supports the standard JDBC 3.0 interface for manipulating databases, but you can also use other types of high-level language connectors for querying and reading data to suit your application. See also the user manual for the specific operation and usage help. + +## Historical data migration + +### 1. Use the tool to migrate data automatically + +To facilitate the migration of historical data, we provide a plug-in for the data synchronization tool DataX, which can automatically write data to TDengine, it should be noted that DataX's automated data migration can only support the data migration process of single-value models. + +DataX Please refer to its help manual [github.com/taosdata/datax](http://github.com/taosdata/datax) for details on how to use DataX and how to use it to write data to TDengine. + +### 2. Migrate data manually + +If you need to use a multi-value model for data writing, you need to develop your tool to export data from OpenTSDB, then confirm which timelines can be merged and imported into the same timeline, and then write the time that can be imported at the same time to the database by SQL statement. + +The manual migration of data requires attention to two issues. + +1) When storing the exported data on the disk, the disk needs to have enough storage space to be able to adequately accommodate the exported data files. To avoid straining the disk file storage after exporting the full amount of data, a partial import mode can be adopted, with the timelines belonging to the same super table being exported first, and then the exported part of the data files are imported into the TDengine system + +(2) Under the full-load operation of the system, if there are enough remaining computing and IO resources, a multi-threaded import mechanism can be established to maximize the efficiency of data migration. Considering the huge load on the CPU brought by data parsing, the maximum number of parallel tasks needs to be controlled to avoid the overall system overload triggered by importing historical data. + +Due to the ease of operation of TDegnine itself, there is no need to perform index maintenance, data format change processing, etc. throughout the process, and the whole process only needs to be executed sequentially. + +Once the historical data is fully imported into TDengine, the two systems are running simultaneously, after which the query requests can be switched to TDengine, thus achieving a seamless application switchover. + +## Appendix 1: Correspondence table of OpenTSDB query functions + +**Avg** + +Equivalent function: avg + +Example. + +SELECT avg(val) FROM (SELECT first(val) FROM super_table WHERE ts >= startTime and ts <= endTime INTERVAL(20s) Fill(linear)) INTERVAL(20s) + +Notes. + +1. the value within the Interval needs to be the same as the interval value of the outer query. +As the interpolation of values in OpenTSDB uses linear interpolation, use fill(linear) to declare the interpolation type in the interpolation clause. The following functions with the same interpolation requirements are handled by this method. 3. +3. The 20s parameter in Interval means that the inner query will generate results in a 20-second window. In a real query, it needs to be adjusted to the time interval between different records. This ensures that the interpolation results are generated equivalently to the original data. +Due to the special interpolation strategy and mechanism of OpenTSDB, the way of interpolation before computation in Aggregate query makes it impossible for the computation result to be the same as TDengine. However, in the case of Downsample, TDengine, and OpenTSDB can obtain the same result (because OpenTSDB uses a completely different interpolation strategy for Aggregate and Downsample queries). +(since OpenTSDB uses a completely different interpolation strategy for aggregated and downsampled queries).[]() + + +**Count** + +Equivalent function: count + +Example. + +select count(*) from super_table_name; + + + +**Dev** + +Equivalent function: stddev + +Example. + +Select stddev(val) from table_name + + + +**Estimated percentiles** + +Equivalent function: apercentile + +Example. + +Select apercentile(col1, 50, “t-digest”) from table_name + +Remark. + +1. t-digest algorithm is used by default in OpenTSDB during approximate query processing, so to get the same calculation result, you need to specify the algorithm used in the apercentile function. tDengine can support two different approximate processing algorithms, which are declared by "default " and "t-digest" to declare. + + + +**First** + +Equivalent function: first + +Example. + +Select first(col1) from table_name + + + +**Last** + +Equivalent function: last + +Example. + +Select last(col1) from table_name + + + +**Max** + +Equivalent function: max + +Example. + +Select max(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s) + +Note: The Max function requires interpolation, for the reasons given above. + + + +**Min** + +Equivalent function: min + +Example. + +Select min(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s); + + + +**MinMax** + +Equivalent function: max + +Select max(val) from table_name + +Note: This function does not require interpolation, so it can be calculated directly. + + + +**MimMin** + +Equivalent function: min + +Select min(val) from table_name + +Note: This function does not require interpolation, so it can be calculated directly. + + + +**Percentile** + +Equivalent function: percentile + +备注: + + + +**Sum** + +Equivalent function: sum + +Select max(value) from (select first(val) value from table_name interval(10s) fill(linear)) interval(10s) + +Note: This function does not require interpolation, so it can be calculated directly. + + + +**Zimsum** + +Equivalent function: sum + +Select sum(val) from table_name + +Note: This function does not require interpolation, so it can be calculated directly. + + + +完整示例: + +```json +//OpenTSDB query JSON +query = { +"start":1510560000, +"end": 1515000009, +"queries":[{ +"aggregator": "count", +"metric":"cpu.usage_user", +}] +} + +// Equivalent SQL: +SELECT count(*) +FROM `cpu.usage_user` +WHERE ts>=1510560000 AND ts<=1515000009 +``` + + + +## Appendix 2: Resource Estimation Methodology + +### Data generation environment + +We still use the hypothetical environment from Chapter 4 with 3 measurements. The data writing rate for temperature and humidity is one record every 5 seconds, with a timeline of 100,000. Air quality is written at a rate of one record every 10 seconds, with a timeline of 10,000, and a query request frequency of 500 QPS. + +### Storage resource estimation + +Assuming that the number of sensor devices that generate data and require storage is `n`, the frequency of data generation is ` t` records/second, and the length of each record is `L` bytes, the size of data generated per day is `n×t×L` bytes. assuming a compression ratio of C, the size of data generated per day is `(n×t×L)/C` bytes. storage resources are estimated to be able to accommodate 1.5 years The storage resources are estimated to be able to accommodate 1.5 years of data size. Under the production environment, the compression ratio C of TDengine is generally between 5 and 7, while adding 20% redundancy to the final result, we can calculate the required storage resources. + +```matlab +(n×t×L)×(365×1.5)×(1+20%)/C +``` + +Combining the above formula and bringing the parameters into the calculation formula, the size of the raw data generated per year without considering tagging information is 11.8 TB. It should be noted that since tagging information is associated with each timeline in TDengine, it is not per record. So the size of the data volume to be recorded is somewhat reduced relative to the data generated, and this part of the tag data as a whole can be neglected. Assuming a compression ratio of 5, the size of the retained data ends up being 2.56 TB. + +### Storage device selection considerations + +The hard disk should be used with a better random read performance hard disk device, if you can have SSD, consider using SSD as much as possible. better random read performance of the disk is extremely helpful to improve the system query performance and can improve the overall query response performance of the system. To obtain better query performance, the performance index of single-threaded random read IOPS of the hard disk device should not be lower than 1000, it is better to reach 5000 IOPS or more. To obtain an evaluation of the current device random read IO performance, it is recommended that fio software be used to evaluate its operational performance (see Appendix 1 for details on how to use it) to confirm whether it can meet the large file random read performance requirements. + +Hard disk write performance has little impact on TDengine; TDengine writes in append write mode, so as long as it has good sequential write performance, both SAS hard disks and SSDs, in general, can meet TDengine's requirements for disk write performance well. + +### Computational resource estimation + +Due to the specificity of IoT data, after the frequency of data generation is fixed, the process of TDengine writing maintains a relatively fixed amount of resource consumption (both computation and storage). As described in [TDengine Operation and Maintenance](https://www.taosdata.com/cn/documentation/administrator), 22,000 writes per second in this system consumes less than 1 CPU core. + +In terms of estimating the CPU resources required for queries, assuming that the application requires 10,000 QPS from the database and each query consumes about 1 ms of CPU time, then each core provides 1,000 QPS of queries per second, and at least 10 cores are required to satisfy 10,000 QPS of query requests. To make the overall CPU load of the system less than 50%, the whole cluster needs 10 cores twice as many, i.e., 20 cores. + +### Memory resource estimation + +The database allocates memory for each Vnode by default 16MB*3 buffers, the cluster system includes 22 CPU cores, then 22 virtual node Vnodes will be established by default, each Vnode contains 1000 tables, then it can accommodate all the tables. Then it takes about 1 and a half hours to write a full block, thus triggering a dropped disk, which can be unadjusted. 22 Vnodes require a total memory cache of about 1GB. considering the memory required for queries, assuming a memory overhead of about 50MB per query, then 500 queries concurrently require about 25GB of memory. + +In summary, a single 16-core 32GB machine can be used, or a cluster of two 8-core 16GB machines can be used. + +## Appendix 3: Cluster Deployment and Startup + +TDengine provides a wealth of help documentation on many aspects of cluster installation and deployment, here is an index of responsive documentation for your reference. + +### Cluster Deployment + +The first step is to install TDengine. Download the latest stable version of TDengine from the official website, unzip it and run install.sh to install it. Please refer to the blog ["Installing and uninstalling TDengine packages"](https://www.taosdata.com/blog/2019/08/09/566.html) for help on using the various installation packages. + +Be careful not to start the taosd service immediately after the installation is complete, but only after the parameters are properly configured. + +### Set the running parameters and start the service + +To ensure that the system can get the necessary information to run properly. Please set the following key parameters correctly on the server-side. + +FQDN, firstEp, secondEP, dataDir, logDir, tmpDir, serverPort. The specific meaning of each parameter and the requirements for setting them can be found in the documentation "TDengine Cluster Installation, Management" (https://www.taosdata.com/cn/ documentation/cluster)". + +Follow the same steps to set the parameters on the node that needs to run and start the taosd service, then add the Dnode to the cluster. + +Finally, start taos and execute the command show dnodes, if you can see all the nodes that have joined the cluster, then the cluster is successfully built. For the specific operation procedure and notes, please refer to the document "[TDengine Cluster Installation, Management](https://www.taosdata.com/cn/documentation/cluster)". + +## Appendix 4: Super table names + +Since the metric name of OpenTSDB has a dot (". "However, the dot has a special meaning in TDengine, as a separator between database and table names. TDengine also provides escapes to allow users to use keywords or special separators (e.g., dot) in (super) table names. To use special characters, the table name needs to be enclosed in escape characters, e.g. `cpu.usage_user` would be a legal (super) table name. + +## Appendix 5: Reference Articles + +1. [Quickly build an IT Ops monitoring system using TDengine + collectd/StatsD + Grafana](https://www.taosdata.com/cn/documentation20/devops/collectd)(Chinese)_ +2. [Writing collection data directly to TDengine via collectd](https://www.taosdata.com/cn/documentation20/insert#collectd) (Chinese) + diff --git a/documentation20/en/images/IT-DevOps-Solutions-Collectd-StatsD.png b/documentation20/en/images/IT-DevOps-Solutions-Collectd-StatsD.png new file mode 100644 index 0000000000000000000000000000000000000000..b34aec45bdbe30bebbce532d6150c40f80399c25 Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-Collectd-StatsD.png differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg b/documentation20/en/images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..d3de5fb7a10a1cb22693468029bc26ad63a96d71 Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Arch.jpg differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg b/documentation20/en/images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg new file mode 100644 index 0000000000000000000000000000000000000000..be3704cb72d6c2614614852bfef17147ce49d061 Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-Immigrate-OpenTSDB-Dashboard.jpg differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg b/documentation20/en/images/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg new file mode 100644 index 0000000000000000000000000000000000000000..fd406a140beea43fbfe2c417c85b872cfd6a2219 Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-Immigrate-TDengine-Arch.jpg differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-Telegraf.png b/documentation20/en/images/IT-DevOps-Solutions-Telegraf.png new file mode 100644 index 0000000000000000000000000000000000000000..e1334bb937febd395eca0b0c44c8a2f315910606 Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-Telegraf.png differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-collectd-dashboard.png b/documentation20/en/images/IT-DevOps-Solutions-collectd-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..17d0fd31b9424b071783696668d5706b90274867 Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-collectd-dashboard.png differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-statsd-dashboard.png b/documentation20/en/images/IT-DevOps-Solutions-statsd-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..f122cbc5dc0bb5b7faccdbc7c4c8bcca59b6c9ed Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-statsd-dashboard.png differ diff --git a/documentation20/en/images/IT-DevOps-Solutions-telegraf-dashboard.png b/documentation20/en/images/IT-DevOps-Solutions-telegraf-dashboard.png new file mode 100644 index 0000000000000000000000000000000000000000..d695a3af30154d2fc2217996f3ff4878abab097c Binary files /dev/null and b/documentation20/en/images/IT-DevOps-Solutions-telegraf-dashboard.png differ diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 284926625f869d7ea82a800f4c470a83e2840404..95bf3e7b74f6e0f782a8cd5caefd196510358f87 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -846,7 +846,7 @@ vercomp () { function is_version_compatible() { - curr_version=`ls ${script_dir}/driver/libtaos.so* |cut -d '.' -f 3-6` + curr_version=`ls ${script_dir}/driver/libtaos.so* | awk -F 'libtaos.so.' '{print $2}'` if [ -f ${script_dir}/driver/vercomp.txt ]; then min_compatible_version=`cat ${script_dir}/driver/vercomp.txt` diff --git a/src/kit/shell/src/shellWindows.c b/src/kit/shell/src/shellWindows.c index 0babd88333c846c1f0b5dbe4baede4a6d38cbcdd..b1c85d951bf1f8cf801286f51b84d47d9c893b5c 100644 --- a/src/kit/shell/src/shellWindows.c +++ b/src/kit/shell/src/shellWindows.c @@ -17,7 +17,7 @@ #include "taos.h" #include "shellCommand.h" -#define SHELL_INPUT_MAX_COMMAND_SIZE 500000 +#define SHELL_INPUT_MAX_COMMAND_SIZE 10000 extern char configDir[]; diff --git a/src/plugins/CMakeLists.txt b/src/plugins/CMakeLists.txt index 9e0de204d78cb54bea240a734f2373b709b6c6f9..c7221a6d301ae09e47bd68c76a90599fd85dff2a 100644 --- a/src/plugins/CMakeLists.txt +++ b/src/plugins/CMakeLists.txt @@ -43,7 +43,7 @@ ELSE () COMMAND git clean -f -d BUILD_COMMAND CGO_CFLAGS=-I${CMAKE_CURRENT_SOURCE_DIR}/../inc CGO_LDFLAGS=-L${CMAKE_BINARY_DIR}/build/lib go build -ldflags "-s -w -X github.com/taosdata/taosadapter/version.CommitID=${taosadapter_commit_sha1}" INSTALL_COMMAND - COMMAND curl -sL https://github.com/upx/upx/releases/download/v3.96/upx-3.96-amd64_linux.tar.xz -o upx.tar.xz && tar xvJf upx.tar.xz --strip-components 1 > /dev/null && ./upx taosadapter || : + COMMAND curl -sL https://github.com/upx/upx/releases/download/v3.96/upx-3.96-amd64_linux.tar.xz -o upx.tar.xz && tar -xvJf upx.tar.xz -C ${CMAKE_BINARY_DIR} --strip-components 1 > /dev/null && ${CMAKE_BINARY_DIR}/upx taosadapter || : COMMAND cmake -E copy taosadapter ${CMAKE_BINARY_DIR}/build/bin COMMAND cmake -E make_directory ${CMAKE_BINARY_DIR}/test/cfg/ COMMAND cmake -E copy ./example/config/taosadapter.toml ${CMAKE_BINARY_DIR}/test/cfg/ diff --git a/src/plugins/taosadapter b/src/plugins/taosadapter index 6397bf5963f62f0aa5c4b9b961b16ed5c62579f1..88346a2e4e2e9282d2ec8b8c5264ca1ec23698a1 160000 --- a/src/plugins/taosadapter +++ b/src/plugins/taosadapter @@ -1 +1 @@ -Subproject commit 6397bf5963f62f0aa5c4b9b961b16ed5c62579f1 +Subproject commit 88346a2e4e2e9282d2ec8b8c5264ca1ec23698a1