diff --git a/README.md b/README.md
index 611d97aac9436bdcc732efcf98822f2dd11d74ab..a8126d74d44b2615f0d0a3e80fc330eaac6a2950 100644
--- a/README.md
+++ b/README.md
@@ -15,11 +15,11 @@
[](https://coveralls.io/github/taosdata/TDengine?branch=develop)
[](https://bestpractices.coreinfrastructure.org/projects/4201)
-English | [简体中文](README-CN.md) | We are hiring, check [here](https://tdengine.com/careers)
+English | [简体中文](README-CN.md) | [Lean more about TSDB](https://tdengine.com/tsdb)
# What is TDengine？
-TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/what-is-a-time-series-database/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages:
+TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages:
- **[High-Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression.
@@ -33,6 +33,8 @@ TDengine is an open source, high-performance, cloud native [time-series database
- **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide.
+For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/)
+
# Documentation
For user manual, system design and architecture, please refer to [TDengine Documentation](https://docs.tdengine.com) ([TDengine 文档](https://docs.taosdata.com))
@@ -319,6 +321,7 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo
Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to the project.
-# Join TDengine WeChat Group
+# Join TDengine User Community
-Add WeChat “tdengine” to join the group，you can communicate with other users.
+- Join [TDengine Discord Channel](https://discord.com/invite/VZdSuUg4pS?utm_id=discord)
+- Join wechat group by adding WeChat “tdengine”
diff --git a/cmake/cmake.platform b/cmake/cmake.platform
index 3aa1ffc07e73acdf480a21b478d55e05153694f8..e4d440d76edbfcf0d1d6f932cfa598fe0a0f43d2 100644
--- a/cmake/cmake.platform
+++ b/cmake/cmake.platform
@@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "")
SET(TD_ARM_32 TRUE)
ADD_DEFINITIONS("-D_TD_ARM_")
ADD_DEFINITIONS("-D_TD_ARM_32")
- ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64")
+ ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)")
MESSAGE(STATUS "The current platform is aarch64")
SET(PLATFORM_ARCH_STR "arm64")
SET(TD_ARM_64 TRUE)
diff --git a/cmake/cmake.version b/cmake/cmake.version
index db29644b387306ce8f3ee473921dab4c7d05b10a..d5721f767125721ef2d7ae897789b64debb6d0a1 100644
--- a/cmake/cmake.version
+++ b/cmake/cmake.version
@@ -2,7 +2,7 @@
IF (DEFINED VERNUMBER)
SET(TD_VER_NUMBER ${VERNUMBER})
ELSE ()
- SET(TD_VER_NUMBER "3.0.0.1")
+ SET(TD_VER_NUMBER "3.0.1.1")
ENDIF ()
IF (DEFINED VERCOMPATIBLE)
diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in
index f182beed33c76200649f93d96b68c153ec452b9a..1a2f5d396f33d74b961feee45e1ebd5c2bed7439 100644
--- a/cmake/taosadapter_CMakeLists.txt.in
+++ b/cmake/taosadapter_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosadapter
ExternalProject_Add(taosadapter
GIT_REPOSITORY https://github.com/taosdata/taosadapter.git
- GIT_TAG abed566
+ GIT_TAG 05fb2ff
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in
index b905e30489a3455fd834ba76a711675e88f28cac..6a3439ada9f6a285de9e59e438e808f2fdfc7238 100644
--- a/cmake/taostools_CMakeLists.txt.in
+++ b/cmake/taostools_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taos-tools
ExternalProject_Add(taos-tools
GIT_REPOSITORY https://github.com/taosdata/taos-tools.git
- GIT_TAG a4d9b92
+ GIT_TAG 125c77a
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in
index 54e898ecc093a5401b93080a13156f284a4cd0a4..04b1262cafd6f1dd984f568b847454c409d301ed 100644
--- a/cmake/taosws_CMakeLists.txt.in
+++ b/cmake/taosws_CMakeLists.txt.in
@@ -2,7 +2,7 @@
# taosws-rs
ExternalProject_Add(taosws-rs
GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git
- GIT_TAG 6fc47d7
+ GIT_TAG e771403
SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs"
BINARY_DIR ""
#BUILD_IN_SOURCE TRUE
diff --git a/docs/en/01-index.md b/docs/en/01-index.md
index 5265be42f81c4f43fa73e5b7d603d8989c2a5671..13552ea9dcfbae71a368cad9d55b4a0030ac266a 100644
--- a/docs/en/01-index.md
+++ b/docs/en/01-index.md
@@ -4,7 +4,7 @@ sidebar_label: Documentation Home
slug: /
---
-TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) time-series database optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It’s written mainly for architects, developers, and system administrators.
+TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) [time-series database](https://tdengine.com/tsdb/) optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It’s written mainly for architects, developers, and system administrators.
To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section.
@@ -22,6 +22,8 @@ If you want to know more about TDengine tools, the REST API, and connectors for
If you are very interested in the internal design of TDengine, please read the chapter [Inside TDengine](./tdinternal), which introduces the cluster design, data partitioning, sharding, writing, and reading processes in detail. If you want to study TDengine code or even contribute code, please read this chapter carefully.
+To get more general introduction about time series database, please read through [a series of articles](https://tdengine.com/tsdb/). To lean more competitive advantages about TDengine, please read through [a series of blogs](https://tdengine.com/tdengine/).
+
TDengine is an open-source database, and we would love for you to be a part of TDengine. If you find any errors in the documentation or see parts where more clarity or elaboration is needed, please click "Edit this page" at the bottom of each page to edit it directly.
Together, we make a difference!
diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md
index d385845d7c57203d6e1cc8ddb8d53307f2655914..20840883523607512bd8c21643d49c5b50d3a528 100644
--- a/docs/en/02-intro/index.md
+++ b/docs/en/02-intro/index.md
@@ -3,7 +3,7 @@ title: Introduction
toc_max_heading_level: 2
---
-TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](../develop/cache), [stream processing](../develop/stream), [data subscription](../develop/tmq) and other functionalities to reduce the system complexity and cost of development and operation.
+TDengine is an [open source](https://tdengine.com/tdengine/open-source-time-series-database/), [high-performance](https://tdengine.com/tdengine/high-performance-time-series-database/), [cloud native](https://tdengine.com/tdengine/cloud-native-time-series-database/) [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](../develop/cache), [stream processing](../develop/stream), [data subscription](../develop/tmq) and other functionalities to reduce the system complexity and cost of development and operation.
This section introduces the major features, competitive advantages, typical use-cases and benchmarks to help you get a high level overview of TDengine.
@@ -43,7 +43,7 @@ For more details on features, please read through the entire documentation.
## Competitive Advantages
-By making full use of [characteristics of time series data](https://tdengine.com/tsdb/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases, with the following advantages.
+By making full use of [characteristics of time series data](https://tdengine.com/tsdb/characteristics-of-time-series-data/), TDengine differentiates itself from other [time series databases](https://tdengine.com/tsdb), with the following advantages.
- **[High-Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression.
@@ -127,3 +127,8 @@ As a high-performance, scalable and SQL supported time-series database, TDengine
- [TDengine vs OpenTSDB](https://tdengine.com/2019/09/12/710.html)
- [TDengine vs Cassandra](https://tdengine.com/2019/09/12/708.html)
- [TDengine vs InfluxDB](https://tdengine.com/2019/09/12/706.html)
+
+## More readings
+- [Introduction to Time-Series Database](https://tdengine.com/tsdb/)
+- [Introduction to TDengine competitive advantages](https://tdengine.com/tdengine/)
+
diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md
index b0a0c25d85e99244858a461317ff54359d1ceff8..0b1b226c17100d56313b5480e26f437841afe8c7 100644
--- a/docs/en/04-concept/index.md
+++ b/docs/en/04-concept/index.md
@@ -6,101 +6,100 @@ In order to explain the basic concepts and provide some sample code, the TDengin
-
- Device ID
- Time Stamp
- Collected Metrics
- Tags
+
+
+ Device ID
+ Timestamp
+ Collected Metrics
+ Tags
-
-Device ID
-Time Stamp
-current
-voltage
-phase
-location
-groupId
-
-
-
-
-d1001
-1538548685000
-10.3
-219
-0.31
-California.SanFrancisco
-2
-
-
-d1002
-1538548684000
-10.2
-220
-0.23
-California.SanFrancisco
-3
-
-
-d1003
-1538548686500
-11.5
-221
-0.35
-California.LosAngeles
-3
-
-
-d1004
-1538548685500
-13.4
-223
-0.29
-California.LosAngeles
-2
-
-
-d1001
-1538548695000
-12.6
-218
-0.33
-California.SanFrancisco
-2
-
-
-d1004
-1538548696600
-11.8
-221
-0.28
-California.LosAngeles
-2
-
-
-d1002
-1538548696650
-10.3
-218
-0.25
-California.SanFrancisco
-3
-
-
-d1001
-1538548696800
-12.3
-221
-0.31
-California.SanFrancisco
-2
-
-
+
+ current
+ voltage
+ phase
+ location
+ groupid
+
+
+
+
+ d1001
+ 1538548685000
+ 10.3
+ 219
+ 0.31
+ California.SanFrancisco
+ 2
+
+
+ d1002
+ 1538548684000
+ 10.2
+ 220
+ 0.23
+ California.SanFrancisco
+ 3
+
+
+ d1003
+ 1538548686500
+ 11.5
+ 221
+ 0.35
+ California.LosAngeles
+ 3
+
+
+ d1004
+ 1538548685500
+ 13.4
+ 223
+ 0.29
+ California.LosAngeles
+ 2
+
+
+ d1001
+ 1538548695000
+ 12.6
+ 218
+ 0.33
+ California.SanFrancisco
+ 2
+
+
+ d1004
+ 1538548696600
+ 11.8
+ 221
+ 0.28
+ California.LosAngeles
+ 2
+
+
+ d1002
+ 1538548696650
+ 10.3
+ 218
+ 0.25
+ California.SanFrancisco
+ 3
+
+
+ d1001
+ 1538548696800
+ 12.3
+ 221
+ 0.31
+ California.SanFrancisco
+ 2
+
+
Table 1: Smart meter example data
-Each row contains the device ID, time stamp, collected metrics (current, voltage, phase as above), and static tags (location and groupId in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated time stamps.
+Each row contains the device ID, timestamp, collected metrics (`current`, `voltage`, `phase` as above), and static tags (`location` and `groupid` in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated timestamps.
## Metric
@@ -112,22 +111,22 @@ Label/Tag refers to the static properties of sensors, equipment or other types o
## Data Collection Point
-Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same time stamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points.
+Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same timestamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points.
## Table
Since time-series data is most likely to be structured data, TDengine adopts the traditional relational database model to process them with a short learning curve. You need to create a database, create tables, then insert data points and execute queries to explore the data.
-To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices D1001, D1002, D1003, and D1004 to store the data collected. This design has several benefits:
+To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices d1001, d1002, d1003, and d1004 to store the data collected. This design has several benefits:
1. Since the metric data from different DCP are fully independent, the data source of each DCP is unique, and a table has only one writer. In this way, data points can be written in a lock-free manner, and the writing speed can be greatly improved.
2. For a DCP, the metric data generated by DCP is ordered by timestamp, so the write operation can be implemented by simple appending, which further greatly improves the data writing speed.
3. The metric data from a DCP is continuously stored, block by block. If you read data for a period of time, it can greatly reduce random read operations and improve read and query performance by orders of magnitude.
4. Inside a data block for a DCP, columnar storage is used, and different compression algorithms are used for different data types. Metrics generally don't vary as significantly between themselves over a time range as compared to other metrics, which allows for a higher compression rate.
-If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. ** One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.**
+If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. **One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.**
-TDengine suggests using DCP ID as the table name (like D1001 in the above table). Each DCP may collect one or multiple metrics (like the current, voltage, phase as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the time stamp as the index, and won’t build the index on any metrics stored. Column wise storage is used.
+TDengine suggests using DCP ID as the table name (like d1001 in the above table). Each DCP may collect one or multiple metrics (like the `current`, `voltage`, `phase` as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the timestamp as the index, and won’t build the index on any metrics stored. Column wise storage is used.
Complex devices, such as connected cars, may have multiple DCPs. In this case, multiple tables are created for a single device, one table per DCP.
@@ -156,9 +155,16 @@ The relationship between a STable and the subtables created based on this STable
Queries can be executed on both a table (subtable) and a STable. For a query on a STable, TDengine will treat the data in all its subtables as a whole data set for processing. TDengine will first find the subtables that meet the tag filter conditions, then scan the time-series data of these subtables to perform aggregation operation, which reduces the number of data sets to be scanned which in turn greatly improves the performance of data aggregation across multiple DCPs. In essence, querying a supertable is a very efficient aggregate query on multiple DCPs of the same type.
-In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table meters.
+In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table `meters`.
+
+To better understand the data model using metrics, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example.
+
+
+
+
-To better understand the data model using metri, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example. 
+Figure 1. Meters Data Model Diagram
## Database
@@ -172,4 +178,4 @@ FQDN (Fully Qualified Domain Name) is the full domain name of a specific compute
Each node of a TDengine cluster is uniquely identified by an End Point, which consists of an FQDN and a Port, such as h1.tdengine.com:6030. In this way, when the IP changes, we can still use the FQDN to dynamically find the node without changing any configuration of the cluster. In addition, FQDN is used to facilitate unified access to the same cluster from the Intranet and the Internet.
-TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management.
+TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management.
diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md
index de5b620a779557a8a3b8422a14caf67b354d1e7a..6191492b37e09096ad1d7c8da4567130579ca20b 100644
--- a/docs/en/05-get-started/01-docker.md
+++ b/docs/en/05-get-started/01-docker.md
@@ -13,7 +13,7 @@ If Docker is already installed on your computer, run the following command:
docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
```
-Note that TDengine Server uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
+Note that TDengine Server 3.0 uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed.
Run the following command to ensure that your container is running:
@@ -21,7 +21,7 @@ Run the following command to ensure that your container is running:
docker ps
```
-Enter the container and open the bash shell:
+Enter the container and open the `bash` shell:
```shell
docker exec -it bash
@@ -31,68 +31,68 @@ You can now access TDengine or run other Linux commands.
Note: For information about installing docker, see the [official documentation](https://docs.docker.com/get-docker/).
-## Insert Data into TDengine
-
-You can use the `taosBenchmark` tool included with TDengine to write test data into your deployment.
+## Open the TDengine CLI
-To do so, run the following command:
+On the container, run the following command to open the TDengine CLI:
- ```bash
- $ taosBenchmark
-
- ```
+```
+$ taos
-This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
+taos>
- The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required depends on the hardware specifications of the local system.
+```
- You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](/reference/taosbenchmark).
+## Test data insert performance
-## Open the TDengine CLI
+After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
-On the container, run the following command to open the TDengine CLI:
+Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal.
+```bash
+taosBenchmark
```
-$ taos
-taos>
+This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `California.Campbell`, `California.Cupertino`, `California.LosAngeles`, `California.MountainView`, `California.PaloAlto`, `California.SanDiego`, `California.SanFrancisco`, `California.SanJose`, `California.SantaClara` or `California.Sunnyvale`.
-```
+The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds.
-## Query Data in TDengine
+You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
-After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance. For example:
+## Test data query performance
-From the TDengine CLI query the number of rows in the `meters` supertable:
+After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance:
+
+From the TDengine CLI (taos) query the number of rows in the `meters` supertable:
```sql
-select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
Query the average, maximum, and minimum values of all 100 million rows of data:
```sql
-select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-Query the number of rows whose `location` tag is `San Francisco`:
+Query the number of rows whose `location` tag is `California.SanFrancisco`:
```sql
-select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco";
```
Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`:
```sql
-select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-Query the average, maximum, and minimum values for table `d10` in 1 second intervals:
+Query the average, maximum, and minimum values for table `d10` in 10 second intervals:
```sql
-select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
-In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
+
+In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
## Additional Information
diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md
index 88096a759c58529d4150c0a750a4354a88da988f..292fa82183b5fb6d8c3449a5282e360f4841710a 100644
--- a/docs/en/05-get-started/03-package.md
+++ b/docs/en/05-get-started/03-package.md
@@ -9,23 +9,24 @@ import PkgListV3 from "/components/PkgListV3";
For information about installing TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine).
-The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter).
+The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface (CLI, taos), and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter).
-The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download a lite package that includes only `taosd` and the C/C++ connector.
+The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector.
-The TDengine Community Edition is released as .deb and .rpm packages. The .deb package can be installed on Debian, Ubuntu, and derivative systems. The .rpm package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the .deb or .rpm package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows servers.
+The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows.
## Installation
-1. Download the .deb installation package.
-
+1. Download the Deb installation package.
+
2. In the directory where the package is located, use `dpkg` to install the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
sudo dpkg -i TDengine-server--Linux-x64.deb
```
@@ -34,11 +35,12 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
1. Download the .rpm installation package.
-
+
2. In the directory where the package is located, use rpm to install the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
sudo rpm -ivh TDengine-server--Linux-x64.rpm
```
@@ -47,11 +49,12 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
1. Download the .tar.gz installation package.
-
+
2. In the directory where the package is located, use `tar` to decompress the package:
+> Please replace `` with the corresponding version of the package downloaded
+
```bash
-# Enter the name of the package that you downloaded.
tar -zxvf TDengine-server--Linux-x64.tar.gz
```
@@ -96,23 +99,23 @@ sudo apt-get install tdengine
This installation method is supported only for Debian and Ubuntu.
::::
-
+
-Note: TDengine only supports Windows Server 2016/2019 and windows 10/11 system versions on the windows platform.
+Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the Windows platform.
1. Download the Windows installation package.
-
+
2. Run the downloaded package to install TDengine.
:::info
-For information about TDengine releases, see [Release History](../../releases).
+For information about TDengine releases, see [Release History](../../releases/tdengine).
:::
:::note
-On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the end point of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine.
+On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the endpoint of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine.
:::
@@ -147,7 +150,7 @@ Active: inactive (dead)
After confirming that TDengine is running, run the `taos` command to access the TDengine CLI.
-The following `systemctl` commands can help you manage TDengine:
+The following `systemctl` commands can help you manage TDengine service:
- Start TDengine Server: `systemctl start taosd`
@@ -159,7 +162,7 @@ The following `systemctl` commands can help you manage TDengine:
:::info
-- The `systemctl` command requires _root_ privileges. If you are not logged in as the `root` user, use the `sudo` command.
+- The `systemctl` command requires _root_ privileges. If you are not logged in as the _root_ user, use the `sudo` command.
- The `systemctl stop taosd` command does not instantly stop TDengine Server. The server is stopped only after all data in memory is flushed to disk. The time required depends on the cache size.
- If your system does not include `systemd`, you can run `/usr/local/taos/bin/taosd` to start TDengine manually.
@@ -174,23 +177,9 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin
-## Test data insert performance
+## Command Line Interface (CLI)
-After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
-
-```bash
-taosBenchmark
-```
-
-This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`.
-
-The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in less than a minute.
-
-You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
-
-## Command Line Interface
-
-You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, run the following command:
+You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line.
```bash
taos
@@ -205,52 +194,71 @@ taos>
For example, you can create and delete databases and tables and run all types of queries. Each SQL command must be end with a semicolon (;). For example:
```sql
-create database demo;
-use demo;
-create table t (ts timestamp, speed int);
-insert into t values ('2019-07-15 00:00:00', 10);
-insert into t values ('2019-07-15 01:00:00', 20);
-select * from t;
+CREATE DATABASE demo;
+USE demo;
+CREATE TABLE t (ts TIMESTAMP, speed INT);
+INSERT INTO t VALUES ('2019-07-15 00:00:00', 10);
+INSERT INTO t VALUES ('2019-07-15 01:00:00', 20);
+SELECT * FROM t;
+
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
+
Query OK, 2 row(s) in set (0.003128s)
```
You can also can monitor the deployment status, add and remove user accounts, and manage running instances. You can run the TDengine CLI on either Linux or Windows machines. For more information, see [TDengine CLI](../../reference/taos-shell/).
-
+
+## Test data insert performance
+
+After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance:
+
+Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal.
+
+```bash
+taosBenchmark
+```
+
+This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `California.Campbell`, `California.Cupertino`, `California.LosAngeles`, `California.MountainView`, `California.PaloAlto`, `California.SanDiego`, `California.SanFrancisco`, `California.SanJose`, `California.SantaClara` or `California.Sunnyvale`.
+
+The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds.
+
+You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark).
+
## Test data query performance
-After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance:
+After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance:
-From the TDengine CLI query the number of rows in the `meters` supertable:
+From the TDengine CLI (taos) query the number of rows in the `meters` supertable:
```sql
-select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
Query the average, maximum, and minimum values of all 100 million rows of data:
```sql
-select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-Query the number of rows whose `location` tag is `San Francisco`:
+Query the number of rows whose `location` tag is `California.SanFrancisco`:
```sql
-select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco";
```
Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`:
```sql
-select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-Query the average, maximum, and minimum values for table `d10` in 1 second intervals:
+Query the average, maximum, and minimum values for table `d10` in 10 second intervals:
```sql
-select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
-In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
+
+In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/).
diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md
index 1f861119250e9e32fcac3344a5bdfb760e6191a7..118a2483737743ddbca7eece14f0a8a9a88993c4 100644
--- a/docs/en/12-taos-sql/06-select.md
+++ b/docs/en/12-taos-sql/06-select.md
@@ -348,19 +348,15 @@ SELECT ... FROM (SELECT ... FROM ...) ...;
:::info
-- Only one layer of nesting is allowed, that means no sub query is allowed within a sub query
-- The result set returned by the inner query will be used as a "virtual table" by the outer query. The "virtual table" can be renamed using `AS` keyword for easy reference in the outer query.
-- Sub query is not allowed in continuous query.
+- The result of a nested query is returned as a virtual table used by the outer query. It's recommended to give an alias to this table for the convenience of using it in the outer query.
- JOIN operation is allowed between tables/STables inside both inner and outer queries. Join operation can be performed on the result set of the inner query.
-- UNION operation is not allowed in either inner query or outer query.
-- The functions that can be used in the inner query are the same as those that can be used in a non-nested query.
+- The features that can be used in the inner query are the same as those that can be used in a non-nested query.
- `ORDER BY` inside the inner query is unnecessary and will slow down the query performance significantly. It is best to avoid the use of `ORDER BY` inside the inner query.
- Compared to the non-nested query, the functionality that can be used in the outer query has the following restrictions:
- Functions
- - If the result set returned by the inner query doesn't contain timestamp column, then functions relying on timestamp can't be used in the outer query, like `TOP`, `BOTTOM`, `FIRST`, `LAST`, `DIFF`.
- - Functions that need to scan the data twice can't be used in the outer query, like `STDDEV`, `PERCENTILE`.
- - `IN` operator is not allowed in the outer query but can be used in the inner query.
- - `GROUP BY` is not supported in the outer query.
+ - If the result set returned by the inner query doesn't contain timestamp column, then functions relying on timestamp can't be used in the outer query, like INTERP,DERIVATIVE, IRATE, LAST_ROW, FIRST, LAST, TWA, STATEDURATION, TAIL, UNIQUE.
+ - If the result set returned by the inner query are not sorted in order by timestamp, then functions relying on data ordered by timestamp can't be used in the outer query, like LEASTSQUARES, ELAPSED, INTERP, DERIVATIVE, IRATE, TWA, DIFF, STATECOUNT, STATEDURATION, CSUM, MAVG, TAIL, UNIQUE.
+ - Functions that need to scan the data twice can't be used in the outer query, like PERCENTILE.
:::
diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md
index f74d0dbe5c6ae2019b266df4c55a13a49630bf71..ab1d2f900b993cee80109fec22f01face20a3dbb 100644
--- a/docs/en/12-taos-sql/10-function.md
+++ b/docs/en/12-taos-sql/10-function.md
@@ -126,7 +126,7 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause]
SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
```
-**Description**: The rounded down value of a specific field
+**Description**: The rounded down value of a specific field
**More explanations**: The restrictions are same as those of the `CEIL` function.
#### LOG
@@ -173,7 +173,7 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause]
SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
```
-**Description**: The rounded value of a specific field.
+**Description**: The rounded value of a specific field.
**More explanations**: The restrictions are same as those of the `CEIL` function.
@@ -434,7 +434,7 @@ SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause];
**More explanations**:
- You can specify a time zone in the following format: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。 For example, TO_ISO8601(1, "+00:00").
-- If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp
+- If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp
- If the input is a column of TIMESTAMP type, the precision of the returned value is same as the precision set for the current data base in use
@@ -769,14 +769,14 @@ SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_nam
**Explanations**：
- bin_type: parameter to indicate the bucket type, valid inputs are: "user_input", "linear_bin", "log_bin"。
-- bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively:
- - "user_input": "[1, 3, 5, 7]":
+- bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively:
+ - "user_input": "[1, 3, 5, 7]":
User specified bin values.
-
+
- "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}"
"start" - bin starting point. "width" - bin offset. "count" - number of bins generated. "infinity" - whether to add（-inf, inf）as start/end point in generated set of bins.
The above "linear_bin" descriptor generates a set of bins: [-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf].
-
+
- "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}"
"start" - bin starting point. "factor" - exponential factor of bin offset. "count" - number of bins generated. "infinity" - whether to add（-inf, inf）as start/end point in generated range of bins.
The above "linear_bin" descriptor generates a set of bins: [-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf].
@@ -862,9 +862,9 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] RA
- `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter.
- The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input.
-- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified.
-- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter.
-- Interpolation is performed based on `FILL` parameter.
+- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified.
+- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter.
+- Interpolation is performed based on `FILL` parameter.
- `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable.
### LAST
@@ -917,7 +917,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
**Return value type**:Same as the data type of the column being operated upon
-**Applicable data types**: Numeric, Timestamp
+**Applicable data types**: Numeric
**Applicable table types**: standard tables and supertables
@@ -932,7 +932,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
**Return value type**:Same as the data type of the column being operated upon
-**Applicable data types**: Numeric, Timestamp
+**Applicable data types**: Numeric
**Applicable table types**: standard tables and supertables
@@ -968,7 +968,7 @@ SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
**Applicable table types**: standard tables and supertables
-**More explanations**:
+**More explanations**:
This function cannot be used in expression calculation.
- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline
@@ -1046,10 +1046,10 @@ SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause]
**Applicable table types**: standard tables and supertables
-**More explanations**:
-
+**More explanations**:
+
- Arithmetic operation can't be performed on the result of `csum` function
-- Can only be used with aggregate functions This function can be used with supertables and standard tables.
+- Can only be used with aggregate functions This function can be used with supertables and standard tables.
- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline
@@ -1067,8 +1067,8 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER
**Applicable table types**: standard tables and supertables
-**More explanation**:
-
+**More explanation**:
+
- It can be used together with `PARTITION BY tbname` against a STable.
- It can be used together with a selected column. For example: select \_rowts, DERIVATIVE() from。
@@ -1086,7 +1086,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER
**Applicable table types**: standard tables and supertables
-**More explanation**:
+**More explanation**:
- The number of result rows is the number of rows subtracted by one, no output for the first row
- It can be used together with a selected column. For example: select \_rowts, DIFF() from。
@@ -1123,9 +1123,9 @@ SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
**Applicable table types**: standard tables and supertables
-**More explanations**:
-
-- Arithmetic operation can't be performed on the result of `MAVG`.
+**More explanations**:
+
+- Arithmetic operation can't be performed on the result of `MAVG`.
- Can only be used with data columns, can't be used with tags. - Can't be used with aggregate functions.
- Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline
diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md
index 707089abe54fc12bb09de47c1c51af1a32b8cbcd..296c2376427b28ea0fa8404517af7215ffa7030f 100644
--- a/docs/en/12-taos-sql/12-distinguished.md
+++ b/docs/en/12-taos-sql/12-distinguished.md
@@ -5,11 +5,11 @@ title: Time-Series Extensions
As a purpose-built database for storing and processing time-series data, TDengine provides time-series-specific extensions to standard SQL.
-These extensions include tag-partitioned queries and windowed queries.
+These extensions include partitioned queries and windowed queries.
-## Tag-Partitioned Queries
+## Partitioned Queries
-When you query a supertable, you may need to partition the supertable by tag and perform additional operations on a specific partition. In this case, you can use the following SQL clause:
+When you query a supertable, you may need to partition the supertable by some dimensions and perform additional operations on a specific partition. In this case, you can use the following SQL clause:
```sql
PARTITION BY part_list
@@ -17,22 +17,24 @@ PARTITION BY part_list
part_list can be any scalar expression, such as a column, constant, scalar function, or a combination of the preceding items.
-A PARTITION BY clause with a tag is processed as follows:
+A PARTITION BY clause is processed as follows:
-- The PARTITION BY clause must occur after the WHERE clause and cannot be used with a JOIN clause.
-- The PARTITION BY clause partitions the super table by the specified tag group, and the specified calculation is performed on each partition. The calculation performed is determined by the rest of the statement - a window clause, GROUP BY clause, or SELECT clause.
-- You can use PARTITION BY together with a window clause or GROUP BY clause. In this case, the window or GROUP BY clause takes effect on every partition. For example, the following statement partitions the table by the location tag, performs downsampling over a 10 minute window, and returns the maximum value:
+- The PARTITION BY clause must occur after the WHERE clause
+- The PARTITION BY caluse partitions the data according to the specified dimentions, then perform computation on each partition. The performed computation is determined by the rest of the statement - a window clause, GROUP BY clause, or SELECT clause.
+- The PARTITION BY clause can be used together with a window clause or GROUP BY clause. In this case, the window or GROUP BY clause takes effect on every partition. For example, the following statement partitions the table by the location tag, performs downsampling over a 10 minute window, and returns the maximum value:
```sql
select max(current) from meters partition by location interval(10m)
```
+The most common usage of PARTITION BY is partitioning the data in subtables by tags then perform computation when querying data in a supertable. More specifically, `PARTITION BY TBNAME` partitions the data of each subtable into a single timeline, and this method facilitates the statistical analysis in many use cases of processing timeseries data.
+
## Windowed Queries
Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are three kinds of windows: time window, status window, and session window. There are two kinds of time windows: sliding window and flip time/tumbling window. The query syntax is as follows:
```sql
-SELECT function_list FROM tb_name
+SELECT select_list FROM tb_name
[WHERE where_condition]
[SESSION(ts_col, tol_val)]
[STATE_WINDOW(col)]
@@ -42,15 +44,9 @@ SELECT function_list FROM tb_name
The following restrictions apply:
-### Restricted Functions
-
-- Aggregate functions and select functions can be used in `function_list`, with each function having only one output. For example COUNT, AVG, SUM, STDDEV, LEASTSQUARES, PERCENTILE, MIN, MAX, FIRST, LAST. Functions having multiple outputs, such as DIFF or arithmetic operations can't be used.
-- `LAST_ROW` can't be used together with window aggregate.
-- Scalar functions, like CEIL/FLOOR, can't be used with window aggregate.
-
### Other Rules
-- The window clause must occur after the PARTITION BY clause and before the GROUP BY clause. It cannot be used with a GROUP BY clause.
+- The window clause must occur after the PARTITION BY clause. It cannot be used with a GROUP BY clause.
- SELECT clauses on windows can contain only the following expressions:
- Constants
- Aggregate functions
@@ -82,7 +78,7 @@ These pseudocolumns occur after the aggregation clause.
1. A huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum number of interpolation values that can be returned in a single query is 10,000,000.
2. The result set is in ascending order of timestamp when you aggregate by time window.
-3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `GROUP BY` is not used in the query, the result set will be returned in ascending order of timestamp; otherwise the result set is not exactly in the order of ascending timestamp in each group.
+3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `PARTITION BY` is not used in the query, the result set will be returned in strict ascending order of timestamp; otherwise the result set will be returned in the order of ascending timestamp in each group.
:::
@@ -112,9 +108,9 @@ When using time windows, note the following:
Please note that the `timezone` parameter should be configured to be the same value in the `taos.cfg` configuration file on client side and server side.
- The result set is in ascending order of timestamp when you aggregate by time window.
-### Status Window
+### State Window
-In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two status windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12]. Status window is not applicable to STable for now.
+In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two state windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12].
@@ -124,13 +120,19 @@ In case of using integer, bool, or string to represent the status of a device at
SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status);
```
+Only care about the information of the status window when the status is 2. For example:
+
+```
+SELECT * FROM (SELECT COUNT(*) AS cnt, FIRST(ts) AS fst, status FROM temp_tb_1 STATE_WINDOW(status)) t WHERE status = 2;
+```
+
### Session Window
The primary key, i.e. timestamp, is used to determine which session window a row belongs to. As shown in the figure below, if the limit of time interval for the session window is specified as 12 seconds, then the 6 rows in the figure constitutes 2 time windows, [2019-04-28 14:22:10，2019-04-28 14:22:30] and [2019-04-28 14:23:10，2019-04-28 14:23:30] because the time difference between 2019-04-28 14:22:30 and 2019-04-28 14:23:10 is 40 seconds, which exceeds the time interval limit of 12 seconds.
-If the time interval between two continuous rows are within the time interval specified by `tol_value` they belong to the same session window; otherwise a new session window is started automatically. Session window is not supported on STable for now.
+If the time interval between two continuous rows are within the time interval specified by `tol_value` they belong to the same session window; otherwise a new session window is started automatically.
```
diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md
index 6f166c8034382b0613845d18470556622106e673..f7eb067c960cd9599153a208cd2781b49b090511 100644
--- a/docs/en/12-taos-sql/20-keywords.md
+++ b/docs/en/12-taos-sql/20-keywords.md
@@ -5,7 +5,9 @@ title: Reserved Keywords
## Keyword List
-There are about 200 keywords reserved by TDengine, they can't be used as the name of database, STable or table with either upper case, lower case or mixed case. The following list shows all reserved keywords:
+There are more than 200 keywords reserved by TDengine, they can't be used as the name of database, table, STable, subtable, column or tag with either upper case, lower case or mixed case. If you need to use these keywords, use the symbol `` ` `` to enclose the keywords, e.g. \`ADD\`.
+
+The following list shows all reserved keywords:
### A
@@ -14,15 +16,20 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- ACCOUNTS
- ADD
- AFTER
+- AGGREGATE
- ALL
- ALTER
+- ANALYZE
- AND
+- APPS
- AS
- ASC
+- AT_ONCE
- ATTACH
### B
+- BALANCE
- BEFORE
- BEGIN
- BETWEEN
@@ -32,19 +39,27 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- BITNOT
- BITOR
- BLOCKS
+- BNODE
+- BNODES
- BOOL
+- BUFFER
+- BUFSIZE
- BY
### C
- CACHE
-- CACHELAST
+- CACHEMODEL
+- CACHESIZE
- CASCADE
+- CAST
- CHANGE
+- CLIENT_VERSION
- CLUSTER
- COLON
- COLUMN
- COMMA
+- COMMENT
- COMP
- COMPACT
- CONCAT
@@ -52,15 +67,18 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- CONNECTION
- CONNECTIONS
- CONNS
+- CONSUMER
+- CONSUMERS
+- CONTAINS
- COPY
+- COUNT
- CREATE
-- CTIME
+- CURRENT_USER
### D
- DATABASE
- DATABASES
-- DAYS
- DBS
- DEFERRED
- DELETE
@@ -69,18 +87,23 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- DESCRIBE
- DETACH
- DISTINCT
+- DISTRIBUTED
- DIVIDE
- DNODE
- DNODES
- DOT
- DOUBLE
- DROP
+- DURATION
### E
+- EACH
+- ENABLE
- END
-- EQ
+- EVERY
- EXISTS
+- EXPIRED
- EXPLAIN
### F
@@ -88,18 +111,20 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- FAIL
- FILE
- FILL
+- FIRST
- FLOAT
+- FLUSH
- FOR
- FROM
-- FSYNC
+- FUNCTION
+- FUNCTIONS
### G
-- GE
- GLOB
+- GRANT
- GRANTS
- GROUP
-- GT
### H
@@ -110,15 +135,18 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- ID
- IF
- IGNORE
-- IMMEDIA
+- IMMEDIATE
- IMPORT
- IN
-- INITIAL
+- INDEX
+- INDEXES
+- INITIALLY
+- INNER
- INSERT
- INSTEAD
- INT
- INTEGER
-- INTERVA
+- INTERVAL
- INTO
- IS
- ISNULL
@@ -126,6 +154,7 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
### J
- JOIN
+- JSON
### K
@@ -135,46 +164,57 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
### L
-- LE
+- LAST
+- LAST_ROW
+- LICENCES
- LIKE
- LIMIT
- LINEAR
- LOCAL
-- LP
-- LSHIFT
-- LT
### M
- MATCH
+- MAX_DELAY
- MAXROWS
+- MERGE
+- META
- MINROWS
- MINUS
+- MNODE
- MNODES
- MODIFY
- MODULES
### N
-- NE
+- NCHAR
+- NEXT
+- NMATCH
- NONE
- NOT
- NOTNULL
- NOW
- NULL
+- NULLS
### O
- OF
- OFFSET
+- ON
- OR
- ORDER
+- OUTPUTTYPE
### P
-- PARTITION
+- PAGES
+- PAGESIZE
+- PARTITIONS
- PASS
- PLUS
+- PORT
- PPS
- PRECISION
- PREV
@@ -182,47 +222,63 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
### Q
+- QNODE
+- QNODES
- QTIME
-- QUERIE
+- QUERIES
- QUERY
-- QUORUM
### R
- RAISE
-- REM
+- RANGE
+- RATIO
+- READ
+- REDISTRIBUTE
+- RENAME
- REPLACE
- REPLICA
- RESET
-- RESTRIC
+- RESTRICT
+- RETENTIONS
+- REVOKE
+- ROLLUP
- ROW
-- RP
-- RSHIFT
### S
+- SCHEMALESS
- SCORES
- SELECT
- SEMI
+- SERVER_STATUS
+- SERVER_VERSION
- SESSION
- SET
- SHOW
-- SLASH
+- SINGLE_STABLE
- SLIDING
- SLIMIT
-- SMALLIN
+- SMA
+- SMALLINT
+- SNODE
+- SNODES
- SOFFSET
-- STable
-- STableS
+- SPLIT
+- STABLE
+- STABLES
- STAR
- STATE
-- STATEMEN
-- STATE_WI
+- STATE_WINDOW
+- STATEMENT
- STORAGE
- STREAM
- STREAMS
+- STRICT
- STRING
+- SUBSCRIPTIONS
- SYNCDB
+- SYSINFO
### T
@@ -233,19 +289,24 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
- TBNAME
- TIMES
- TIMESTAMP
+- TIMEZONE
- TINYINT
+- TO
+- TODAY
- TOPIC
- TOPICS
+- TRANSACTION
+- TRANSACTIONS
- TRIGGER
+- TRIM
- TSERIES
+- TTL
### U
-- UMINUS
- UNION
- UNSIGNED
- UPDATE
-- UPLUS
- USE
- USER
- USERS
@@ -253,9 +314,13 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
### V
+- VALUE
- VALUES
+- VARCHAR
- VARIABLE
- VARIABLES
+- VERBOSE
+- VGROUP
- VGROUPS
- VIEW
- VNODES
@@ -263,14 +328,25 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam
### W
- WAL
+- WAL_FSYNC_PERIOD
+- WAL_LEVEL
+- WAL_RETENTION_PERIOD
+- WAL_RETENTION_SIZE
+- WAL_ROLL_PERIOD
+- WAL_SEGMENT_SIZE
+- WATERMARK
- WHERE
+- WINDOW_CLOSE
+- WITH
+- WRITE
### \_
- \_C0
-- \_QSTART
-- \_QSTOP
- \_QDURATION
-- \_WSTART
-- \_WSTOP
+- \_QEND
+- \_QSTART
+- \_ROWTS
- \_WDURATION
+- \_WEND
+- \_WSTART
diff --git a/docs/en/12-taos-sql/25-grant.md b/docs/en/12-taos-sql/25-grant.md
index b9a3fa2321c8d073845d0cf9157ce335c930e06f..f895567c621f123778c8a711a68357a889ca0098 100644
--- a/docs/en/12-taos-sql/25-grant.md
+++ b/docs/en/12-taos-sql/25-grant.md
@@ -9,15 +9,54 @@ This document describes how to manage permissions in TDengine.
## Create a User
```sql
-CREATE USER use_name PASS 'password';
+CREATE USER user_name PASS 'password' [SYSINFO {1|0}];
```
This statement creates a user account.
-The maximum length of use_name is 23 bytes.
+The maximum length of user_name is 23 bytes.
The maximum length of password is 128 bytes. The password can include leters, digits, and special characters excluding single quotation marks, double quotation marks, backticks, backslashes, and spaces. The password cannot be empty.
+`SYSINFO` indicates whether the user is allowed to view system information. `1` means allowed, `0` means not allowed. System information includes server configuration, dnode, vnode, storage. The default value is `1`.
+
+For example, we can create a user whose password is `123456` and is able to view system information.
+
+```sql
+taos> create user test pass '123456' sysinfo 1;
+Query OK, 0 of 0 rows affected (0.001254s)
+```
+
+## View Users
+
+To show the users in the system, please use
+
+```sql
+SHOW USERS;
+```
+
+This is an example:
+
+```sql
+taos> show users;
+ name | super | enable | sysinfo | create_time |
+================================================================================
+ test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 |
+ root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 |
+Query OK, 2 rows in database (0.001657s)
+```
+
+Alternatively, you can get the user information by querying a built-in table, INFORMATION_SCHEMA.INS_USERS. For example:
+
+```sql
+taos> select * from information_schema.ins_users;
+ name | super | enable | sysinfo | create_time |
+================================================================================
+ test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 |
+ root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 |
+Query OK, 2 rows in database (0.001953s)
+```
+
## Delete a User
```sql
@@ -40,6 +79,13 @@ alter_user_clause: {
- ENABLE: Specify whether the user is enabled or disabled. 1 indicates enabled and 0 indicates disabled.
- SYSINFO: Specify whether the user can query system information. 1 indicates that the user can query system information and 0 indicates that the user cannot query system information.
+For example, you can use below command to disable user `test`:
+
+```sql
+taos> alter user test enable 0;
+Query OK, 0 of 0 rows affected (0.001160s)
+```
+
## Grant Permissions
@@ -62,7 +108,7 @@ priv_level : {
}
```
-Grant permissions to a user.
+Grant permissions to a user, this feature is only available in enterprise edition.
Permissions are granted on the database level. You can grant read or write permissions.
@@ -92,4 +138,4 @@ priv_level : {
```
-Revoke permissions from a user.
+Revoke permissions from a user, this feature is only available in enterprise edition.
diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md
index a4679983f2bc77bb4e438f5d43fa1b8beb39b120..74a5564a2a7b82009e6b2406290a5c15afbf3c1d 100644
--- a/docs/en/13-operation/10-monitor.md
+++ b/docs/en/13-operation/10-monitor.md
@@ -2,7 +2,7 @@
title: TDengine Monitoring
---
-After TDengine is started, a database named `log` is created automatically to help with monitoring. Information that includes CPU, memory and disk usage, bandwidth, number of requests, disk I/O speed, slow queries, is written into the `log` database at a predefined interval. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console.
+After TDengine is started, it automatically writes monitoring data including CPU, memory and disk usage, bandwidth, number of requests, disk I/O speed, slow queries, into a designated database at a predefined interval through taosKeeper. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console.
The collection of the monitoring information is enabled by default, but can be disabled by parameter `monitor` in the configuration file.
@@ -10,7 +10,7 @@ The collection of the monitoring information is enabled by default, but can be d
TDinsight is a complete solution which uses the monitoring database `log` mentioned previously, and Grafana, to monitor a TDengine cluster.
-From version 2.3.3.0, more monitoring data has been added in the `log` database. Please refer to [TDinsight Grafana Dashboard](https://grafana.com/grafana/dashboards/15167) to learn more details about using TDinsight to monitor TDengine.
+Please refer to [TDinsight Grafana Dashboard](../../reference/tdinsight) to learn more details about using TDinsight to monitor TDengine.
A script `TDinsight.sh` is provided to deploy TDinsight automatically.
@@ -30,31 +30,14 @@ Prepare：
2. Grafana Alert Notification
-There are two ways to setup Grafana alert notification.
+You can use below command to setup Grafana alert notification.
-- An existing Grafana Notification Channel can be specified with parameter `-E`, the notifier uid of the channel can be obtained by `curl -u admin:admin localhost:3000/api/alert-notifications |jq`
+An existing Grafana Notification Channel can be specified with parameter `-E`, the notifier uid of the channel can be obtained by `curl -u admin:admin localhost:3000/api/alert-notifications |jq`
```bash
sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E
```
-- The AliCloud SMS alert built in TDengine data source plugin can be enabled with parameter `-s`, the parameters of enabling this plugin are listed below:
-
- - `-I`: AliCloud SMS Key ID
- - `-K`: AliCloud SMS Key Secret
- - `-S`: AliCloud SMS Signature
- - `-C`: SMS notification template
- - `-T`: Input parameters in JSON format for the SMS notification template, for example`{"alarm_level":"%s","time":"%s","name":"%s","content":"%s"}`
- - `-B`: List of mobile numbers to be notified
-
- Below is an example of the full command using the AliCloud SMS alert.
-
- ```bash
- sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -s \
- -I XXXXXXX -K XXXXXXXX -S taosdata -C SMS_1111111 -B 18900000000 \
- -T '{"alarm_level":"%s","time":"%s","name":"%s","content":"%s"}'
- ```
-
Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`.
For more use cases and restrictions please refer to [TDinsight](/reference/tdinsight/).
diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx
index 0d391c6ac308c5e9e998e2e7e3423cc5a809905e..530287e2a424592045755eeecf37894f59639ef3 100644
--- a/docs/en/14-reference/03-connector/06-rust.mdx
+++ b/docs/en/14-reference/03-connector/06-rust.mdx
@@ -155,15 +155,15 @@ async fn demo(taos: &Taos, db: &str) -> Result<(), Error> {
let inserted = taos.exec_many([
// create super table
"CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \
- TAGS (`groupid` INT, `location` BINARY(16))",
+ TAGS (`groupid` INT, `location` BINARY(24))",
// create child table
- "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')",
+ "CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')",
// insert into child table
"INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)",
// insert with NULL values
"INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)",
// insert and automatically create table with tags if not exists
- "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)",
+ "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119, 0.33)",
// insert many records in a single sql
"INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)",
]).await?;
diff --git a/docs/en/14-reference/03-connector/_linux_install.mdx b/docs/en/14-reference/03-connector/_linux_install.mdx
index 07f8fb5c7118d84c53017f44d9811a3357944cfc..97f6cd333961d24bd015a2ebcf7223f89fecca2c 100644
--- a/docs/en/14-reference/03-connector/_linux_install.mdx
+++ b/docs/en/14-reference/03-connector/_linux_install.mdx
@@ -4,7 +4,7 @@ import PkgListV3 from "/components/PkgListV3";
Apply external notifier uid to TDinsight dashboard.
-Alibaba Cloud SMS as Notifier:
--s, --sms-enabled To enable tdengine-datasource plugin builtin Alibaba Cloud SMS webhook.
--N, --sms-notifier-name Provisioning notifier name.[default: TDinsight Builtin SMS]
--U, --sms-notifier-uid Provisioning notifier uid, use lowercase notifier name by default.
--D, --sms-notifier-is-default Set notifier as default.
--I, --sms-access-key-id Alibaba Cloud SMS access key id
--K, --sms-access-key-secret Alibaba Cloud SMS access key secret
--S, --sms-sign-name Sign name
--C, --sms-template-code Template code
--T, --sms-template-param Template param, a escaped JSON string like '{"alarm_level":"%s","time":"%s","name":"%s","content":"%s"}'
--B, --sms-phone-numbers Comma-separated numbers list, eg "189xxxxxxxx,132xxxxxxxx"
--L, --sms-listen-addr [default: 127.0.0.1:9100]
```
Most command-line options can take effect the same as environment variables.
@@ -136,17 +132,6 @@ Most command-line options can take effect the same as environment variables.
| -t | --tdinsight-title | TDINSIGHT_DASHBOARD_TITLE | TDinsight dashboard title. [Default: TDinsight] | -e | -tdinsight-title
| -e | --tdinsight-editable | TDINSIGHT_DASHBOARD_EDITABLE | If the dashboard is configured to be editable. [Default: false] | -e | --external
| -E | --external-notifier | EXTERNAL_NOTIFIER | Apply the external notifier uid to the TDinsight dashboard. | -s
-| -s | --sms-enabled | SMS_ENABLED | Enable the tdengine-datasource plugin built into Alibaba Cloud SMS webhook. | -s
-| -N | --sms-notifier-name | SMS_NOTIFIER_NAME | The name of the provisioning notifier. [Default: `TDinsight Builtin SMS`] | -U
-| -U | --sms-notifier-uid | SMS_NOTIFIER_UID | "Notification Channel" `uid`, lowercase of the program name is used by default, other characters are replaced by "-". |-sms
-| -D | --sms-notifier-is-default | SMS_NOTIFIER_IS_DEFAULT | Set built-in SMS notification to default value. |-sms-notifier-is-default
-| -I | --sms-access-key-id | SMS_ACCESS_KEY_ID | Alibaba Cloud SMS access key id |
-| -K | --sms-access-key-secret | SMS_ACCESS_KEY_SECRET | AliCloud SMS-access-secret-key |
-| -S | --sms-sign-name | SMS_SIGN_NAME | Signature |
-| -C | --sms-template-code | SMS_TEMPLATE_CODE | Template code |
-| -T | --sms-template-param | SMS_TEMPLATE_PARAM | JSON template for template parameters |
-| -B | --sms-phone-numbers | SMS_PHONE_NUMBERS | A comma-separated list of phone numbers, e.g. `"189xxxxxxxx,132xxxxxxxx"` |
-| -L | --sms-listen-addr | SMS_LISTEN_ADDR | Built-in SMS webhook listener address, default is `127.0.0.1:9100` |
Suppose you start a TDengine database on host `tdengine` with HTTP API port `6041`, user `root1`, and password `pass5ord`. Execute the script.
@@ -166,24 +151,10 @@ Use the `uid` value obtained above as `-E` input.
sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier
```
-If you want to use the [Alibaba Cloud SMS](https://www.aliyun.com/product/sms) service as a notification channel, you should enable it with the `-s` flag add the following parameters.
-
-- `-N`: Notification Channel name, default is `TDinsight Builtin SMS`.
-- `-U`: Channel uid, default is lowercase of `name`, any other character is replaced with -, for the default `-N`, its uid is `tdinsight-builtin-sms`.
-- `-I`: Alibaba Cloud SMS access key id.
-- `-K`: Alibaba Cloud SMS access secret key.
-- `-S`: Alibaba Cloud SMS signature.
-- `-C`: Alibaba Cloud SMS template id.
-- `-T`: Alibaba Cloud SMS template parameters, for JSON format template, example is as follows `'{"alarm_level":"%s", "time":"%s", "name":"%s", "content":"%s"}'`. There are four parameters: alarm level, time, name and alarm content.
-- `-B`: a list of phone numbers, separated by a comma `,`.
-
If you want to monitor multiple TDengine clusters, you need to set up numerous TDinsight dashboards. Setting up non-default TDinsight requires some changes: the `-n` `-i` `-t` options need to be changed to non-default names, and `-N` and `-L` should also be changed if using the built-in SMS alerting feature.
```bash
sudo . /TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata -i tdinsight-env1 -t 'TDinsight Env1'
-# If using built-in SMS notifications
-sudo . /TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata -i tdinsight-env1 -t 'TDinsight Env1' \
- -s -N 'Env1 SMS' -I xx -K xx -S xx -C SMS_XX -T '' -B 00000000000 -L 127.0.0.01:10611
```
Please note that the configuration data source, notification channel, and dashboard are not changeable on the front end. You should update the configuration again via this script or manually change the configuration file in the `/etc/grafana/provisioning` directory (this is the default directory for Grafana, use the `-P` option to change it as needed).
@@ -249,21 +220,23 @@ Save and test. It will report 'TDengine Data source is working' under normal cir
### Importing dashboards
-Point to **+** / **Create** - **import** (or `/dashboard/import` url).
+In the page of configuring data source, click **Dashboards** tab.
-Type the dashboard ID `15167` in the **Import via grafana.com** location and **Load**.
+Choose `TDengine for 3.x` and click `import`.
+
+After the importing is done, `TDinsight for 3.x` dashboard is available on the page of `search dashboards by name`.
-
+
-Once the import is complete, the full page view of TDinsight is shown below.
+In the `TDinsight for 3.x` dashboard, choose the database used by taosKeeper to store monitoring data, you can see the monitoring result.
-
+
## TDinsight dashboard details
-The TDinsight dashboard is designed to provide the usage and status of TDengine-related resources [dnodes, mnodes, vnodes](../../taos-sql/node/) or databases.
+The TDinsight dashboard is designed to provide the usage and status of TDengine-related resources, e.g. dnodes, mnodes, vnodes and databases.
Details of the metrics are as follows.
@@ -285,7 +258,6 @@ This section contains the current information and status of the cluster, the ale
- **Measuring Points Used**: The number of measuring points used to enable the alert rule (no data available in the community version, healthy by default).
- **Grants Expire Time**: the expiration time of the enterprise version of the enabled alert rule (no data available for the community version, healthy by default).
- **Error Rate**: Aggregate error rate (average number of errors per second) for alert-enabled clusters.
-- **Variables**: `show variables` table display.
### DNodes Status
@@ -294,7 +266,6 @@ This section contains the current information and status of the cluster, the ale
- **DNodes Status**: simple table view of `show dnodes`.
- **DNodes Lifetime**: the time elapsed since the dnode was created.
- **DNodes Number**: the number of DNodes changes.
-- **Offline Reason**: if any dnode status is offline, the reason for offline is shown as a pie chart.
### MNode Overview
@@ -309,7 +280,6 @@ This section contains the current information and status of the cluster, the ale
1. **Requests Rate(Inserts per Second)**: average number of inserts per second.
2. **Requests (Selects)**: number of query requests and change rate (count of second).
-3. **Requests (HTTP)**: number of HTTP requests and request rate (count of second).
### Database
@@ -319,9 +289,8 @@ Database usage, repeated for each value of the variable `$database` i.e. multipl
1. **STables**: number of super tables.
2. **Total Tables**: number of all tables.
-3. **Sub Tables**: the number of all super table subtables.
-4. **Tables**: graph of all normal table numbers over time.
-5. **Tables Number Foreach VGroups**: The number of tables contained in each VGroups.
+3. **Tables**: number of normal tables.
+4. **Table number for each vgroup**: number of tables per vgroup.
### DNode Resource Usage
@@ -356,12 +325,11 @@ Currently, only the number of logins per minute is reported.
Support monitoring taosAdapter request statistics and status details. Includes.
-1. **http_request**: contains the total number of requests, the number of failed requests, and the number of requests being processed
-2. **top 3 request endpoint**: data of the top 3 requests by endpoint group
-3. **Memory Used**: taosAdapter memory usage
-4. **latency_quantile(ms)**: quantile of (1, 2, 5, 9, 99) stages
-5. **top 3 failed request endpoint**: data of the top 3 failed requests by endpoint grouping
-6. **CPU Used**: taosAdapter CPU usage
+1. **http_request_inflight**: number of real-time requests.
+2. **http_request_total**: number of total requests.
+3. **http_request_fail**: number of failed requets.
+4. **CPU Used**: CPU usage of taosAdapter.
+5. **Memory Used**: Memory usage of taosAdapter.
## Upgrade
@@ -403,13 +371,6 @@ services:
TDENGINE_API: ${TDENGINE_API}
TDENGINE_USER: ${TDENGINE_USER}
TDENGINE_PASS: ${TDENGINE_PASS}
- SMS_ACCESS_KEY_ID: ${SMS_ACCESS_KEY_ID}
- SMS_ACCESS_KEY_SECRET: ${SMS_ACCESS_KEY_SECRET}
- SMS_SIGN_NAME: ${SMS_SIGN_NAME}
- SMS_TEMPLATE_CODE: ${SMS_TEMPLATE_CODE}
- SMS_TEMPLATE_PARAM: '${SMS_TEMPLATE_PARAM}'
- SMS_PHONE_NUMBERS: $SMS_PHONE_NUMBERS
- SMS_LISTEN_ADDR: ${SMS_LISTEN_ADDR}
ports:
- 3000:3000
volumes:
diff --git a/docs/en/20-third-party/13-Jupyter.md b/docs/en/20-third-party/13-Jupyter.md
new file mode 100644
index 0000000000000000000000000000000000000000..fbd7e530f0959740c53e48ce1d73d92ce0d6c5c5
--- /dev/null
+++ b/docs/en/20-third-party/13-Jupyter.md
@@ -0,0 +1,98 @@
+---
+sidebar_label: JupyterLab
+title: Connect JupyterLab to TDengine
+---
+
+JupyterLab is the next generation of the ubiquitous Jupyter Notebook. In this note we show you how to install the TDengine Python connector to connect to TDengine in JupyterLab. You can then insert data and perform queries against the TDengine instance within JupyterLab.
+
+## Install JupyterLab
+Installing JupyterLab is very easy. Installation instructions can be found at:
+
+https://jupyterlab.readthedocs.io/en/stable/getting_started/installation.html.
+
+If you don't feel like clicking on the link here are the instructions.
+Jupyter's preferred Python package manager is pip, so we show the instructions for pip.
+You can also use **conda** or **pipenv** if you are managing Python environments.
+````
+pip install jupyterlab
+````
+
+For **conda** you can run:
+````
+conda install -c conda-forge jupyterlab
+````
+
+For **pipenv** you can run:
+````
+pipenv install jupyterlab
+pipenv shell
+````
+
+## Run JupyterLab
+You can start JupyterLab from the command line by running:
+````
+jupyter lab
+````
+This will automatically launch your default browser and connect to your JupyterLab instance, usually on port 8888.
+
+## Install the TDengine Python connector
+You can now install the TDengine Python connector as follows.
+
+Start a new Python kernel in JupyterLab.
+
+If using **conda** run the following:
+````
+# Install a conda package in the current Jupyter kernel
+import sys
+!conda install --yes --prefix {sys.prefix} taospy
+````
+If using **pip** run the following:
+````
+# Install a pip package in the current Jupyter kernel
+import sys
+!{sys.executable} -m pip install taospy
+````
+
+## Connect to TDengine
+You can find detailed examples to use the Python connector, in the TDengine documentation here.
+Once you have installed the TDengine Python connector in your JupyterLab kernel, the process of connecting to TDengine is the same as that you would use if you weren't using JupyterLab.
+Each TDengine instance, has a database called "log" which has monitoring information about the TDengine instance.
+In the "log" database there is a [supertable](https://docs.tdengine.com/taos-sql/stable/) called "disks_info".
+
+The structure of this table is as follows:
+````
+taos> desc disks_info;
+ Field | Type | Length | Note |
+=================================================================================
+ ts | TIMESTAMP | 8 | |
+ datadir_l0_used | FLOAT | 4 | |
+ datadir_l0_total | FLOAT | 4 | |
+ datadir_l1_used | FLOAT | 4 | |
+ datadir_l1_total | FLOAT | 4 | |
+ datadir_l2_used | FLOAT | 4 | |
+ datadir_l2_total | FLOAT | 4 | |
+ dnode_id | INT | 4 | TAG |
+ dnode_ep | BINARY | 134 | TAG |
+Query OK, 9 row(s) in set (0.000238s)
+````
+
+The code below is used to fetch data from this table into a pandas DataFrame.
+
+````
+import sys
+import taos
+import pandas
+
+def sqlQuery(conn):
+ df: pandas.DataFrame = pandas.read_sql("select * from log.disks_info limit 500", conn)
+ print(df)
+ return df
+
+conn = taos.connect()
+
+result = sqlQuery(conn)
+
+print(result)
+````
+
+TDengine has connectors for various languages including Node.js, Go, PHP and there are kernels for these languages which can be found [here](https://github.com/jupyter/jupyter/wiki/Jupyter-kernels).
diff --git a/docs/en/28-releases.md b/docs/en/28-releases.md
deleted file mode 100644
index a0c9eb119999571fb973b5e2243f237b8833b167..0000000000000000000000000000000000000000
--- a/docs/en/28-releases.md
+++ /dev/null
@@ -1,9 +0,0 @@
----
-sidebar_label: Releases
-title: Released Versions
----
-
-import Release from "/components/ReleaseV3";
-
-
-
-
- Device ID
- Time Stamp
- Collected Metrics
- Tags
+
+
+ Device ID
+ Timestamp
+ Collected Metrics
+ Tags
-
-Device ID
-Time Stamp
-current
-voltage
-phase
-location
-groupId
-
-
-
-
-d1001
-1538548685000
-10.3
-219
-0.31
-California.SanFrancisco
-2
-
-
-d1002
-1538548684000
-10.2
-220
-0.23
-California.SanFrancisco
-3
-
-
-d1003
-1538548686500
-11.5
-221
-0.35
-California.LosAngeles
-3
-
-
-d1004
-1538548685500
-13.4
-223
-0.29
-California.LosAngeles
-2
-
-
-d1001
-1538548695000
-12.6
-218
-0.33
-California.SanFrancisco
-2
-
-
-d1004
-1538548696600
-11.8
-221
-0.28
-California.LosAngeles
-2
-
-
-d1002
-1538548696650
-10.3
-218
-0.25
-California.SanFrancisco
-3
-
-
-d1001
-1538548696800
-12.3
-221
-0.31
-California.SanFrancisco
-2
-
-
+
+ current
+ voltage
+ phase
+ location
+ groupid
+
+
+
+
+ d1001
+ 1538548685000
+ 10.3
+ 219
+ 0.31
+ California.SanFrancisco
+ 2
+
+
+ d1002
+ 1538548684000
+ 10.2
+ 220
+ 0.23
+ California.SanFrancisco
+ 3
+
+
+ d1003
+ 1538548686500
+ 11.5
+ 221
+ 0.35
+ California.LosAngeles
+ 3
+
+
+ d1004
+ 1538548685500
+ 13.4
+ 223
+ 0.29
+ California.LosAngeles
+ 2
+
+
+ d1001
+ 1538548695000
+ 12.6
+ 218
+ 0.33
+ California.SanFrancisco
+ 2
+
+
+ d1004
+ 1538548696600
+ 11.8
+ 221
+ 0.28
+ California.LosAngeles
+ 2
+
+
+ d1002
+ 1538548696650
+ 10.3
+ 218
+ 0.25
+ California.SanFrancisco
+ 3
+
+
+ d1001
+ 1538548696800
+ 12.3
+ 221
+ 0.31
+ California.SanFrancisco
+ 2
+
+
-
表 1：智能电表数据示例
+
表 1. 智能电表数据示例
-每一条记录都有设备 ID，时间戳，采集的物理量以及每个设备相关的静态标签。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
+每一条记录都有设备 ID、时间戳、采集的物理量（如上表中的 `current`、`voltage` 和 `phase`）以及每个设备相关的静态标签（`location` 和 `groupid`）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。
-## 采集量 (Metric)
+## 采集量（Metric）
采集量是指传感器、设备或其他类型采集点采集的物理量，比如电流、电压、温度、压力、GPS 位置等，是随时间变化的，数据类型可以是整型、浮点型、布尔型，也可是字符串。随着时间的推移，存储的采集量的数据量越来越大。智能电表示例中的电流、电压、相位就是采集量。
-## 标签 (Label/Tag)
+## 标签（Label/Tag）
-标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的location与groupId就是标签。
+标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的 `location` 与 `groupid` 就是标签。
-## 数据采集点 (Data Collection Point)
+## 数据采集点（Data Collection Point）
-数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的d1001, d1002, d1003, d1004等就是数据采集点。
+数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的 d1001、d1002、d1003、d1004 等就是数据采集点。
-## 表 (Table)
+## 表（Table）
因为采集量一般是结构化数据，同时为降低学习门槛，TDengine 采用传统的关系型数据库模型管理数据。用户需要先创建库，然后创建表，之后才能插入或查询数据。
@@ -129,50 +128,56 @@ description: TDengine 的数据模型和基本概念
如果采用传统的方式，将多个数据采集点的数据写入一张表，由于网络延时不可控，不同数据采集点的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个数据采集点的数据是难以保证连续存储在一起的。**采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。**
-TDengine 建议用数据采集点的名字（如上表中的 D1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 current，voltage，phase），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。
-
-对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一台汽车建立多张表。
+TDengine 建议用数据采集点的名字（如上表中的 d1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 `current`、`voltage` 和 `phase`），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 Timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。
+对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一辆汽车建立多张表。
-## 超级表 (STable)
+## 超级表（STable）
由于一个数据采集点一张表，导致表的数量巨增，难以管理，而且应用经常需要做采集点之间的聚合操作，聚合的操作也变得复杂起来。为解决这个问题，TDengine 引入超级表（Super Table，简称为 STable）的概念。
-超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 schema，标签的数据类型可以是整数、浮点数、字符串，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。
+超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 Schema，标签的数据类型可以是整数、浮点数、字符串、JSON，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。
-在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表meters.
+在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表 `meters`.
-## 子表 (Subtable)
+## 子表（Subtable）
当为某个具体数据采集点创建表时，用户可以使用超级表的定义做模板，同时指定该具体采集点（表）的具体标签值来创建该表。**通过超级表创建的表称之为子表**。正常的表与子表的差异在于：
-1. 子表就是表，因此所有正常表的SQL操作都可以在子表上执行。
+1. 子表就是表，因此所有正常表的 SQL 操作都可以在子表上执行。
2. 子表在正常表的基础上有扩展，它是带有静态标签的，而且这些标签可以事后增加、删除、修改，而正常的表没有。
3. 子表一定属于一张超级表，但普通表不属于任何超级表
4. 普通表无法转为子表，子表也无法转为普通表。
超级表与与基于超级表建立的子表之间的关系表现在：
-1. 一张超级表包含有多张子表，这些子表具有相同的采集量 schema，但带有不同的标签值。
+1. 一张超级表包含有多张子表，这些子表具有相同的采集量 Schema，但带有不同的标签值。
2. 不能通过子表调整数据或标签的模式，对于超级表的数据模式修改立即对所有的子表生效。
3. 超级表只定义一个模板，自身不存储任何数据或标签信息。因此，不能向一个超级表写入数据，只能将数据写入子表中。
查询既可以在表上进行，也可以在超级表上进行。针对超级表的查询，TDengine 将把所有子表中的数据视为一个整体数据集进行处理，会先把满足标签过滤条件的表从超级表中找出来，然后再扫描这些表的时序数据，进行聚合操作，这样需要扫描的数据集会大幅减少，从而显著提高查询的性能。本质上，TDengine 通过对超级表查询的支持，实现了多个同类数据采集点的高效聚合。
-TDengine系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表meters创建子表d1001, d1002, d1003, d1004等。
+TDengine 系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表 meters 创建子表 d1001、d1002、d1003、d1004 等。
+
+为了更好地理解采集量、标签、超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。
+
+
+
+
-为了更好地理解超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。 
+图 1. 智能电表数据模型示意图
-## 库 (database)
+## 库（Database）
库是指一组表的集合。TDengine 容许一个运行实例有多个库，而且每个库可以配置不同的存储策略。不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的超级表创建在不同的库里。
一个库里，可以有一到多个超级表，但一个超级表只属于一个库。一个超级表所拥有的子表全部存在一个库里。
-## FQDN & End Point
+## FQDN & Endpoint
-FQDN (fully qualified domain name, 完全限定域名)是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS(Domain Name System)，负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。
+FQDN（Fully Qualified Domain Name，完全限定域名）是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS（Domain Name System），负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。
-TDengine 集群的每个节点是由 End Point 来唯一标识的，End Point 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。
+TDengine 集群的每个节点是由 Endpoint 来唯一标识的，Endpoint 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。
TDengine 不建议采用直接的 IP 地址访问集群，不利于管理。不了解 FQDN 概念，请看博文[《一篇文章说清楚 TDengine 的 FQDN》](https://www.taosdata.com/blog/2020/09/11/1824.html)。
diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md
index e2be4195176a3f1ac7712a036d04b60b2fb77718..0f004581b5be470f3c99f48e1aaecb3578c442ab 100644
--- a/docs/zh/05-get-started/01-docker.md
+++ b/docs/zh/05-get-started/01-docker.md
@@ -4,11 +4,11 @@ title: 通过 Docker 快速体验 TDengine
description: 使用 Docker 快速体验 TDengine 的高效写入和查询
---
-本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装.
+本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine)下载源码构建和安装。
## 启动 TDengine
-如果已经安装了 docker， 只需执行下面的命令。
+如果已经安装了 Docker，只需执行下面的命令：
```shell
docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine
@@ -16,84 +16,84 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043
注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。
-确定该容器已经启动并且在正常运行
+确定该容器已经启动并且在正常运行。
```shell
docker ps
```
-进入该容器并执行 bash
+进入该容器并执行 `bash`
```shell
docker exec -it bash
```
-然后就可以执行相关的 Linux 命令操作和访问 TDengine
+然后就可以执行相关的 Linux 命令操作和访问 TDengine。
-注: Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。
+注：Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。
## 运行 TDengine CLI
-进入容器，执行 taos
+进入容器，执行 `taos`：
```
$ taos
-taos>
-
+taos>
```
-## 写入数据
+## 使用 taosBenchmark 体验写入速度
-可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入。
+可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。
-进入容器，启动 taosBenchmark：
+启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）：
- ```bash
- $ taosBenchmark
-
- ```
+```bash
+$ taosBenchmark
+```
- 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"等城市名称。
+该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `California.Campbell`、`California.Cupertino`、`California.LosAngeles`、`California.MountainView`、`California.PaloAlto`、`California.SanDiego`、`California.SanFrancisco`、`California.SanJose`、`California.SantaClara` 或者 `California.Sunnyvale`。
- 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能。
+这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。
- taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
+taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
-## 体验查询
+## 使用 TDengine CLI 体验查询速度
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。
+使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。
-查询超级表下记录总条数：
+查询超级表 `meters` 下的记录总条数：
```sql
-taos> select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
查询 1 亿条记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-查询 location="San Francisco" 的记录总条数：
+查询 location = "California.SanFrancisco" 的记录总条数：
```sql
-taos> select count(*) from test.meters where location="San Francisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco";
```
-查询 groupId=10 的所有记录的平均值、最大值、最小值等：
+查询 groupId = 10 的所有记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
+对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
+在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。
+
## 其它
-更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)
+更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)。
diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md
index 3e0fb056a5913d3a82a473bf879a79e398176075..2c857d37f3a8591316949f9a7440a115740a8e2d 100644
--- a/docs/zh/05-get-started/03-package.md
+++ b/docs/zh/05-get-started/03-package.md
@@ -10,23 +10,24 @@ import PkgListV3 from "/components/PkgListV3";
您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装.
-TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。
+TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。
-为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 lite 版本的安装包。
+为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。
-在 Linux 系统上，TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，rpm 和 deb 包不含 taosdump 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。
+在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。
## 安装
-1. 从列表中下载获得 deb 安装包；
-
+1. 从列表中下载获得 Deb 安装包；
+
2. 进入到安装包所在目录，执行如下的安装命令：
+> 请将 `` 替换为下载的安装包版本
+
```bash
-# 替换为下载的安装包版本
sudo dpkg -i TDengine-server--Linux-x64.deb
```
@@ -34,12 +35,13 @@ sudo dpkg -i TDengine-server--Linux-x64.deb
-1. 从列表中下载获得 rpm 安装包；
-
+1. 从列表中下载获得 RPM 安装包；
+
2. 进入到安装包所在目录，执行如下的安装命令：
+> 请将 `` 替换为下载的安装包版本
+
```bash
-# 替换为下载的安装包版本
sudo rpm -ivh TDengine-server--Linux-x64.rpm
```
@@ -48,44 +50,47 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm
1. 从列表中下载获得 tar.gz 安装包；
-
-2. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本：
+
+2. 进入到安装包所在目录，使用 `tar` 解压安装包；
+3. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本。
+
+> 请将 `` 替换为下载的安装包版本
```bash
-# 替换为下载的安装包版本
tar -zxvf TDengine-server--Linux-x64.tar.gz
```
-解压后进入相应路径，执行
+解压文件后，进入相应子目录，执行其中的 `install.sh` 安装脚本：
```bash
sudo ./install.sh
```
:::info
-install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。
+install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以运行 `./install.sh -e no`。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。
:::
-可以使用 apt-get 工具从官方仓库安装。
-**安装包仓库**
+可以使用 `apt-get` 工具从官方仓库安装。
+
+**配置包仓库**
```bash
wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add -
echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list
```
-如果安装 Beta 版需要安装包仓库
+如果安装 Beta 版需要安装包仓库：
```bash
wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add -
echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list
```
-**使用 apt-get 命令安装**
+**使用 `apt-get` 命令安装**
```bash
sudo apt-get update
@@ -94,26 +99,26 @@ sudo apt-get install tdengine
```
:::tip
-apt-get 方式只适用于 Debian 或 Ubuntu 系统
+apt-get 方式只适用于 Debian 或 Ubuntu 系统。
::::
-
+
-注意：目前 TDengine 在 Windows 平台上只支持 Windows server 2016/2019 和 Windows 10/11 系统版本。
+注意：目前 TDengine 在 Windows 平台上只支持 Windows Server 2016/2019 和 Windows 10/11。
1. 从列表中下载获得 exe 安装程序；
-
+
2. 运行可执行程序来安装 TDengine。
:::info
-下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)
+下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)。
:::
:::note
-当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。
+当安装第一个节点时，出现 `Enter FQDN:` 提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。
:::
@@ -148,7 +153,7 @@ Active: inactive (dead)
如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。
-systemctl 命令汇总：
+如下 `systemctl` 命令可以帮助你管理 TDengine 服务：
- 启动服务进程：`systemctl start taosd`
@@ -160,7 +165,7 @@ systemctl 命令汇总：
:::info
-- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。
+- `systemctl` 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 `sudo`。
- `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。
- 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。
@@ -170,87 +175,93 @@ systemctl 命令汇总：
-安装后，在 C:\TDengine 目录下，运行 taosd.exe 来启动 TDengine 服务进程。
+安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。
-## TDengine 命令行 (CLI)
+## TDengine 命令行（CLI）
-为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。
+为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。
```bash
taos
```
-如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下：
+如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。TDengine CLI 的提示符号如下：
```cmd
taos>
```
-在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例：
+在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库（Database）插入查询操作。在终端中运行的 SQL 语句需要以分号（;）结束来运行。示例：
```sql
-create database demo;
-use demo;
-create table t (ts timestamp, speed int);
-insert into t values ('2019-07-15 00:00:00', 10);
-insert into t values ('2019-07-15 01:00:00', 20);
-select * from t;
+CREATE DATABASE demo;
+USE demo;
+CREATE TABLE t (ts TIMESTAMP, speed INT);
+INSERT INTO t VALUES ('2019-07-15 00:00:00', 10);
+INSERT INTO t VALUES ('2019-07-15 01:00:00', 20);
+SELECT * FROM t;
+
ts | speed |
========================================
2019-07-15 00:00:00.000 | 10 |
2019-07-15 01:00:00.000 | 20 |
+
Query OK, 2 row(s) in set (0.003128s)
```
-除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../../reference/taos-shell/)
+除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [TDengine 命令行](../../reference/taos-shell/)。
## 使用 taosBenchmark 体验写入速度
-启动 TDengine 的服务，在 Linux 或 windows 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）：
+可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。
+
+启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）：
```bash
-taosBenchmark
+$ taosBenchmark
```
-该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。
+该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `California.Campbell`、`California.Cupertino`、`California.LosAngeles`、`California.MountainView`、`California.PaloAlto`、`California.SanDiego`、`California.SanFrancisco`、`California.SanJose`、`California.SantaClara` 或者 `California.Sunnyvale`。
这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。
-taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。
+taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。
## 使用 TDengine CLI 体验查询速度
-使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。
+使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。
-查询超级表下记录总条数：
+查询超级表 `meters` 下的记录总条数：
```sql
-taos> select count(*) from test.meters;
+SELECT COUNT(*) FROM test.meters;
```
查询 1 亿条记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters;
```
-查询 location="California.SanFrancisco" 的记录总条数：
+查询 location = "California.SanFrancisco" 的记录总条数：
```sql
-taos> select count(*) from test.meters where location="California.SanFrancisco";
+SELECT COUNT(*) FROM test.meters WHERE location = "Calaifornia.SanFrancisco";
```
-查询 groupId=10 的所有记录的平均值、最大值、最小值等：
+查询 groupId = 10 的所有记录的平均值、最大值、最小值等：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10;
+SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10;
```
-对表 d10 按 10s 进行平均值、最大值和最小值聚合统计：
+对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计：
```sql
-taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s);
+SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s);
```
+
+在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。
diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx
index 2f5c13d9b0bc0e3940fb99b45c693e2ae80c8f47..d9a35ab4eb545872a0f9a8e10e6e40dcf6ee4ec7 100644
--- a/docs/zh/07-develop/07-tmq.mdx
+++ b/docs/zh/07-develop/07-tmq.mdx
@@ -218,7 +218,7 @@ void Close()
```sql
DROP DATABASE IF EXISTS tmqdb;
CREATE DATABASE tmqdb;
-CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16) TAGS(t1 INT, t3 VARCHAR(16));
+CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16));
CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0");
CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1");
INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00');
diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md
index ef1cd4797a217b601b1b8e3eaa0e74b8c2907c88..3239eae49b05180c4a0dba5850de9f1c5e08a4f3 100644
--- a/docs/zh/07-develop/09-udf.md
+++ b/docs/zh/07-develop/09-udf.md
@@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。
参数的具体含义是：
- inputDataBlock: 输入的数据块
- - resultColumn: 输出列。输出列
+ - resultColumn: 输出列
### 聚合接口函数
diff --git a/docs/zh/08-connector/02-rest-api.mdx b/docs/zh/08-connector/02-rest-api.mdx
index e254244657b457e10bc2daab020b230c9a8bb2cc..12a9cf5347c9f2bb5490a1a19999da79b9d5c58f 100644
--- a/docs/zh/08-connector/02-rest-api.mdx
+++ b/docs/zh/08-connector/02-rest-api.mdx
@@ -4,7 +4,7 @@ sidebar_label: REST API
description: 详细介绍 TDengine 提供的 RESTful API.
---
-为支持各种不同类型平台的开发，TDengine 提供符合 REST 设计标准的 API，即 REST API。为最大程度降低学习成本，不同于其他数据库 REST API 的设计方法，TDengine 直接通过 HTTP POST 请求 BODY 中包含的 SQL 语句来操作数据库，仅需要一个 URL。REST 连接器的使用参见 [视频教程](https://www.taosdata.com/blog/2020/11/11/1965.html)。
+为支持各种不同类型平台的开发，TDengine 提供符合 RESTful 设计标准的 API，即 REST API。为最大程度降低学习成本，不同于其他数据库 REST API 的设计方法，TDengine 直接通过 HTTP POST 请求 BODY 中包含的 SQL 语句来操作数据库，仅需要一个 URL。REST API 的使用参见 [视频教程](https://www.taosdata.com/blog/2020/11/11/1965.html)。
:::note
与原生连接器的一个区别是，RESTful 接口是无状态的，因此 `USE db_name` 指令没有效果，所有对表名、超级表名的引用都需要指定数据库名前缀。支持在 RESTful URL 中指定 db_name，这时如果 SQL 语句中没有指定数据库名前缀的话，会使用 URL 中指定的这个 db_name。
@@ -18,7 +18,7 @@ RESTful 接口不依赖于任何 TDengine 的库，因此客户端不需要安
在已经安装 TDengine 服务器端的情况下，可以按照如下方式进行验证。
-下面以 Ubuntu 环境中使用 curl 工具（确认已经安装）来验证 RESTful 接口的正常，验证前请确认 taosAdapter 服务已开启，在 Linux 系统上此服务默认由 systemd 管理，使用命令 `systemctl start taosadapter` 启动。
+下面以 Ubuntu 环境中使用 `curl` 工具（请确认已经安装）来验证 RESTful 接口是否工作正常，验证前请确认 taosAdapter 服务已开启，在 Linux 系统上此服务默认由 systemd 管理，使用命令 `systemctl start taosadapter` 启动。
下面示例是列出所有的数据库，请把 h1.taosdata.com 和 6041（缺省值）替换为实际运行的 TDengine 服务 FQDN 和端口号：
diff --git a/docs/zh/08-connector/05-schemaless-api.mdx b/docs/zh/08-connector/05-schemaless-api.mdx
new file mode 100644
index 0000000000000000000000000000000000000000..f6d7e09212c71ed7b142eeabf2a2d69c5f79d439
--- /dev/null
+++ b/docs/zh/08-connector/05-schemaless-api.mdx
@@ -0,0 +1,38 @@
+---
+title: Schemaless API
+sidebar_label: Schemaless API
+description: 详细介绍 TDengine 提供的 Schemaless API.
+---
+
+TDengine 提供了兼容 InfluxDB (v1) 和 OpenTSDB 行协议的 Schemaless API。支持 InfluxDB（v1) 或 OpenTSDB 行协议写入数据的第三方软件无需修改代码，只要修改配置的 EndPoint URL 就可以直接把数据写入 TDengine 数据库。
+
+### 兼容 InfluxDB 行协议写入的方法
+
+您可以配置任何支持使用 InfluxDB（v1） 行协议的应用访问地址 `http://:6041/` 来写入 InfluxDB 兼容格式的数据到 TDengine。EndPoint 如下：
+```text
+/influxdb/v1/write??...
+```
+
+支持 InfluxDB 查询参数如下：
+
+- `db` 指定 TDengine 使用的数据库名
+- `precision` TDengine 使用的时间精度
+- `u` TDengine 用户名
+- `p` TDengine 密码
+
+注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。
+
+参考链接：[InfluxDB v1 写接口](https://docs.influxdata.com/influxdb/v2.0/reference/api/influxdb-1x/write/)
+
+### 兼容 OpenTSDB 行协议写入的方法
+
+您可以配置任何支持 OpenTSDB 行协议的应用访问地址 `http://:6041/` 来写入 OpenTSDB 兼容格式的数据到 TDengine。EndPoint 如下：
+
+```text
+/opentsdb/v1/put/json/
+/opentsdb/v1/put/telnet/
+```
+
+参考链接：
+ - [OpenTSDB JSON](http://opentsdb.net/docs/build/html/api_http/put.html)
+ - [OpenTSDB Telnet](http://opentsdb.net/docs/build/html/api_telnet/put.html)
diff --git a/docs/zh/08-connector/03-cpp.mdx b/docs/zh/08-connector/10-cpp.mdx
similarity index 100%
rename from docs/zh/08-connector/03-cpp.mdx
rename to docs/zh/08-connector/10-cpp.mdx
diff --git a/docs/zh/08-connector/04-java.mdx b/docs/zh/08-connector/14-java.mdx
similarity index 100%
rename from docs/zh/08-connector/04-java.mdx
rename to docs/zh/08-connector/14-java.mdx
diff --git a/docs/zh/08-connector/05-go.mdx b/docs/zh/08-connector/20-go.mdx
similarity index 100%
rename from docs/zh/08-connector/05-go.mdx
rename to docs/zh/08-connector/20-go.mdx
diff --git a/docs/zh/08-connector/06-rust.mdx b/docs/zh/08-connector/26-rust.mdx
similarity index 98%
rename from docs/zh/08-connector/06-rust.mdx
rename to docs/zh/08-connector/26-rust.mdx
index 26f53c82d630fda168dd98b4c8ec993afc5e3a1d..b838e5c5a2f5c33f09dce6cf1a8d5f61ddc5f264 100644
--- a/docs/zh/08-connector/06-rust.mdx
+++ b/docs/zh/08-connector/26-rust.mdx
@@ -155,15 +155,15 @@ async fn demo(taos: &Taos, db: &str) -> Result<(), Error> {
let inserted = taos.exec_many([
// create super table
"CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \
- TAGS (`groupid` INT, `location` BINARY(16))",
+ TAGS (`groupid` INT, `location` BINARY(24))",
// create child table
- "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')",
+ "CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')",
// insert into child table
"INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)",
// insert with NULL values
"INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)",
// insert and automatically create table with tags if not exists
- "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)",
+ "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119, 0.33)",
// insert many records in a single sql
"INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)",
]).await?;
diff --git a/docs/zh/08-connector/07-python.mdx b/docs/zh/08-connector/30-python.mdx
similarity index 100%
rename from docs/zh/08-connector/07-python.mdx
rename to docs/zh/08-connector/30-python.mdx
diff --git a/docs/zh/08-connector/08-node.mdx b/docs/zh/08-connector/35-node.mdx
similarity index 100%
rename from docs/zh/08-connector/08-node.mdx
rename to docs/zh/08-connector/35-node.mdx
diff --git a/docs/zh/08-connector/09-csharp.mdx b/docs/zh/08-connector/40-csharp.mdx
similarity index 100%
rename from docs/zh/08-connector/09-csharp.mdx
rename to docs/zh/08-connector/40-csharp.mdx
diff --git a/docs/zh/08-connector/10-php.mdx b/docs/zh/08-connector/45-php.mdx
similarity index 100%
rename from docs/zh/08-connector/10-php.mdx
rename to docs/zh/08-connector/45-php.mdx
diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md
index 17de8e926cd9a3633dc8746b0fb49c38ff8ca61f..f54470f7420ada71c2cd283eff52c5fc6e9ada1a 100644
--- a/docs/zh/08-connector/index.md
+++ b/docs/zh/08-connector/index.md
@@ -41,14 +41,14 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
### 使用原生接口（taosc）
-| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
-| -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- |
-| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
-| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 |
+| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** |
+| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- |
+| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 |
:::info
由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。
@@ -56,16 +56,15 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器
### 使用 http (REST 或 WebSocket) 接口
-| **功能特性** | **Java** | **Python** | **Go** | **C#（暂不支持）** | **Node.js** | **Rust** |
-| ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- |
-| **连接管理** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 |
-| **普通查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 |
-| **连续查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 |
-| **参数绑定** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 |
-| ** TMQ ** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 |
-| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 暂不支持 |
-| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | N/A | 不支持 | 支持 |
-| **DataFrame** | 不支持 | 支持 | 不支持 | N/A | 不支持 | 不支持 |
+| **功能特性** | **Java** | **Python** | **Go** | **C# ** | **Node.js** | **Rust** |
+| ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- |
+| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 |
+| **参数绑定** | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 支持 |
+| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 支持 |
+| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 |
+| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | 支持 | 暂不支持 | 支持 |
+| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 |
:::warning
diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md
index 03b4ce30f980cd77e9845076ce9bb35c4474f948..eecb86ce415b6ad0a24a7fcd0d6d9d9ce1357a33 100644
--- a/docs/zh/10-deployment/01-deploy.md
+++ b/docs/zh/10-deployment/01-deploy.md
@@ -168,7 +168,7 @@ Query OK, 8 row(s) in set (0.001154s)
## 删除数据节点
-先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行：
+启动 CLI 程序 taos，执行：
```sql
DROP DNODE "fqdn:port";
diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md
index 2337f2a272cad45ae762349d2bf355594615cbb5..f6790e3c692b815c1031413933c47eb7ad203204 100644
--- a/docs/zh/12-taos-sql/03-table.md
+++ b/docs/zh/12-taos-sql/03-table.md
@@ -23,10 +23,7 @@ create_subtable_clause: {
}
create_definition:
- col_name column_definition
-
-column_definition:
- type_name [comment 'string_value']
+ col_name column_type
table_options:
table_option ...
diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md
index b0a7d88efea4811b193117f220871af64341ff97..c49351b0c901f60b5a02f3a08a5d1c44135631c3 100644
--- a/docs/zh/12-taos-sql/06-select.md
+++ b/docs/zh/12-taos-sql/06-select.md
@@ -104,7 +104,7 @@ SELECT location, groupid, current FROM d1001 LIMIT 2;
### 结果去重
-`DISINTCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。
+`DISTINCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。
对标签列去重：
@@ -356,7 +356,7 @@ SELECT ... FROM (SELECT ... FROM ...) ...;
- 与非嵌套的查询语句相比，外层查询所能支持的功能特性存在如下限制：
- 计算函数部分：
- 如果内层查询的结果数据未提供时间戳，那么计算过程隐式依赖时间戳的函数在外层会无法正常工作。例如：INTERP, DERIVATIVE, IRATE, LAST_ROW, FIRST, LAST, TWA, STATEDURATION, TAIL, UNIQUE。
- - 如果内层查询的结果数据不是有效的时间序列，那么计算过程依赖数据为时间序列的函数在外层会无法正常工作。例如：LEASTSQUARES, ELAPSED, INTERP, DERIVATIVE, IRATE, TWA, DIFF, STATECOUNT, STATEDURATION, CSUM, MAVG, TAIL, UNIQUE。
+ - 如果内层查询的结果数据不是按时间戳有序，那么计算过程依赖数据按时间有序的函数在外层会无法正常工作。例如：LEASTSQUARES, ELAPSED, INTERP, DERIVATIVE, IRATE, TWA, DIFF, STATECOUNT, STATEDURATION, CSUM, MAVG, TAIL, UNIQUE。
- 计算过程需要两遍扫描的函数，在外层查询中无法正常工作。例如：此类函数包括：PERCENTILE。
:::
diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md
index 9c5b7f771ecaf52da55a693ed5c789197ab57b05..86e9aaa80fdebecbd390ae231437491530d6c2b5 100644
--- a/docs/zh/12-taos-sql/10-function.md
+++ b/docs/zh/12-taos-sql/10-function.md
@@ -127,7 +127,7 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause]
SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause];
```
-**功能说明**：获得指定字段的向下取整数的结果。
+**功能说明**：获得指定字段的向下取整数的结果。
其他使用说明参见 CEIL 函数描述。
#### LOG
@@ -174,7 +174,7 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause]
SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause];
```
-**功能说明**：获得指定字段的四舍五入的结果。
+**功能说明**：获得指定字段的四舍五入的结果。
其他使用说明参见 CEIL 函数描述。
@@ -435,7 +435,7 @@ SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause];
**使用说明**：
- timezone 参数允许输入的时区格式为: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。
-- 如果输入是表示 UNIX 时间戳的整形，返回格式精度由时间戳的位数决定;
+- 如果输入是表示 UNIX 时间戳的整形，返回格式精度由时间戳的位数决定;
- 如果输入是 TIMESTAMP 类型的列，返回格式的时间戳精度与当前 DATABASE 设置的时间精度一致。
@@ -770,14 +770,14 @@ SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_nam
**详细说明**：
- bin_type 用户指定的分桶类型, 有效输入类型为"user_input“, ”linear_bin", "log_bin"。
-- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)：
- - "user_input": "[1, 3, 5, 7]"
+- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)：
+ - "user_input": "[1, 3, 5, 7]"
用户指定 bin 的具体数值。
-
+
- "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}"
"start" 表示数据起始点，"width" 表示每次 bin 偏移量, "count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点，
生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。
-
+
- "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}"
"start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点，
生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。
@@ -918,7 +918,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause];
**返回数据类型**：同应用的字段。
-**适用数据类型**：数值类型，时间戳类型。
+**适用数据类型**：数值类型。
**适用于**：表和超级表。
@@ -933,7 +933,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause];
**返回数据类型**：同应用的字段。
-**适用数据类型**：数值类型，时间戳类型。
+**适用数据类型**：数值类型。
**适用于**：表和超级表。
@@ -969,7 +969,7 @@ SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
**适用于**：表和超级表。
-**使用说明**：
+**使用说明**：
- 不能参与表达式计算；该函数可以应用在普通表和超级表上；
- 使用在超级表上的时候，需要搭配 PARTITION by tbname 使用，将结果强制规约到单个时间线。
@@ -1047,10 +1047,10 @@ SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause]
**适用于**：表和超级表。
-**使用说明**：
-
+**使用说明**：
+
- 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。
-- 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。
+- 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。
- 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。
@@ -1068,8 +1068,8 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER
**适用于**：表和超级表。
-**使用说明**:
-
+**使用说明**:
+
- DERIVATIVE 函数可以在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname）。
- 可以与选择相关联的列一起使用。 例如: select \_rowts, DERIVATIVE() from。
@@ -1087,7 +1087,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER
**适用于**：表和超级表。
-**使用说明**:
+**使用说明**:
- 输出结果行数是范围内总行数减一，第一行没有结果输出。
- 可以与选择相关联的列一起使用。 例如: select \_rowts, DIFF() from。
@@ -1124,9 +1124,9 @@ SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause]
**适用于**：表和超级表。
-**使用说明**：
-
-- 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1);
+**使用说明**：
+
+- 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1);
- 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用；
- 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。
diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md
index 268712e757304fe22848318befd16d1a93de5dac..861ef4ebb7f307f6653c72066b5cae7548c14855 100644
--- a/docs/zh/12-taos-sql/12-distinguished.md
+++ b/docs/zh/12-taos-sql/12-distinguished.md
@@ -46,7 +46,7 @@ SELECT select_list FROM tb_name
### 窗口子句的规则
-- 窗口子句位于数据切分子句之后，GROUP BY 子句之前，且不可以和 GROUP BY 子句一起使用。
+- 窗口子句位于数据切分子句之后，不可以和 GROUP BY 子句一起使用。
- 窗口子句将数据按窗口进行切分，对每个窗口进行 SELECT 列表中的表达式的计算，SELECT 列表中的表达式只能包含：
- 常量。
- _wstart伪列、_wend伪列和_wduration伪列。
@@ -71,7 +71,7 @@ FILL 语句指定某一窗口区间数据缺失的情况下的填充模式。填
1. 使用 FILL 语句的时候可能生成大量的填充输出，务必指定查询的时间区间。针对每次查询，系统可返回不超过 1 千万条具有插值的结果。
2. 在时间维度聚合中，返回的结果中时间序列严格单调递增。
-3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 PARTITION BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 PARTITION BY 语句分组，则返回结果中每个 PARTITION 内不按照时间序列严格单调递增。
+3. 如果查询对象是超级表，则聚合函数会作用于该超级表下满足值过滤条件的所有表的数据。如果查询中没有使用 PARTITION BY 语句，则返回的结果按照时间序列严格单调递增；如果查询中使用了 PARTITION BY 语句分组，则返回结果中每个 PARTITION 内按照时间序列严格单调递增。
:::
@@ -113,6 +113,12 @@ SELECT COUNT(*) FROM temp_tb_1 INTERVAL(1m) SLIDING(2m);
SELECT COUNT(*), FIRST(ts), status FROM temp_tb_1 STATE_WINDOW(status);
```
+仅关心 status 为 2 时的状态窗口的信息。例如：
+
+```
+SELECT * FROM (SELECT COUNT(*) AS cnt, FIRST(ts) AS fst, status FROM temp_tb_1 STATE_WINDOW(status)) t WHERE status = 2;
+```
+
### 会话窗口
会话窗口根据记录的时间戳主键的值来确定是否属于同一个会话。如下图所示，如果设置时间戳的连续的间隔小于等于 12 秒，则以下 6 条记录构成 2 个会话窗口，分别是：[2019-04-28 14:22:10，2019-04-28 14:22:30]和[2019-04-28 14:23:10，2019-04-28 14:23:30]。因为 2019-04-28 14:22:30 与 2019-04-28 14:23:10 之间的时间间隔是 40 秒，超过了连续时间间隔（12 秒）。
diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md
index 09fea45b7a8221ce6ed955a78cec502e235506a0..7530e803db44f49048f02e2ad6980a288c14fa12 100644
--- a/docs/zh/12-taos-sql/20-keywords.md
+++ b/docs/zh/12-taos-sql/20-keywords.md
@@ -6,7 +6,8 @@ description: TDengine 保留关键字的详细列表
## 保留关键字
-目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写如果需要用作库名、表名、STable 名、数据列名及标签列名等，需要使用符合``将关键字括起来使用，例如`ADD`。
+目前 TDengine 有 200 多个内部保留关键字，这些关键字如果需要用作库名、表名、超级表名、子表名、数据列名及标签列名等，无论大小写，需要使用符号 `` ` `` 将关键字括起来使用，例如 \`ADD\`。
+
关键字列表如下：
### A
@@ -16,15 +17,20 @@ description: TDengine 保留关键字的详细列表
- ACCOUNTS
- ADD
- AFTER
+- AGGREGATE
- ALL
- ALTER
+- ANALYZE
- AND
+- APPS
- AS
- ASC
+- AT_ONCE
- ATTACH
### B
+- BALANCE
- BEFORE
- BEGIN
- BETWEEN
@@ -34,19 +40,27 @@ description: TDengine 保留关键字的详细列表
- BITNOT
- BITOR
- BLOCKS
+- BNODE
+- BNODES
- BOOL
+- BUFFER
+- BUFSIZE
- BY
### C
- CACHE
-- CACHELAST
+- CACHEMODEL
+- CACHESIZE
- CASCADE
+- CAST
- CHANGE
+- CLIENT_VERSION
- CLUSTER
- COLON
- COLUMN
- COMMA
+- COMMENT
- COMP
- COMPACT
- CONCAT
@@ -54,15 +68,18 @@ description: TDengine 保留关键字的详细列表
- CONNECTION
- CONNECTIONS
- CONNS
+- CONSUMER
+- CONSUMERS
+- CONTAINS
- COPY
+- COUNT
- CREATE
-- CTIME
+- CURRENT_USER
### D
- DATABASE
- DATABASES
-- DAYS
- DBS
- DEFERRED
- DELETE
@@ -71,18 +88,23 @@ description: TDengine 保留关键字的详细列表
- DESCRIBE
- DETACH
- DISTINCT
+- DISTRIBUTED
- DIVIDE
- DNODE
- DNODES
- DOT
- DOUBLE
- DROP
+- DURATION
### E
+- EACH
+- ENABLE
- END
-- EQ
+- EVERY
- EXISTS
+- EXPIRED
- EXPLAIN
### F
@@ -90,18 +112,20 @@ description: TDengine 保留关键字的详细列表
- FAIL
- FILE
- FILL
+- FIRST
- FLOAT
+- FLUSH
- FOR
- FROM
-- FSYNC
+- FUNCTION
+- FUNCTIONS
### G
-- GE
- GLOB
+- GRANT
- GRANTS
- GROUP
-- GT
### H
@@ -112,15 +136,18 @@ description: TDengine 保留关键字的详细列表
- ID
- IF
- IGNORE
-- IMMEDIA
+- IMMEDIATE
- IMPORT
- IN
-- INITIAL
+- INDEX
+- INDEXES
+- INITIALLY
+- INNER
- INSERT
- INSTEAD
- INT
- INTEGER
-- INTERVA
+- INTERVAL
- INTO
- IS
- ISNULL
@@ -128,6 +155,7 @@ description: TDengine 保留关键字的详细列表
### J
- JOIN
+- JSON
### K
@@ -137,46 +165,57 @@ description: TDengine 保留关键字的详细列表
### L
-- LE
+- LAST
+- LAST_ROW
+- LICENCES
- LIKE
- LIMIT
- LINEAR
- LOCAL
-- LP
-- LSHIFT
-- LT
### M
- MATCH
+- MAX_DELAY
- MAXROWS
+- MERGE
+- META
- MINROWS
- MINUS
+- MNODE
- MNODES
- MODIFY
- MODULES
### N
-- NE
+- NCHAR
+- NEXT
+- NMATCH
- NONE
- NOT
- NOTNULL
- NOW
- NULL
+- NULLS
### O
- OF
- OFFSET
+- ON
- OR
- ORDER
+- OUTPUTTYPE
### P
-- PARTITION
+- PAGES
+- PAGESIZE
+- PARTITIONS
- PASS
- PLUS
+- PORT
- PPS
- PRECISION
- PREV
@@ -184,47 +223,63 @@ description: TDengine 保留关键字的详细列表
### Q
+- QNODE
+- QNODES
- QTIME
-- QUERIE
+- QUERIES
- QUERY
-- QUORUM
### R
- RAISE
-- REM
+- RANGE
+- RATIO
+- READ
+- REDISTRIBUTE
+- RENAME
- REPLACE
- REPLICA
- RESET
-- RESTRIC
+- RESTRICT
+- RETENTIONS
+- REVOKE
+- ROLLUP
- ROW
-- RP
-- RSHIFT
### S
+- SCHEMALESS
- SCORES
- SELECT
- SEMI
+- SERVER_STATUS
+- SERVER_VERSION
- SESSION
- SET
- SHOW
-- SLASH
+- SINGLE_STABLE
- SLIDING
- SLIMIT
-- SMALLIN
+- SMA
+- SMALLINT
+- SNODE
+- SNODES
- SOFFSET
-- STable
-- STableS
+- SPLIT
+- STABLE
+- STABLES
- STAR
- STATE
-- STATEMEN
-- STATE_WI
+- STATE_WINDOW
+- STATEMENT
- STORAGE
- STREAM
- STREAMS
+- STRICT
- STRING
+- SUBSCRIPTIONS
- SYNCDB
+- SYSINFO
### T
@@ -235,20 +290,24 @@ description: TDengine 保留关键字的详细列表
- TBNAME
- TIMES
- TIMESTAMP
+- TIMEZONE
- TINYINT
+- TO
+- TODAY
- TOPIC
- TOPICS
+- TRANSACTION
+- TRANSACTIONS
- TRIGGER
+- TRIM
- TSERIES
- TTL
### U
-- UMINUS
- UNION
- UNSIGNED
- UPDATE
-- UPLUS
- USE
- USER
- USERS
@@ -256,9 +315,13 @@ description: TDengine 保留关键字的详细列表
### V
+- VALUE
- VALUES
+- VARCHAR
- VARIABLE
- VARIABLES
+- VERBOSE
+- VGROUP
- VGROUPS
- VIEW
- VNODES
@@ -266,14 +329,25 @@ description: TDengine 保留关键字的详细列表
### W
- WAL
+- WAL_FSYNC_PERIOD
+- WAL_LEVEL
+- WAL_RETENTION_PERIOD
+- WAL_RETENTION_SIZE
+- WAL_ROLL_PERIOD
+- WAL_SEGMENT_SIZE
+- WATERMARK
- WHERE
+- WINDOW_CLOSE
+- WITH
+- WRITE
### \_
- \_C0
-- \_QSTART
-- \_QSTOP
- \_QDURATION
-- \_WSTART
-- \_WSTOP
+- \_QEND
+- \_QSTART
+- \_ROWTS
- \_WDURATION
+- \_WEND
+- \_WSTART
diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md
index 71bf5f4223ae97cf2c1153aaea3b8f946e213522..82efcab06fbe8107583e49485770842c77ff4850 100644
--- a/docs/zh/14-reference/04-taosadapter.md
+++ b/docs/zh/14-reference/04-taosadapter.md
@@ -196,7 +196,7 @@ AllowWebSockets
- `u` TDengine 用户名
- `p` TDengine 密码
-注意： 目前不支持 InfluxDB 的 token 验证方式只支持 Basic 验证和查询参数验证。
+注意： 目前不支持 InfluxDB 的 token 验证方式，仅支持 Basic 验证和查询参数验证。
### OpenTSDB
diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md
index ae0a496f03e8e545525fce49ae2394a10696c09c..7780dc2fe9a5742ce9d6855599ca080804a493b5 100644
--- a/docs/zh/14-reference/14-taosKeeper.md
+++ b/docs/zh/14-reference/14-taosKeeper.md
@@ -79,7 +79,7 @@ password = "taosdata"
# 需要被监控的 taosAdapter
[taosAdapter]
-address = ["127.0.0.1:6041","192.168.1.95:6041"]
+address = ["127.0.0.1:6041"]
[metrics]
# 监控指标前缀
@@ -92,7 +92,7 @@ cluster = "production"
database = "log"
# 指定需要监控的普通表
-tables = ["normal_table"]
+tables = []
```
### 获取监控指标
@@ -141,4 +141,4 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1
# HELP taos_cluster_info_first_ep
# TYPE taos_cluster_info_first_ep gauge
taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1
-```
\ No newline at end of file
+```
diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md
index 704524fd210152af34e15d248d3d4dbe050e4fef..b128b9d438371f22763b0ccea605e19d2dc9aa7b 100644
--- a/docs/zh/21-tdinternal/01-arch.md
+++ b/docs/zh/21-tdinternal/01-arch.md
@@ -26,7 +26,7 @@ TDengine 分布式架构的逻辑结构图如下：
**管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、超级表等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M1，M2，M3）。mnode 支持多副本，采用 RAFT 一致性协议，保证系统的高可用与高可靠，任何数据更新操作只能在 Leader 上进行。mnode 集群的第一个节点在集群部署时自动完成，其他节点的创建与删除由用户通过 SQL 命令完成。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。
-**弹性计算节点（qnode）：** 一个虚拟的逻辑单元，运行查询计算任务，也包括基于系统表来实现的 show 命令（图中 Q）。集群中可配置多个 qnode，在整个集群内部共享使用（图中 Q1，Q2，Q3）。qnode 不与具体的 DB 绑定，即一个 qnode 可以同时执行多个 DB 的查询任务。每个 dnode 上至多有一个 qnode，由所属的数据节点的 EP 来唯一标识。客户端通过与 mnode 交互，获取可用的 qnode 列表，当没有可用的 qnode 时，计算任务在 vnode 中执行。
+**计算节点（qnode）：** 一个虚拟的逻辑单元，运行查询计算任务，也包括基于系统表来实现的 show 命令（图中 Q）。集群中可配置多个 qnode，在整个集群内部共享使用（图中 Q1，Q2，Q3）。qnode 不与具体的 DB 绑定，即一个 qnode 可以同时执行多个 DB 的查询任务。每个 dnode 上至多有一个 qnode，由所属的数据节点的 EP 来唯一标识。客户端通过与 mnode 交互，获取可用的 qnode 列表，当没有可用的 qnode 时，计算任务在 vnode 中执行。当一个查询执行时，依赖执行计划，调度器会安排一个或多个 qnode 来一起执行。qnode 能从 vnode 获取数据，也可以将自己的计算结果发给其他 qnode 做进一步的处理。通过引入独立的计算节点，TDengine 实现了存储和计算分离。
**流计算节点（snode）：** 一个虚拟的逻辑单元，只运行流计算任务（图中 S）。集群中可配置多个 snode，在整个集群内部共享使用（图中 S1，S2，S3）。snode 不与具体的 stream 绑定，即一个 snode 可以同时执行多个 stream 的计算任务。每个 dnode 上至多有一个 snode，由所属的数据节点的 EP 来唯一标识。由 mnode 调度可用的 snode 完成流计算任务，当没有可用的 snode 时，流计算任务在 vnode 中执行。
diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md
index e3e146313115fee12e539a161792234c2df671a5..59a241b6c8458b42313b55f8e814ea574c6fd32c 100644
--- a/docs/zh/28-releases/01-tdengine.md
+++ b/docs/zh/28-releases/01-tdengine.md
@@ -6,11 +6,12 @@ description: TDengine 发布历史、Release Notes 及下载链接
import Release from "/components/ReleaseV3";
-## 3.0.0.1
-
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
- SArray* aSstBlk; // SArray
- SBlockIdx* pBlockIdx;
- SSttBlk* pSstBlk;
-
- int32_t iBlockIdx;
- int32_t iBlockL;
- SMapData mBlock; // SMapData
- int32_t iBlock;
- SBlockData oBlockData;
- SBlockData nBlockData;
+ SFDataIter* pIter;
+ SRBTree rbt;
+ SFDataIter aFDataIter[TSDB_MAX_STT_TRIGGER + 1];
+ SBlockData bData;
+ SSkmInfo skmTable;
// for del file
int8_t delDone;
SDelFReader* pDelFReader;
SArray* aDelIdx; // SArray
int32_t iDelIdx;
SArray* aDelData; // SArray
+ uint8_t* aBuf[5];
};
-static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
+extern int32_t tRowInfoCmprFn(const void* p1, const void* p2);
+extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData);
+extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo);
+
+static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) {
int32_t code = 0;
- STsdb* pTsdb = pReader->pTsdb;
- while (true) {
- if (pReader->pDataFReader == NULL) {
- // next
- SDFileSet dFileSet = {.fid = pReader->fid};
- SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
- if (pSet == NULL) goto _exit;
- pReader->fid = pSet->fid;
-
- // load
- code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet);
- if (code) goto _err;
+ SDFileSet dFileSet = {.fid = pReader->fid};
+ SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT);
+ if (pSet == NULL) return code;
- code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx);
- if (code) goto _err;
+ pReader->fid = pSet->fid;
+ code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet);
+ if (code) goto _err;
+
+ pReader->pIter = NULL;
+ tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn);
+
+ // .data file
+ SFDataIter* pIter = &pReader->aFDataIter[0];
+ pIter->type = SNAP_DATA_FILE_ITER;
+
+ code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx);
+ if (code) goto _err;
- code = tsdbReadSttBlk(pReader->pDataFReader, 0, pReader->aSstBlk);
+ for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) {
+ pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
+
+ code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
+ if (code) goto _err;
+
+ for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk);
+
+ if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
+
+ code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
if (code) goto _err;
- // init
- pReader->iBlockIdx = 0;
- if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
- pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+ ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid);
+ ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid);
- code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
- if (code) goto _err;
+ for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
- pReader->iBlock = 0;
- } else {
- pReader->pBlockIdx = NULL;
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.suid = pIter->pBlockIdx->suid;
+ pIter->rInfo.uid = pIter->pBlockIdx->uid;
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _add_iter_and_break;
+ }
}
+ }
- pReader->iBlockL = 0;
- while (true) {
- if (pReader->iBlockL >= taosArrayGetSize(pReader->aSstBlk)) {
- pReader->pSstBlk = NULL;
- break;
- }
+ continue;
- pReader->pSstBlk = (SSttBlk*)taosArrayGet(pReader->aSstBlk, pReader->iBlockL);
- if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
- // TODO
- break;
- }
+ _add_iter_and_break:
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter);
+ break;
+ }
- pReader->iBlockL++;
- }
+ // .stt file
+ pIter = &pReader->aFDataIter[1];
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ pIter->type = SNAP_STT_FILE_ITER;
+ pIter->iStt = iStt;
- tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path,
- pReader->fid);
+ code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk);
+ if (code) goto _err;
+
+ for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
+ SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
+
+ if (pSttBlk->minVer > pReader->ever) continue;
+ if (pSttBlk->maxVer < pReader->sver) continue;
+
+ code = tsdbReadSttBlock(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.suid = pIter->bData.suid;
+ pIter->rInfo.uid = pIter->bData.uid;
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _add_iter;
+ }
+ }
}
- while (true) {
- if (pReader->pBlockIdx && pReader->pSstBlk) {
- TABLEID id = {.suid = pReader->pSstBlk->suid, .uid = pReader->pSstBlk->minUid};
+ continue;
- ASSERT(0);
+ _add_iter:
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter);
+ pIter++;
+ }
- // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) {
- // // TODO
- // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) {
- // // TODO
- // } else {
- // // TODO
- // }
- } else if (pReader->pBlockIdx) {
- while (pReader->iBlock < pReader->mBlock.nItem) {
- SDataBlk block;
- tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetDataBlk);
-
- if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) {
- // load data (todo)
- }
+ tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode),
+ pReader->pTsdb->path, pReader->fid);
+ return code;
- // next
- pReader->iBlock++;
- if (*ppData) break;
+_err:
+ tsdbError("vgId:%d vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode),
+ tstrerror(code));
+ return code;
+}
+
+static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; }
+
+static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) {
+ int32_t code = 0;
+
+ if (pReader->pIter) {
+ SFDataIter* pIter = pReader->pIter;
+
+ while (true) {
+ _find_row:
+ for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) {
+ int64_t rowVer = pIter->bData.aVersion[pIter->iRow];
+
+ if (rowVer >= pReader->sver && rowVer <= pReader->ever) {
+ pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow];
+ pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow);
+ goto _out;
}
+ }
- if (pReader->iBlock >= pReader->mBlock.nItem) {
- pReader->iBlockIdx++;
- if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) {
- pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx);
+ if (pIter->type == SNAP_DATA_FILE_ITER) {
+ while (true) {
+ for (pIter->iBlock++; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk);
- code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock);
+ if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue;
+
+ code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData);
if (code) goto _err;
- pReader->iBlock = 0;
- } else {
- pReader->pBlockIdx = NULL;
+ pIter->iRow = -1;
+ goto _find_row;
}
+
+ pIter->iBlockIdx++;
+ if (pIter->iBlockIdx >= taosArrayGetSize(pIter->aBlockIdx)) break;
+
+ pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx);
+ code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock);
+ if (code) goto _err;
+ pIter->iBlock = -1;
}
- if (*ppData) goto _exit;
- } else if (pReader->pSstBlk) {
- while (pReader->pSstBlk) {
- if (pReader->pSstBlk->minVer <= pReader->ever && pReader->pSstBlk->maxVer >= pReader->sver) {
- // load data (todo)
- }
+ pReader->pIter = NULL;
+ } else if (pIter->type == SNAP_STT_FILE_ITER) {
+ for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) {
+ SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk);
- // next
- pReader->iBlockL++;
- if (pReader->iBlockL < taosArrayGetSize(pReader->aSstBlk)) {
- pReader->pSstBlk = (SSttBlk*)taosArrayGetSize(pReader->aSstBlk);
- } else {
- pReader->pSstBlk = NULL;
- }
+ if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue;
- if (*ppData) goto _exit;
+ code = tsdbReadSttBlock(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData);
+ if (code) goto _err;
+
+ pIter->iRow = -1;
+ goto _find_row;
}
+
+ pReader->pIter = NULL;
+ } else {
+ ASSERT(0);
+ }
+ }
+
+ _out:
+ pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt);
+ if (pReader->pIter && pIter) {
+ int32_t c = tRowInfoCmprFn(&pReader->pIter->rInfo, &pIter->rInfo);
+ if (c > 0) {
+ tRBTreePut(&pReader->rbt, (SRBTreeNode*)pReader->pIter);
+ pReader->pIter = NULL;
} else {
+ ASSERT(c);
+ }
+ }
+ }
+
+ if (pReader->pIter == NULL) {
+ pReader->pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt);
+ if (pReader->pIter) {
+ tRBTreeDrop(&pReader->rbt, (SRBTreeNode*)pReader->pIter);
+ }
+ }
+
+ return code;
+
+_err:
+ return code;
+}
+
+static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) {
+ int32_t code = 0;
+
+ ASSERT(pReader->bData.nRow);
+
+ int32_t aBufN[5] = {0};
+ code = tCmprBlockData(&pReader->bData, TWO_STAGE_COMP, NULL, NULL, pReader->aBuf, aBufN);
+ if (code) goto _exit;
+
+ int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3];
+ *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size);
+ if (*ppData == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData;
+ pHdr->type = SNAP_DATA_TSDB;
+ pHdr->size = size;
+
+ memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]);
+ memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]);
+ if (aBufN[1]) {
+ memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]);
+ }
+ if (aBufN[0]) {
+ memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]);
+ }
+
+_exit:
+ return code;
+}
+
+static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) {
+ int32_t code = 0;
+ STsdb* pTsdb = pReader->pTsdb;
+
+ while (true) {
+ if (pReader->pDataFReader == NULL) {
+ code = tsdbSnapReadOpenFile(pReader);
+ if (code) goto _err;
+ }
+
+ if (pReader->pDataFReader == NULL) break;
+
+ SRowInfo* pRowInfo = tsdbSnapGetRow(pReader);
+ if (pRowInfo == NULL) {
+ tsdbDataFReaderClose(&pReader->pDataFReader);
+ continue;
+ }
+
+ TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid};
+ SBlockData* pBlockData = &pReader->bData;
+
+ code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable);
+ if (code) goto _err;
+
+ code = tBlockDataInit(pBlockData, id.suid, id.uid, pReader->skmTable.pTSchema);
+ if (code) goto _err;
+
+ while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) {
+ code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid);
+ if (code) goto _err;
+
+ code = tsdbSnapNextRow(pReader);
+ if (code) goto _err;
+
+ pRowInfo = tsdbSnapGetRow(pReader);
+ if (pRowInfo == NULL) {
tsdbDataFReaderClose(&pReader->pDataFReader);
break;
}
+
+ if (pBlockData->nRow >= 4096) break;
}
+
+ code = tsdbSnapCmprData(pReader, ppData);
+ if (code) goto _err;
+
+ break;
}
-_exit:
return code;
_err:
@@ -216,7 +394,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) {
size += tPutDelData(NULL, pDelData);
}
}
-
if (size == 0) continue;
// org data
@@ -292,23 +469,33 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type
goto _err;
}
+ // data
pReader->fid = INT32_MIN;
- pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
- if (pReader->aBlockIdx == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- pReader->aSstBlk = taosArrayInit(0, sizeof(SSttBlk));
- if (pReader->aSstBlk == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) {
+ SFDataIter* pIter = &pReader->aFDataIter[iIter];
+
+ if (iIter == 0) {
+ pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pIter->aBlockIdx == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ } else {
+ pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pIter->aSttBlk == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
+ }
+
+ code = tBlockDataCreate(&pIter->bData);
+ if (code) goto _err;
}
- pReader->mBlock = tMapDataInit();
- code = tBlockDataCreate(&pReader->oBlockData);
- if (code) goto _err;
- code = tBlockDataCreate(&pReader->nBlockData);
+
+ code = tBlockDataCreate(&pReader->bData);
if (code) goto _err;
+ // del
pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx));
if (pReader->aDelIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -335,18 +522,26 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
int32_t code = 0;
STsdbSnapReader* pReader = *ppReader;
- if (pReader->pDataFReader) {
- tsdbDataFReaderClose(&pReader->pDataFReader);
- }
- taosArrayDestroy(pReader->aSstBlk);
- taosArrayDestroy(pReader->aBlockIdx);
- tMapDataClear(&pReader->mBlock);
- tBlockDataDestroy(&pReader->oBlockData, 1);
- tBlockDataDestroy(&pReader->nBlockData, 1);
-
- if (pReader->pDelFReader) {
- tsdbDelFReaderClose(&pReader->pDelFReader);
+ // data
+ if (pReader->pDataFReader) tsdbDataFReaderClose(&pReader->pDataFReader);
+ for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) {
+ SFDataIter* pIter = &pReader->aFDataIter[iIter];
+
+ if (iIter == 0) {
+ taosArrayDestroy(pIter->aBlockIdx);
+ tMapDataClear(&pIter->mBlock);
+ } else {
+ taosArrayDestroy(pIter->aSttBlk);
+ }
+
+ tBlockDataDestroy(&pIter->bData, 1);
}
+
+ tBlockDataDestroy(&pReader->bData, 1);
+ tTSchemaDestroy(pReader->skmTable.pTSchema);
+
+ // del
+ if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader);
taosArrayDestroy(pReader->aDelIdx);
taosArrayDestroy(pReader->aDelData);
@@ -354,6 +549,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) {
tsdbInfo("vgId:%d, vnode snapshot tsdb reader closed for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path);
+ for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) {
+ tFree(pReader->aBuf[iBuf]);
+ }
+
taosMemoryFree(pReader);
*ppReader = NULL;
return code;
@@ -410,40 +609,37 @@ struct STsdbSnapWriter {
STsdbFS fs;
// config
- int32_t minutes;
- int8_t precision;
- int32_t minRow;
- int32_t maxRow;
- int8_t cmprAlg;
- int64_t commitID;
-
+ int32_t minutes;
+ int8_t precision;
+ int32_t minRow;
+ int32_t maxRow;
+ int8_t cmprAlg;
+ int64_t commitID;
uint8_t* aBuf[5];
+
// for data file
SBlockData bData;
-
- int32_t fid;
- SDataFReader* pDataFReader;
- SArray* aBlockIdx; // SArray
- int32_t iBlockIdx;
- SBlockIdx* pBlockIdx;
- SMapData mBlock; // SMapData
- int32_t iBlock;
- SBlockData* pBlockData;
- int32_t iRow;
- SBlockData bDataR;
- SArray* aSstBlk; // SArray
- int32_t iBlockL;
- SBlockData lDataR;
-
- SDataFWriter* pDataFWriter;
- SBlockIdx* pBlockIdxW; // NULL when no committing table
- SDataBlk blockW;
- SBlockData bDataW;
- SBlockIdx blockIdxW;
-
- SMapData mBlockW; // SMapData
- SArray* aBlockIdxW; // SArray
- SArray* aBlockLW; // SArray
+ int32_t fid;
+ TABLEID id;
+ SSkmInfo skmTable;
+ struct {
+ SDataFReader* pReader;
+ SArray* aBlockIdx;
+ int32_t iBlockIdx;
+ SBlockIdx* pBlockIdx;
+ SMapData mDataBlk;
+ int32_t iDataBlk;
+ SBlockData bData;
+ int32_t iRow;
+ } dReader;
+ struct {
+ SDataFWriter* pWriter;
+ SArray* aBlockIdx;
+ SMapData mDataBlk;
+ SArray* aSttBlk;
+ SBlockData bData;
+ SBlockData sData;
+ } dWriter;
// for del file
SDelFReader* pDelFReader;
@@ -454,520 +650,447 @@ struct STsdbSnapWriter {
SArray* aDelIdxW;
};
-static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
+// SNAP_DATA_TSDB
+extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg);
+extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg);
+
+static int32_t tsdbSnapNextTableData(STsdbSnapWriter* pWriter) {
int32_t code = 0;
- ASSERT(pWriter->pDataFWriter);
+ ASSERT(pWriter->dReader.iRow >= pWriter->dReader.bData.nRow);
- if (pWriter->pBlockIdxW == NULL) goto _exit;
+ if (pWriter->dReader.iBlockIdx < taosArrayGetSize(pWriter->dReader.aBlockIdx)) {
+ pWriter->dReader.pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->dReader.aBlockIdx, pWriter->dReader.iBlockIdx);
- // consume remain rows
- if (pWriter->pBlockData) {
- ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
- while (pWriter->iRow < pWriter->pBlockData->nRow) {
- code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL,
- 0); // todo
- if (code) goto _err;
+ code = tsdbReadDataBlk(pWriter->dReader.pReader, pWriter->dReader.pBlockIdx, &pWriter->dReader.mDataBlk);
+ if (code) goto _exit;
- if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) {
- // pWriter->blockW.last = 0;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- if (code) goto _err;
+ pWriter->dReader.iBlockIdx++;
+ } else {
+ pWriter->dReader.pBlockIdx = NULL;
+ tMapDataReset(&pWriter->dReader.mDataBlk);
+ }
+ pWriter->dReader.iDataBlk = 0; // point to the next one
+ tBlockDataReset(&pWriter->dReader.bData);
+ pWriter->dReader.iRow = 0;
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
+_exit:
+ return code;
+}
- tDataBlkReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
+static int32_t tsdbSnapWriteCopyData(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ while (true) {
+ if (pWriter->dReader.pBlockIdx == NULL) break;
+ if (tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, pId) >= 0) break;
+
+ SBlockIdx blkIdx = *pWriter->dReader.pBlockIdx;
+ code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dReader.mDataBlk, &blkIdx);
+ if (code) goto _exit;
- pWriter->iRow++;
+ if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blkIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
}
+
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _exit;
}
- // write remain data if has
- if (pWriter->bDataW.nRow > 0) {
- // pWriter->blockW.last = 0;
- if (pWriter->bDataW.nRow < pWriter->minRow) {
- if (pWriter->iBlock > pWriter->mBlock.nItem) {
- // pWriter->blockW.last = 1;
- }
- }
+_exit:
+ return code;
+}
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
+static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
+ code = tsdbSnapWriteCopyData(pWriter, pId);
+ if (code) goto _err;
+
+ pWriter->id.suid = pId->suid;
+ pWriter->id.uid = pId->uid;
+
+ code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable);
+ if (code) goto _err;
+
+ tMapDataReset(&pWriter->dWriter.mDataBlk);
+ code = tBlockDataInit(&pWriter->dWriter.bData, pId->suid, pId->uid, pWriter->skmTable.pTSchema);
+ if (code) goto _err;
+
+ return code;
+
+_err:
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
+ return code;
+}
+
+static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) {
+ int32_t code = 0;
+
+ if (pWriter->id.suid == 0 && pWriter->id.uid == 0) return code;
+
+ int32_t c = 1;
+ if (pWriter->dReader.pBlockIdx) {
+ c = tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &pWriter->id);
+ ASSERT(c >= 0);
}
- while (true) {
- if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
+ if (c == 0) {
+ SBlockData* pBData = &pWriter->dWriter.bData;
- SDataBlk block;
- tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
+ for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) {
+ TSDBROW row = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow);
- // if (block.last) {
- // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
- // if (code) goto _err;
+ code = tBlockDataAppendRow(pBData, &row, NULL, pWriter->id.uid);
+ if (code) goto _err;
- // tBlockReset(&block);
- // block.last = 1;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block,
- // pWriter->cmprAlg);
- // if (code) goto _err;
- // }
+ if (pBData->nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+ }
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg);
if (code) goto _err;
- pWriter->iBlock++;
+ for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk);
+
+ code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk);
+ if (code) goto _err;
+ }
+
+ code = tsdbSnapNextTableData(pWriter);
+ if (code) goto _err;
}
- // SDataBlk
- // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW);
- // if (code) goto _err;
+ if (pWriter->dWriter.mDataBlk.nItem) {
+ SBlockIdx blockIdx = {.suid = pWriter->id.suid, .uid = pWriter->id.uid};
+ code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dWriter.mDataBlk, &blockIdx);
- // SBlockIdx
- if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blockIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _err;
+ }
}
-_exit:
- tsdbInfo("vgId:%d, tsdb snapshot write table data end for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
+ pWriter->id.suid = 0;
+ pWriter->id.uid = 0;
+
return code;
_err:
- tsdbError("vgId:%d, tsdb snapshot write table data end for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
return code;
}
-static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) {
+static int32_t tsdbSnapWriteOpenFile(STsdbSnapWriter* pWriter, int32_t fid) {
int32_t code = 0;
+ STsdb* pTsdb = pWriter->pTsdb;
- code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock);
- if (code) goto _err;
+ ASSERT(pWriter->dWriter.pWriter == NULL);
+
+ pWriter->fid = fid;
+ pWriter->id = (TABLEID){0};
+ SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
- // SBlockData
- SDataBlk block;
- tMapDataReset(&pWriter->mBlockW);
- for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) {
- tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetDataBlk);
-
- // if (block.last) {
- // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL);
- // if (code) goto _err;
-
- // tBlockReset(&block);
- // block.last = 1;
- // code =
- // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block,
- // pWriter->cmprAlg);
- // if (code) goto _err;
- // }
-
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
+ // Reader
+ if (pSet) {
+ code = tsdbDataFReaderOpen(&pWriter->dReader.pReader, pWriter->pTsdb, pSet);
if (code) goto _err;
- }
- // SDataBlk
- SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid};
- code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx);
+ code = tsdbReadBlockIdx(pWriter->dReader.pReader, pWriter->dReader.aBlockIdx);
+ if (code) goto _err;
+ } else {
+ ASSERT(pWriter->dReader.pReader == NULL);
+ taosArrayClear(pWriter->dReader.aBlockIdx);
+ }
+ pWriter->dReader.iBlockIdx = 0; // point to the next one
+ code = tsdbSnapNextTableData(pWriter);
if (code) goto _err;
- // SBlockIdx
- if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
+ // Writer
+ SHeadFile fHead = {.commitID = pWriter->commitID};
+ SDataFile fData = {.commitID = pWriter->commitID};
+ SSmaFile fSma = {.commitID = pWriter->commitID};
+ SSttFile fStt = {.commitID = pWriter->commitID};
+ SDFileSet wSet = {.fid = pWriter->fid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma};
+ if (pSet) {
+ wSet.diskId = pSet->diskId;
+ fData = *pSet->pDataF;
+ fSma = *pSet->pSmaF;
+ for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) {
+ wSet.aSttF[iStt] = pSet->aSttF[iStt];
+ }
+ wSet.nSttF = pSet->nSttF + 1; // TODO: fix pSet->nSttF == pTsdb->maxFile
+ } else {
+ SDiskID did = {0};
+ tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did);
+ tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did);
+ wSet.diskId = did;
+ wSet.nSttF = 1;
}
+ wSet.aSttF[wSet.nSttF - 1] = &fStt;
+
+ code = tsdbDataFWriterOpen(&pWriter->dWriter.pWriter, pWriter->pTsdb, &wSet);
+ if (code) goto _err;
+ taosArrayClear(pWriter->dWriter.aBlockIdx);
+ tMapDataReset(&pWriter->dWriter.mDataBlk);
+ taosArrayClear(pWriter->dWriter.aSttBlk);
+ tBlockDataReset(&pWriter->dWriter.bData);
+ tBlockDataReset(&pWriter->dWriter.sData);
-_exit:
return code;
_err:
- tsdbError("vgId:%d, tsdb snapshot move write table data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
return code;
}
-static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) {
- int32_t code = 0;
- SBlockData* pBlockData = &pWriter->bData;
- int32_t iRow = 0;
- TSDBROW row;
- TSDBROW* pRow = &row;
+static int32_t tsdbSnapWriteCloseFile(STsdbSnapWriter* pWriter) {
+ int32_t code = 0;
- // // correct schema
- // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData);
- // if (code) goto _err;
+ ASSERT(pWriter->dWriter.pWriter);
- // loop to merge
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- while (true) {
- if (pRow == NULL) break;
+ code = tsdbSnapWriteTableDataEnd(pWriter);
+ if (code) goto _err;
- if (pWriter->pBlockData) {
- ASSERT(pWriter->iRow < pWriter->pBlockData->nRow);
+ // copy remain table data
+ TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX};
+ code = tsdbSnapWriteCopyData(pWriter, &id);
+ if (code) goto _err;
- int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow));
+ code =
+ tsdbWriteSttBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.sData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
+ if (code) goto _err;
- ASSERT(c);
+ // Indices
+ code = tsdbWriteBlockIdx(pWriter->dWriter.pWriter, pWriter->dWriter.aBlockIdx);
+ if (code) goto _err;
- if (c < 0) {
- // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
- // if (code) goto _err;
-
- iRow++;
- if (iRow < pWriter->pBlockData->nRow) {
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
- } else {
- pRow = NULL;
- }
- } else if (c > 0) {
- // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow),
- // NULL); if (code) goto _err;
+ code = tsdbWriteSttBlk(pWriter->dWriter.pWriter, pWriter->dWriter.aSttBlk);
+ if (code) goto _err;
- pWriter->iRow++;
- if (pWriter->iRow >= pWriter->pBlockData->nRow) {
- pWriter->pBlockData = NULL;
- }
- }
- } else {
- TSDBKEY key = TSDBROW_KEY(pRow);
+ code = tsdbUpdateDFileSetHeader(pWriter->dWriter.pWriter);
+ if (code) goto _err;
- while (true) {
- if (pWriter->iBlock >= pWriter->mBlock.nItem) break;
+ code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->dWriter.pWriter->wSet);
+ if (code) goto _err;
- SDataBlk block;
- int32_t c;
+ code = tsdbDataFWriterClose(&pWriter->dWriter.pWriter, 1);
+ if (code) goto _err;
- tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetDataBlk);
+ if (pWriter->dReader.pReader) {
+ code = tsdbDataFReaderClose(&pWriter->dReader.pReader);
+ if (code) goto _err;
+ }
- // if (block.last) {
- // pWriter->pBlockData = &pWriter->bDataR;
+_exit:
+ return code;
- // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
- // NULL); if (code) goto _err; pWriter->iRow = 0;
+_err:
+ return code;
+}
- // pWriter->iBlock++;
- // break;
- // }
+static int32_t tsdbSnapWriteToDataFile(STsdbSnapWriter* pWriter, int32_t iRow, int8_t* done) {
+ int32_t code = 0;
- c = tsdbKeyCmprFn(&block.maxKey, &key);
+ SBlockData* pBData = &pWriter->bData;
+ TABLEID id = {.suid = pBData->suid, .uid = pBData->uid ? pBData->uid : pBData->aUid[iRow]};
+ TSDBROW row = tsdbRowFromBlockData(pBData, iRow);
+ TSDBKEY key = TSDBROW_KEY(&row);
- ASSERT(c);
+ *done = 0;
+ while (pWriter->dReader.iRow < pWriter->dReader.bData.nRow ||
+ pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem) {
+ // Merge row by row
+ for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) {
+ TSDBROW trow = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow);
+ TSDBKEY tKey = TSDBROW_KEY(&trow);
- if (c < 0) {
- if (pWriter->bDataW.nRow) {
- // pWriter->blockW.last = 0;
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
+ ASSERT(pWriter->dReader.bData.suid == id.suid && pWriter->dReader.bData.uid == id.uid);
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
+ int32_t c = tsdbKeyCmprFn(&key, &tKey);
+ if (c < 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid);
+ if (code) goto _err;
+ } else if (c > 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &trow, NULL, id.uid);
+ if (code) goto _err;
+ } else {
+ ASSERT(0);
+ }
- tDataBlkReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
- }
+ if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
+ }
- code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutDataBlk);
- if (code) goto _err;
+ if (c < 0) {
+ *done = 1;
+ goto _exit;
+ }
+ }
- pWriter->iBlock++;
- } else {
- c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey);
+ // Merge row by block
+ SDataBlk tDataBlk = {.minKey = key, .maxKey = key};
+ for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) {
+ SDataBlk dataBlk;
+ tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk);
- ASSERT(c);
+ int32_t c = tDataBlkCmprFn(&dataBlk, &tDataBlk);
+ if (c < 0) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
- if (c > 0) {
- pWriter->pBlockData = &pWriter->bDataR;
- // code =
- // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL,
- // NULL);
- // if (code) goto _err;
- pWriter->iRow = 0;
+ code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk);
+ if (code) goto _err;
+ } else if (c > 0) {
+ code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid);
+ if (code) goto _err;
- pWriter->iBlock++;
- }
- break;
+ if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) {
+ code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk,
+ pWriter->cmprAlg);
+ if (code) goto _err;
}
- }
-
- if (pWriter->pBlockData) continue;
-
- // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL);
- // if (code) goto _err;
- iRow++;
- if (iRow < pBlockData->nRow) {
- *pRow = tsdbRowFromBlockData(pBlockData, iRow);
+ *done = 1;
+ goto _exit;
} else {
- pRow = NULL;
+ code = tsdbReadDataBlockEx(pWriter->dReader.pReader, &dataBlk, &pWriter->dReader.bData);
+ if (code) goto _err;
+ pWriter->dReader.iRow = 0;
+
+ pWriter->dReader.iDataBlk++;
+ break;
}
}
-
- _check_write:
- if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue;
-
- _write_block:
- // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW,
- // &pWriter->blockW, pWriter->cmprAlg);
- // if (code) goto _err;
-
- code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutDataBlk);
- if (code) goto _err;
-
- tDataBlkReset(&pWriter->blockW);
- tBlockDataClear(&pWriter->bDataW);
}
+_exit:
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb write table data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
return code;
}
-static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) {
- int32_t code = 0;
- SBlockData* pBlockData = &pWriter->bData;
- TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData);
- TSDBKEY keyLast = tBlockDataLastKey(pBlockData);
-
- // end last table write if should
- if (pWriter->pBlockIdxW) {
- int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id);
- if (c < 0) {
- // end
- code = tsdbSnapWriteTableDataEnd(pWriter);
- if (code) goto _err;
-
- // reset
- pWriter->pBlockIdxW = NULL;
- } else if (c > 0) {
- ASSERT(0);
- }
- }
-
- // start new table data write if need
- if (pWriter->pBlockIdxW == NULL) {
- // write table data ahead
- while (true) {
- if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break;
-
- SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
- int32_t c = tTABLEIDCmprFn(pBlockIdx, &id);
+static int32_t tsdbSnapWriteToSttFile(STsdbSnapWriter* pWriter, int32_t iRow) {
+ int32_t code = 0;
- if (c >= 0) break;
+ TABLEID id = {.suid = pWriter->bData.suid,
+ .uid = pWriter->bData.uid ? pWriter->bData.uid : pWriter->bData.aUid[iRow]};
+ TSDBROW row = tsdbRowFromBlockData(&pWriter->bData, iRow);
+ SBlockData* pBData = &pWriter->dWriter.sData;
- code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx);
+ if (pBData->suid || pBData->uid) {
+ if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) {
+ code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
if (code) goto _err;
- pWriter->iBlockIdx++;
+ pBData->suid = 0;
+ pBData->uid = 0;
}
+ }
- // reader
- pWriter->pBlockIdx = NULL;
- if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
- ASSERT(pWriter->pDataFReader);
-
- SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx);
- int32_t c = tTABLEIDCmprFn(pBlockIdx, &id);
-
- ASSERT(c >= 0);
-
- if (c == 0) {
- pWriter->pBlockIdx = pBlockIdx;
- pWriter->iBlockIdx++;
- }
- }
-
- if (pWriter->pBlockIdx) {
- code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock);
- if (code) goto _err;
- } else {
- tMapDataReset(&pWriter->mBlock);
- }
- pWriter->iBlock = 0;
- pWriter->pBlockData = NULL;
- pWriter->iRow = 0;
-
- // writer
- pWriter->pBlockIdxW = &pWriter->blockIdxW;
- pWriter->pBlockIdxW->suid = id.suid;
- pWriter->pBlockIdxW->uid = id.uid;
+ if (pBData->suid == 0 && pBData->uid == 0) {
+ code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable);
+ if (code) goto _err;
- tDataBlkReset(&pWriter->blockW);
- tBlockDataReset(&pWriter->bDataW);
- tMapDataReset(&pWriter->mBlockW);
+ code = tBlockDataInit(pBData, pWriter->id.suid, pWriter->id.suid ? 0 : pWriter->id.uid, pWriter->skmTable.pTSchema);
+ if (code) goto _err;
}
- ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid);
- ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid));
-
- code = tsdbSnapWriteTableDataImpl(pWriter);
+ code = tBlockDataAppendRow(pBData, &row, NULL, id.uid);
if (code) goto _err;
+ if (pBData->nRow >= pWriter->maxRow) {
+ code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg);
+ if (code) goto _err;
+ }
+
_exit:
- tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path);
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb write data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode),
- pWriter->pTsdb->path, tstrerror(code));
return code;
}
-static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) {
+static int32_t tsdbSnapWriteRowData(STsdbSnapWriter* pWriter, int32_t iRow) {
int32_t code = 0;
- STsdb* pTsdb = pWriter->pTsdb;
-
- if (pWriter->pDataFWriter == NULL) goto _exit;
- // finish current table
- code = tsdbSnapWriteTableDataEnd(pWriter);
- if (code) goto _err;
+ SBlockData* pBlockData = &pWriter->bData;
+ TABLEID id = {.suid = pBlockData->suid, .uid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[iRow]};
- // move remain table
- while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) {
- code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx));
+ // End last table data write if need
+ if (tTABLEIDCmprFn(&pWriter->id, &id) != 0) {
+ code = tsdbSnapWriteTableDataEnd(pWriter);
if (code) goto _err;
-
- pWriter->iBlockIdx++;
}
- // write remain stuff
- if (taosArrayGetSize(pWriter->aBlockLW) > 0) {
- code = tsdbWriteSttBlk(pWriter->pDataFWriter, pWriter->aBlockIdxW);
+ // Start new table data write if need
+ if (pWriter->id.suid == 0 && pWriter->id.uid == 0) {
+ code = tsdbSnapWriteTableDataStart(pWriter, &id);
if (code) goto _err;
}
- if (taosArrayGetSize(pWriter->aBlockIdx) > 0) {
- code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW);
+ // Merge with .data file data
+ int8_t done = 0;
+ if (pWriter->dReader.pBlockIdx && tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &id) == 0) {
+ code = tsdbSnapWriteToDataFile(pWriter, iRow, &done);
if (code) goto _err;
}
- code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet);
- if (code) goto _err;
-
- code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1);
- if (code) goto _err;
-
- if (pWriter->pDataFReader) {
- code = tsdbDataFReaderClose(&pWriter->pDataFReader);
+ // Append to the .stt data block (todo: check if need to set/reload sst block)
+ if (!done) {
+ code = tsdbSnapWriteToSttFile(pWriter, iRow);
if (code) goto _err;
}
_exit:
- tsdbInfo("vgId:%d, vnode snapshot tsdb writer data end for %s", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
_err:
- tsdbError("vgId:%d, vnode snapshot tsdb writer data end for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
- tstrerror(code));
+ tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code));
return code;
}
static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
- int32_t code = 0;
- STsdb* pTsdb = pWriter->pTsdb;
- SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
- TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
- int64_t n;
-
- // decode
+ int32_t code = 0;
+ STsdb* pTsdb = pWriter->pTsdb;
SBlockData* pBlockData = &pWriter->bData;
- code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData,
- pWriter->aBuf);
- if (code) goto _err;
- // open file
- TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]};
- TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1],
- .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]};
+ // Decode data
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+ code = tDecmprBlockData(pHdr->data, pHdr->size, pBlockData, pWriter->aBuf);
+ if (code) goto _err;
- int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision);
- ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision));
- if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) {
- // end last file data write if need
- code = tsdbSnapWriteDataEnd(pWriter);
- if (code) goto _err;
+ ASSERT(pBlockData->nRow > 0);
- pWriter->fid = fid;
+ // Loop to handle each row
+ for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) {
+ TSKEY ts = pBlockData->aTSKEY[iRow];
+ int32_t fid = tsdbKeyFid(ts, pWriter->minutes, pWriter->precision);
- // read
- SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ);
- if (pSet) {
- code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet);
- if (code) goto _err;
+ if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) {
+ if (pWriter->dWriter.pWriter) {
+ ASSERT(fid > pWriter->fid);
- code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx);
- if (code) goto _err;
+ code = tsdbSnapWriteCloseFile(pWriter);
+ if (code) goto _err;
+ }
- code = tsdbReadSttBlk(pWriter->pDataFReader, 0, pWriter->aSstBlk);
+ code = tsdbSnapWriteOpenFile(pWriter, fid);
if (code) goto _err;
- } else {
- ASSERT(pWriter->pDataFReader == NULL);
- taosArrayClear(pWriter->aBlockIdx);
- taosArrayClear(pWriter->aSstBlk);
- }
- pWriter->iBlockIdx = 0;
- pWriter->pBlockIdx = NULL;
- tMapDataReset(&pWriter->mBlock);
- pWriter->iBlock = 0;
- pWriter->pBlockData = NULL;
- pWriter->iRow = 0;
- pWriter->iBlockL = 0;
- tBlockDataReset(&pWriter->bDataR);
- tBlockDataReset(&pWriter->lDataR);
-
- // write
- SHeadFile fHead;
- SDataFile fData;
- SSttFile fLast;
- SSmaFile fSma;
- SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .aSttF[0] = &fLast, .pSmaF = &fSma};
-
- if (pSet) {
- wSet.diskId = pSet->diskId;
- wSet.fid = fid;
- wSet.nSttF = 1;
- fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
- fData = *pSet->pDataF;
- fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0};
- fSma = *pSet->pSmaF;
- } else {
- wSet.diskId = (SDiskID){.level = 0, .id = 0};
- wSet.fid = fid;
- wSet.nSttF = 1;
- fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0};
- fData = (SDataFile){.commitID = pWriter->commitID, .size = 0};
- fLast = (SSttFile){.commitID = pWriter->commitID, .size = 0, .offset = 0};
- fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0};
}
- code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet);
+ code = tsdbSnapWriteRowData(pWriter, iRow);
if (code) goto _err;
-
- taosArrayClear(pWriter->aBlockIdxW);
- taosArrayClear(pWriter->aBlockLW);
- tMapDataReset(&pWriter->mBlockW);
- pWriter->pBlockIdxW = NULL;
- tBlockDataReset(&pWriter->bDataW);
}
- code = tsdbSnapWriteTableData(pWriter, id);
- if (code) goto _err;
-
- tsdbInfo("vgId:%d, vnode snapshot tsdb write data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d",
- TD_VID(pTsdb->pVnode), pTsdb->path, fid, id.suid, id.suid, pBlockData->nRow);
return code;
_err:
@@ -976,10 +1099,41 @@ _err:
return code;
}
+// SNAP_DATA_DEL
+static int32_t tsdbSnapMoveWriteDelData(STsdbSnapWriter* pWriter, TABLEID* pId) {
+ int32_t code = 0;
+
+ while (true) {
+ if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
+
+ SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
+
+ if (tTABLEIDCmprFn(pDelIdx, pId) >= 0) break;
+
+ code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
+ if (code) goto _exit;
+
+ SDelIdx delIdx = *pDelIdx;
+ code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
+ if (code) goto _exit;
+
+ if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
+ code = TSDB_CODE_OUT_OF_MEMORY;
+ goto _exit;
+ }
+
+ pWriter->iDelIdx++;
+ }
+
+_exit:
+ return code;
+}
+
static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
+ // Open del file if not opened yet
if (pWriter->pDelFWriter == NULL) {
SDelFile* pDelFile = pWriter->fs.pDelFile;
@@ -990,38 +1144,28 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR);
if (code) goto _err;
+ } else {
+ taosArrayClear(pWriter->aDelIdxR);
}
+ pWriter->iDelIdx = 0;
// writer
- SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0};
+ SDelFile delFile = {.commitID = pWriter->commitID};
code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb);
if (code) goto _err;
+ taosArrayClear(pWriter->aDelIdxW);
}
- // process the del data
- TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr));
-
- while (true) {
- if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break;
- if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break;
-
- SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
-
- code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
- if (code) goto _err;
-
- SDelIdx delIdx = *pDelIdx;
- code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
- if (code) goto _err;
+ SSnapDataHdr* pHdr = (SSnapDataHdr*)pData;
+ TABLEID id = *(TABLEID*)pHdr->data;
- if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
+ ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData);
- pWriter->iDelIdx++;
- }
+ // Move write data < id
+ code = tsdbSnapMoveWriteDelData(pWriter, &id);
+ if (code) goto _err;
+ // Merge incoming data with current
if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) &&
tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) {
SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
@@ -1055,7 +1199,6 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32
goto _err;
}
-_exit:
return code;
_err:
@@ -1068,23 +1211,14 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
int32_t code = 0;
STsdb* pTsdb = pWriter->pTsdb;
- if (pWriter->pDelFWriter == NULL) goto _exit;
-
- for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) {
- SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx);
-
- code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData);
- if (code) goto _err;
+ if (pWriter->pDelFWriter == NULL) return code;
- SDelIdx delIdx = *pDelIdx;
- code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx);
- if (code) goto _err;
+ TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX};
+ code = tsdbSnapMoveWriteDelData(pWriter, &id);
+ if (code) goto _err;
- if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
- }
+ code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdxW);
+ if (code) goto _err;
code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter);
if (code) goto _err;
@@ -1100,7 +1234,6 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) {
if (code) goto _err;
}
-_exit:
tsdbInfo("vgId:%d, vnode snapshot tsdb write del for %s end", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
@@ -1110,6 +1243,7 @@ _err:
return code;
}
+// APIs
int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) {
int32_t code = 0;
STsdbSnapWriter* pWriter = NULL;
@@ -1135,39 +1269,38 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression;
pWriter->commitID = pTsdb->pVnode->state.commitID;
- // for data file
+ // SNAP_DATA_TSDB
code = tBlockDataCreate(&pWriter->bData);
-
if (code) goto _err;
- pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
- if (pWriter->aBlockIdx == NULL) {
+
+ pWriter->fid = INT32_MIN;
+ pWriter->id = (TABLEID){0};
+ // Reader
+ pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pWriter->dReader.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
- code = tBlockDataCreate(&pWriter->bDataR);
+ code = tBlockDataCreate(&pWriter->dReader.bData);
if (code) goto _err;
- pWriter->aSstBlk = taosArrayInit(0, sizeof(SSttBlk));
- if (pWriter->aSstBlk == NULL) {
+ // Writer
+ pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx));
+ if (pWriter->dWriter.aBlockIdx == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
-
- pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx));
- if (pWriter->aBlockIdxW == NULL) {
+ pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk));
+ if (pWriter->dWriter.aSttBlk == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
goto _err;
}
- code = tBlockDataCreate(&pWriter->bDataW);
+ code = tBlockDataCreate(&pWriter->dWriter.bData);
+ if (code) goto _err;
+ code = tBlockDataCreate(&pWriter->dWriter.sData);
if (code) goto _err;
- pWriter->aBlockLW = taosArrayInit(0, sizeof(SSttBlk));
- if (pWriter->aBlockLW == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _err;
- }
-
- // for del file
+ // SNAP_DATA_DEL
pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx));
if (pWriter->aDelIdxR == NULL) {
code = TSDB_CODE_OUT_OF_MEMORY;
@@ -1188,6 +1321,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr
tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path);
return code;
+
_err:
tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path,
tstrerror(code));
@@ -1198,14 +1332,17 @@ _err:
int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
int32_t code = 0;
STsdbSnapWriter* pWriter = *ppWriter;
+ STsdb* pTsdb = pWriter->pTsdb;
if (rollback) {
ASSERT(0);
// code = tsdbFSRollback(pWriter->pTsdb->pFS);
// if (code) goto _err;
} else {
- code = tsdbSnapWriteDataEnd(pWriter);
- if (code) goto _err;
+ if (pWriter->dWriter.pWriter) {
+ code = tsdbSnapWriteCloseFile(pWriter);
+ if (code) goto _err;
+ }
code = tsdbSnapWriteDelEnd(pWriter);
if (code) goto _err;
@@ -1213,14 +1350,44 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) {
code = tsdbFSCommit1(pWriter->pTsdb, &pWriter->fs);
if (code) goto _err;
+ // lock
+ taosThreadRwlockWrlock(&pTsdb->rwLock);
+
code = tsdbFSCommit2(pWriter->pTsdb, &pWriter->fs);
- if (code) goto _err;
+ if (code) {
+ taosThreadRwlockUnlock(&pTsdb->rwLock);
+ goto _err;
+ }
+
+ // unlock
+ taosThreadRwlockUnlock(&pTsdb->rwLock);
}
+ // SNAP_DATA_DEL
+ taosArrayDestroy(pWriter->aDelIdxW);
+ taosArrayDestroy(pWriter->aDelData);
+ taosArrayDestroy(pWriter->aDelIdxR);
+
+ // SNAP_DATA_TSDB
+
+ // Writer
+ tBlockDataDestroy(&pWriter->dWriter.sData, 1);
+ tBlockDataDestroy(&pWriter->dWriter.bData, 1);
+ taosArrayDestroy(pWriter->dWriter.aSttBlk);
+ tMapDataClear(&pWriter->dWriter.mDataBlk);
+ taosArrayDestroy(pWriter->dWriter.aBlockIdx);
+
+ // Reader
+ tBlockDataDestroy(&pWriter->dReader.bData, 1);
+ tMapDataClear(&pWriter->dReader.mDataBlk);
+ taosArrayDestroy(pWriter->dReader.aBlockIdx);
+
+ tBlockDataDestroy(&pWriter->bData, 1);
+ tTSchemaDestroy(pWriter->skmTable.pTSchema);
+
for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) {
tFree(pWriter->aBuf[iBuf]);
}
-
tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
taosMemoryFree(pWriter);
*ppWriter = NULL;
@@ -1245,8 +1412,8 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
goto _exit;
} else {
- if (pWriter->pDataFWriter) {
- code = tsdbSnapWriteDataEnd(pWriter);
+ if (pWriter->dWriter.pWriter) {
+ code = tsdbSnapWriteCloseFile(pWriter);
if (code) goto _err;
}
}
@@ -1259,7 +1426,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData)
_exit:
tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path);
-
return code;
_err:
diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c
index caeca45e01d0ec2def492d65f4ce9c6bd112e322..64c150484b4f873bd15fb69c2b238bd840e1ea1f 100644
--- a/source/dnode/vnode/src/tsdb/tsdbUtil.c
+++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c
@@ -909,248 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr
return code;
}
-// SColData ========================================
-void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) {
- pColData->cid = cid;
- pColData->type = type;
- pColData->smaOn = smaOn;
- tColDataReset(pColData);
-}
-
-void tColDataReset(SColData *pColData) {
- pColData->nVal = 0;
- pColData->flag = 0;
- pColData->nData = 0;
-}
-
-void tColDataClear(void *ph) {
- SColData *pColData = (SColData *)ph;
-
- tFree(pColData->pBitMap);
- tFree((uint8_t *)pColData->aOffset);
- tFree(pColData->pData);
-}
-
-int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) {
- int32_t code = 0;
- int64_t size;
- SValue value = {0};
- SValue *pValue = &value;
-
- ASSERT(pColVal->cid == pColData->cid);
- ASSERT(pColVal->type == pColData->type);
-
- // realloc bitmap
- size = BIT2_SIZE(pColData->nVal + 1);
- code = tRealloc(&pColData->pBitMap, size);
- if (code) goto _exit;
- if ((pColData->nVal & 3) == 0) {
- pColData->pBitMap[pColData->nVal >> 2] = 0;
- }
-
- // put value
- if (pColVal->isNone) {
- pColData->flag |= HAS_NONE;
- SET_BIT2(pColData->pBitMap, pColData->nVal, 0);
- } else if (pColVal->isNull) {
- pColData->flag |= HAS_NULL;
- SET_BIT2(pColData->pBitMap, pColData->nVal, 1);
- } else {
- pColData->flag |= HAS_VALUE;
- SET_BIT2(pColData->pBitMap, pColData->nVal, 2);
- pValue = &pColVal->value;
- }
-
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- // offset
- code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1));
- if (code) goto _exit;
- pColData->aOffset[pColData->nVal] = pColData->nData;
-
- // value
- if ((!pColVal->isNone) && (!pColVal->isNull)) {
- code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData);
- if (code) goto _exit;
- memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData);
- pColData->nData += pColVal->value.nData;
- }
- } else {
- code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type));
- if (code) goto _exit;
- pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type);
- }
-
- pColData->nVal++;
-
-_exit:
- return code;
-}
-
-int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) {
- int32_t code = 0;
- int32_t size;
-
- ASSERT(pColDataSrc->nVal > 0);
- ASSERT(pColDataDest->cid = pColDataSrc->cid);
- ASSERT(pColDataDest->type = pColDataSrc->type);
-
- pColDataDest->smaOn = pColDataSrc->smaOn;
- pColDataDest->nVal = pColDataSrc->nVal;
- pColDataDest->flag = pColDataSrc->flag;
-
- // bitmap
- if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) {
- size = BIT2_SIZE(pColDataSrc->nVal);
- code = tRealloc(&pColDataDest->pBitMap, size);
- if (code) goto _exit;
- memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size);
- }
-
- // offset
- if (IS_VAR_DATA_TYPE(pColDataDest->type)) {
- size = sizeof(int32_t) * pColDataSrc->nVal;
-
- code = tRealloc((uint8_t **)&pColDataDest->aOffset, size);
- if (code) goto _exit;
-
- memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size);
- }
-
- // value
- pColDataDest->nData = pColDataSrc->nData;
- code = tRealloc(&pColDataDest->pData, pColDataSrc->nData);
- if (code) goto _exit;
- memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData);
-
-_exit:
- return code;
-}
-
-int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) {
- int32_t code = 0;
-
- ASSERT(iVal < pColData->nVal);
- ASSERT(pColData->flag);
-
- if (pColData->flag == HAS_NONE) {
- *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
- goto _exit;
- } else if (pColData->flag == HAS_NULL) {
- *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
- goto _exit;
- } else if (pColData->flag != HAS_VALUE) {
- uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
- if (v == 0) {
- *pColVal = COL_VAL_NONE(pColData->cid, pColData->type);
- goto _exit;
- } else if (v == 1) {
- *pColVal = COL_VAL_NULL(pColData->cid, pColData->type);
- goto _exit;
- }
- }
-
- // get value
- SValue value;
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- if (iVal + 1 < pColData->nVal) {
- value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal];
- } else {
- value.nData = pColData->nData - pColData->aOffset[iVal];
- }
-
- value.pData = pColData->pData + pColData->aOffset[iVal];
- } else {
- tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type);
- }
- *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value);
-
-_exit:
- return code;
-}
-
-int32_t tPutColData(uint8_t *p, SColData *pColData) {
- int32_t n = 0;
-
- n += tPutI16v(p ? p + n : p, pColData->cid);
- n += tPutI8(p ? p + n : p, pColData->type);
- n += tPutI8(p ? p + n : p, pColData->smaOn);
- n += tPutI32v(p ? p + n : p, pColData->nVal);
- n += tPutU8(p ? p + n : p, pColData->flag);
-
- if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
- if (pColData->flag != HAS_VALUE) {
- // bitmap
-
- int32_t size = BIT2_SIZE(pColData->nVal);
- if (p) {
- memcpy(p + n, pColData->pBitMap, size);
- }
- n += size;
- }
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- // offset
-
- int32_t size = sizeof(int32_t) * pColData->nVal;
- if (p) {
- memcpy(p + n, pColData->aOffset, size);
- }
- n += size;
- }
- n += tPutI32v(p ? p + n : p, pColData->nData);
- if (p) {
- memcpy(p + n, pColData->pData, pColData->nData);
- }
- n += pColData->nData;
-
-_exit:
- return n;
-}
-
-int32_t tGetColData(uint8_t *p, SColData *pColData) {
- int32_t n = 0;
-
- n += tGetI16v(p + n, &pColData->cid);
- n += tGetI8(p + n, &pColData->type);
- n += tGetI8(p + n, &pColData->smaOn);
- n += tGetI32v(p + n, &pColData->nVal);
- n += tGetU8(p + n, &pColData->flag);
-
- if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit;
- if (pColData->flag != HAS_VALUE) {
- // bitmap
-
- int32_t size = BIT2_SIZE(pColData->nVal);
- pColData->pBitMap = p + n;
- n += size;
- }
- if (IS_VAR_DATA_TYPE(pColData->type)) {
- // offset
-
- int32_t size = sizeof(int32_t) * pColData->nVal;
- pColData->aOffset = (int32_t *)(p + n);
- n += size;
- }
- n += tGetI32v(p + n, &pColData->nData);
- pColData->pData = p + n;
- n += pColData->nData;
-
-_exit:
- return n;
-}
-
-static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) {
- SColData *pColData1 = (SColData *)p1;
- SColData *pColData2 = (SColData *)p2;
-
- if (pColData1->cid < pColData2->cid) {
- return -1;
- } else if (pColData1->cid > pColData2->cid) {
- return 1;
- }
-
- return 0;
-}
-
// SBlockData ======================================================
int32_t tBlockDataCreate(SBlockData *pBlockData) {
int32_t code = 0;
@@ -1182,7 +940,7 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) {
tFree((uint8_t *)pBlockData->aVersion);
tFree((uint8_t *)pBlockData->aTSKEY);
taosArrayDestroy(pBlockData->aIdx);
- taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL);
+ taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL);
pBlockData->aUid = NULL;
pBlockData->aVersion = NULL;
pBlockData->aTSKEY = NULL;
@@ -1251,7 +1009,7 @@ void tBlockDataClear(SBlockData *pBlockData) {
pBlockData->nRow = 0;
for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) {
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData);
- tColDataReset(pColData);
+ tColDataClear(pColData);
}
}
@@ -1501,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD
while (lidx <= ridx) {
int32_t midx = (lidx + ridx) / 2;
SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx);
- int32_t c = tColDataCmprFn(pColData, &(SColData){.cid = cid});
+ int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1);
if (c == 0) {
*ppColData = pColData;
@@ -1986,47 +1744,16 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol
int32_t size = 0;
// bitmap
if (pColData->flag != HAS_VALUE) {
- uint8_t *pBitMap = pColData->pBitMap;
- int32_t szBitMap = BIT2_SIZE(pColData->nVal);
-
- // BIT2 to BIT1
- if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+ int32_t szBitMap;
+ if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+ szBitMap = BIT2_SIZE(pColData->nVal);
+ } else {
szBitMap = BIT1_SIZE(pColData->nVal);
- pBitMap = taosMemoryCalloc(1, szBitMap);
- if (pBitMap == NULL) {
- code = TSDB_CODE_OUT_OF_MEMORY;
- goto _exit;
- }
-
- for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) {
- uint8_t v = GET_BIT2(pColData->pBitMap, iVal);
- switch (pColData->flag) {
- case (HAS_NULL | HAS_NONE):
- SET_BIT1(pBitMap, iVal, v);
- break;
- case (HAS_VALUE | HAS_NONE):
- if (v) {
- SET_BIT1(pBitMap, iVal, 1);
- } else {
- SET_BIT1(pBitMap, iVal, 0);
- }
- break;
- case (HAS_VALUE | HAS_NULL):
- SET_BIT1(pBitMap, iVal, v - 1);
- break;
- default:
- ASSERT(0);
- }
- }
}
- code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap,
- ppBuf);
+ code = tsdbCmprData(pColData->pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size,
+ &pBlockCol->szBitmap, ppBuf);
if (code) goto _exit;
-
- if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
- taosMemoryFree(pBitMap);
- }
}
size += pBlockCol->szBitmap;
@@ -2064,46 +1791,15 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in
uint8_t *p = pIn;
// bitmap
if (pBlockCol->szBitmap) {
- if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) {
- uint8_t *pBitMap = NULL;
- code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap,
- BIT1_SIZE(pColData->nVal), ppBuf);
- if (code) goto _exit;
-
- code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal));
- if (code) {
- tFree(pBitMap);
- goto _exit;
- }
-
- // BIT1 to BIT2
- for (int32_t iVal = 0; iVal < nVal; iVal++) {
- uint8_t v = GET_BIT1(pBitMap, iVal);
- switch (pBlockCol->flag) {
- case (HAS_NULL | HAS_NONE):
- SET_BIT2(pColData->pBitMap, iVal, v);
- break;
- case (HAS_VALUE | HAS_NONE):
- if (v) {
- SET_BIT2(pColData->pBitMap, iVal, 2);
- } else {
- SET_BIT2(pColData->pBitMap, iVal, 0);
- }
- break;
- case (HAS_VALUE | HAS_NULL):
- SET_BIT2(pColData->pBitMap, iVal, v + 1);
- break;
- default:
- ASSERT(0);
- }
- }
-
- tFree(pBitMap);
+ int32_t szBitMap;
+ if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) {
+ szBitMap = BIT2_SIZE(pColData->nVal);
} else {
- code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap,
- BIT2_SIZE(pColData->nVal), ppBuf);
- if (code) goto _exit;
+ szBitMap = BIT1_SIZE(pColData->nVal);
}
+
+ code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, szBitMap, ppBuf);
+ if (code) goto _exit;
}
p += pBlockCol->szBitmap;
diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c
index f379d9f71afdf62089fb3f864ff2d0ac376d6997..5adb2eb3592e7bcba7803b5e2972f0bdd3689adb 100644
--- a/source/dnode/vnode/src/vnd/vnodeCfg.c
+++ b/source/dnode/vnode/src/vnd/vnodeCfg.c
@@ -13,8 +13,8 @@
* along with this program. If not, see
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free
+ * Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
+#include
+#include
+#include
+#include
+#include "index.h"
+#include "indexCache.h"
+#include "indexFst.h"
+#include "indexFstUtil.h"
+#include "indexInt.h"
+#include "indexTfile.h"
+#include "indexUtil.h"
+#include "tskiplist.h"
+#include "tutil.h"
+using namespace std;
+
+static std::string logDir = TD_TMP_DIR_PATH "log";
+
+static void initLog() {
+ const char *defaultLogFileNamePrefix = "taoslog";
+ const int32_t maxLogFileNum = 10;
+
+ tsAsyncLog = 0;
+ idxDebugFlag = 143;
+ strcpy(tsLogDir, logDir.c_str());
+ taosRemoveDir(tsLogDir);
+ taosMkDir(tsLogDir);
+
+ if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+ printf("failed to open log file in directory:%s\n", tsLogDir);
+ }
+}
+
+struct WriteBatch {
+ SIndexMultiTerm *terms;
+};
+class Idx {
+ public:
+ Idx(int _cacheSize = 1024 * 1024 * 4, const char *_path = "tindex") {
+ opts.cacheSize = _cacheSize;
+ path += TD_TMP_DIR_PATH;
+ path += _path;
+ }
+ int SetUp(bool remove) {
+ initLog();
+
+ if (remove) taosRemoveDir(path.c_str());
+
+ int ret = indexJsonOpen(&opts, path.c_str(), &index);
+ return ret;
+ }
+ int Write(WriteBatch *batch, uint64_t uid) {
+ // write batch
+ indexJsonPut(index, batch->terms, uid);
+ return 0;
+ }
+ int Read(const char *json, void *key, int64_t *id) {
+ // read batch
+ return 0;
+ }
+
+ void TearDown() { indexJsonClose(index); }
+
+ std::string path;
+
+ SIndexOpts opts;
+ SIndex *index;
+};
+
+SIndexTerm *indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName,
+ int32_t nColName, const char *colVal, int32_t nColVal) {
+ char buf[256] = {0};
+ int16_t sz = nColVal;
+ memcpy(buf, (uint16_t *)&sz, 2);
+ memcpy(buf + 2, colVal, nColVal);
+ if (colType == TSDB_DATA_TYPE_BINARY) {
+ return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf));
+ } else {
+ return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal);
+ }
+ return NULL;
+}
+int initWriteBatch(WriteBatch *wb, int batchSize) {
+ SIndexMultiTerm *terms = indexMultiTermCreate();
+
+ std::string colName;
+ std::string colVal;
+
+ for (int i = 0; i < 64; i++) {
+ colName += '0' + i;
+ colVal += '0' + i;
+ }
+
+ for (int i = 0; i < batchSize; i++) {
+ colVal[i % colVal.size()] = '0' + i % 128;
+ colName[i % colName.size()] = '0' + i % 128;
+ SIndexTerm *term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(),
+ colVal.c_str(), colVal.size());
+ indexMultiTermAdd(terms, term);
+ }
+
+ wb->terms = terms;
+ return 0;
+}
+
+int BenchWrite(Idx *idx, int batchSize, int limit) {
+ for (int i = 0; i < limit; i += batchSize) {
+ WriteBatch wb;
+ idx->Write(&wb, i);
+ }
+ return 0;
+}
+
+int BenchRead(Idx *idx) { return 0; }
+
+int main() {
+ // Idx *idx = new Idx;
+ // if (idx->SetUp(true) != 0) {
+ // std::cout << "failed to setup index" << std::endl;
+ // return 0;
+ // } else {
+ // std::cout << "succ to setup index" << std::endl;
+ // }
+ // BenchWrite(idx, 100, 10000);
+ return 1;
+}
diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc
index 5b76de2ef89fdce4780fcf94a1360d68f7684a9e..08bf84ff60fdc07393abf546630c67dd52f6abc1 100644
--- a/source/libs/index/test/indexTests.cc
+++ b/source/libs/index/test/indexTests.cc
@@ -271,20 +271,20 @@ void validateFst() {
}
delete m;
}
-static std::string logDir = TD_TMP_DIR_PATH "log";
-
-static void initLog() {
- const char* defaultLogFileNamePrefix = "taoslog";
- const int32_t maxLogFileNum = 10;
- tsAsyncLog = 0;
- idxDebugFlag = 143;
- strcpy(tsLogDir, logDir.c_str());
- taosRemoveDir(tsLogDir);
- taosMkDir(tsLogDir);
-
- if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
- printf("failed to open log file in directory:%s\n", tsLogDir);
+static std::string logDir = TD_TMP_DIR_PATH "log";
+static void initLog() {
+ const char* defaultLogFileNamePrefix = "taoslog";
+ const int32_t maxLogFileNum = 10;
+
+ tsAsyncLog = 0;
+ idxDebugFlag = 143;
+ strcpy(tsLogDir, logDir.c_str());
+ taosRemoveDir(tsLogDir);
+ taosMkDir(tsLogDir);
+
+ if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) {
+ printf("failed to open log file in directory:%s\n", tsLogDir);
}
}
class IndexEnv : public ::testing::Test {
diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c
index eb0b604d37459342a403ab5120c22e2bf5dc4b13..48e21e137f9f7167bebfdfba7f3f281dbd6232b2 100644
--- a/source/libs/nodes/src/nodesCloneFuncs.c
+++ b/source/libs/nodes/src/nodesCloneFuncs.c
@@ -399,6 +399,8 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi
COPY_SCALAR_FIELD(modifyType);
COPY_SCALAR_FIELD(msgType);
CLONE_NODE_FIELD(pAffectedRows);
+ CLONE_NODE_FIELD(pStartTs);
+ CLONE_NODE_FIELD(pEndTs);
COPY_SCALAR_FIELD(tableId);
COPY_SCALAR_FIELD(stableId);
COPY_SCALAR_FIELD(tableType);
diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c
index 8976daadbdada2365dbb2e3bf2af4f807c4ac26f..afd05eff1d1c6a2c895581b4be0f76e75e9ca631 100644
--- a/source/libs/nodes/src/nodesCodeFuncs.c
+++ b/source/libs/nodes/src/nodesCodeFuncs.c
@@ -362,18 +362,14 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code);
- ;
}
return code;
@@ -406,14 +402,11 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name);
@@ -466,26 +459,20 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo);
@@ -926,7 +913,6 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToLogicPlanNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs);
@@ -2445,6 +2431,8 @@ static const char* jkDeletePhysiPlanTsColName = "TsColName";
static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey";
static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey";
static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows";
+static const char* jkDeletePhysiPlanStartTs = "StartTs";
+static const char* jkDeletePhysiPlanEndTs = "EndTs";
static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj;
@@ -2471,6 +2459,12 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) {
if (TSDB_CODE_SUCCESS == code) {
code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonAddObject(pJson, jkDeletePhysiPlanStartTs, nodeToJson, pNode->pStartTs);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = tjsonAddObject(pJson, jkDeletePhysiPlanEndTs, nodeToJson, pNode->pEndTs);
+ }
return code;
}
@@ -2500,6 +2494,12 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) {
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows);
}
+ if (TSDB_CODE_SUCCESS == code) {
+ code = jsonToNodeObject(pJson, jkDeletePhysiPlanStartTs, &pNode->pStartTs);
+ }
+ if (TSDB_CODE_SUCCESS == code) {
+ code = jsonToNodeObject(pJson, jkDeletePhysiPlanEndTs, &pNode->pEndTs);
+ }
return code;
}
@@ -2815,7 +2815,6 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) {
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName);
@@ -3118,7 +3117,6 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft);
@@ -3153,7 +3151,6 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToExprNode(pJson, pObj);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList);
@@ -3442,11 +3439,9 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) {
int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr);
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code);
- ;
}
if (TSDB_CODE_SUCCESS == code) {
tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code);
- ;
}
return code;
@@ -3624,7 +3619,6 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) {
int32_t code;
tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code);
- ;
if (TSDB_CODE_SUCCESS == code) {
code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues);
}
diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c
new file mode 100644
index 0000000000000000000000000000000000000000..ade5be472244a13e5ff2eb293ff6561351dbc4f4
--- /dev/null
+++ b/source/libs/nodes/src/nodesMsgFuncs.c
@@ -0,0 +1,3641 @@
+/*
+ * Copyright (c) 2019 TAOS Data, Inc.
+ *
+ * This program is free software: you can use, redistribute, and/or modify
+ * it under the terms of the GNU Affero General Public License, version 3
+ * or later ("AGPL"), as published by the Free Software Foundation.
+ *
+ * This program is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program. If not, see
#include
+#include
#include "cmdnodes.h"
#include "mockCatalogService.h"
@@ -251,6 +252,7 @@ class PlannerTestBaseImpl {
string splitLogicPlan_;
string scaledLogicPlan_;
string physiPlan_;
+ string physiPlanMsg_;
vector physiSubplans_;
};
@@ -274,6 +276,7 @@ class PlannerTestBaseImpl {
res_.splitLogicPlan_.clear();
res_.scaledLogicPlan_.clear();
res_.physiPlan_.clear();
+ res_.physiPlanMsg_.clear();
res_.physiSubplans_.clear();
}
@@ -408,6 +411,8 @@ class PlannerTestBaseImpl {
SNode* pSubplan;
FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); }
}
+ res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan));
+ cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl;
}
void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) {
@@ -446,12 +451,50 @@ class PlannerTestBaseImpl {
string toString(const SNode* pRoot) {
char* pStr = NULL;
int32_t len = 0;
+
+ auto start = chrono::steady_clock::now();
DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len)
+ if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) {
+ cout << "nodesNodeToString: "
+ << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl;
+ }
+
string str(pStr);
taosMemoryFreeClear(pStr);
return str;
}
+ string toMsg(const SNode* pRoot) {
+ char* pStr = NULL;
+ int32_t len = 0;
+
+ auto start = chrono::steady_clock::now();
+ DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len)
+ cout << "nodesNodeToMsg: "
+ << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl;
+
+ SNode* pNode = NULL;
+ char* pNewStr = NULL;
+ int32_t newlen = 0;
+ DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode)
+ DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen)
+ if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) {
+ cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl;
+ taosMemoryFreeClear(pNewStr);
+ DO_WITH_THROW(nodesNodeToString, pRoot, false, &pNewStr, &newlen)
+ cout << "orac node: " << pNewStr << endl;
+ taosMemoryFreeClear(pNewStr);
+ DO_WITH_THROW(nodesNodeToString, pNode, false, &pNewStr, &newlen)
+ cout << "new node: " << pNewStr << endl;
+ }
+ nodesDestroyNode(pNode);
+ taosMemoryFreeClear(pNewStr);
+
+ string str(pStr, len);
+ taosMemoryFreeClear(pStr);
+ return str;
+ }
+
caseEnv caseEnv_;
stmtEnv stmtEnv_;
stmtRes res_;
diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c
index d848016e46482614972d5e85469e4297136d6cc0..8162b922cec8fd779907df851bc1d8b545cafc28 100644
--- a/source/libs/qcom/src/queryUtil.c
+++ b/source/libs/qcom/src/queryUtil.c
@@ -134,8 +134,7 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code)
schedMsg.thandle = execParam;
schedMsg.msg = code;
- taosScheduleTask(&pTaskQueue, &schedMsg);
- return 0;
+ return taosScheduleTask(&pTaskQueue, &schedMsg);
}
void destroySendMsgInfo(SMsgSendInfo* pMsgBody) {
@@ -472,5 +471,3 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) {
return TSDB_CODE_SUCCESS;
}
-
-
diff --git a/source/libs/qworker/src/qwUtil.c b/source/libs/qworker/src/qwUtil.c
index c7bf7ab7e7e6fbe30d2a8ae9abd2c51d51dac590..581eea4a81f90c01333233253d99d8c21f8e1a64 100644
--- a/source/libs/qworker/src/qwUtil.c
+++ b/source/libs/qworker/src/qwUtil.c
@@ -275,6 +275,7 @@ void qwFreeTaskHandle(qTaskInfo_t *taskHandle) {
qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle);
if (otaskHandle && atomic_val_compare_exchange_ptr(taskHandle, otaskHandle, NULL)) {
qDestroyTask(otaskHandle);
+ qDebug("task handle destryed");
}
}
@@ -305,6 +306,7 @@ void qwFreeTaskCtx(SQWTaskCtx *ctx) {
if (ctx->sinkHandle) {
dsDestroyDataSinker(ctx->sinkHandle);
ctx->sinkHandle = NULL;
+ qDebug("sink handle destryed");
}
}
@@ -452,6 +454,10 @@ void qwDestroySchStatus(SQWSchStatus *pStatus) { taosHashCleanup(pStatus->tasksH
void qwDestroyImpl(void *pMgmt) {
SQWorker *mgmt = (SQWorker *)pMgmt;
+ int8_t nodeType = mgmt->nodeType;
+ int32_t nodeId = mgmt->nodeId;
+
+ qDebug("start to destroy qworker, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt);
taosTmrStop(mgmt->hbTimer);
mgmt->hbTimer = NULL;
@@ -484,6 +490,8 @@ void qwDestroyImpl(void *pMgmt) {
atomic_sub_fetch_32(&gQwMgmt.qwNum, 1);
qwCloseRef();
+
+ qDebug("qworker destroyed, type:%d, id:%d, handle:%p", nodeType, nodeId, mgmt);
}
int32_t qwOpenRef(void) {
diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c
index f006096ce20a45e18a5b9d990c9c63b621638ac5..4dcc06b4706b6c85b626bb03877c578e3fe2f8fe 100644
--- a/source/libs/qworker/src/qworker.c
+++ b/source/libs/qworker/src/qworker.c
@@ -9,6 +9,7 @@
#include "tcommon.h"
#include "tmsg.h"
#include "tname.h"
+#include "tdatablock.h"
SQWorkerMgmt gQwMgmt = {
.lock = 0,
@@ -16,6 +17,11 @@ SQWorkerMgmt gQwMgmt = {
.qwNum = 0,
};
+static void freeBlock(void* param) {
+ SSDataBlock* pBlock = *(SSDataBlock**)param;
+ blockDataDestroy(pBlock);
+}
+
int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) {
int32_t code = 0;
SSchedulerHbRsp rsp = {0};
@@ -88,6 +94,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
// if *taskHandle is NULL, it's killed right now
if (taskHandle) {
qwDbgSimulateSleep();
+
code = qExecTaskOpt(taskHandle, pResList, &useconds);
if (code) {
if (code != TSDB_CODE_OPS_NOT_SUPPORT) {
@@ -150,8 +157,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) {
}
_return:
-
- taosArrayDestroy(pResList);
+ taosArrayDestroyEx(pResList, freeBlock);
QW_RET(code);
}
@@ -559,7 +565,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) {
// QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg);
- code = qStringToSubplan(qwMsg->msg, &plan);
+ code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan);
if (TSDB_CODE_SUCCESS != code) {
code = TSDB_CODE_INVALID_MSG;
QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code));
@@ -915,13 +921,13 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) {
void *pIter = taosHashIterate(mgmt->schHash, NULL);
while (pIter) {
- SQWSchStatus *sch = (SQWSchStatus *)pIter;
- if (NULL == sch->hbConnInfo.handle) {
+ SQWSchStatus *sch1 = (SQWSchStatus *)pIter;
+ if (NULL == sch1->hbConnInfo.handle) {
uint64_t *sId = taosHashGetKey(pIter, NULL);
QW_TLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId);
- if (sch->hbBrokenTs > 0 && ((currentMs - sch->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) &&
- taosHashGetSize(sch->tasksHash) <= 0) {
+ if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) &&
+ taosHashGetSize(sch1->tasksHash) <= 0) {
taosArrayPush(pExpiredSch, sId);
}
@@ -968,7 +974,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) {
DataSinkHandle sinkHandle = NULL;
SQWTaskCtx ctx = {0};
- code = qStringToSubplan(qwMsg->msg, &plan);
+ code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan);
if (TSDB_CODE_SUCCESS != code) {
code = TSDB_CODE_INVALID_MSG;
QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code));
diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h
index 957fd46ba5a767858a3bb5bbe50142b4f1c1ce47..7fea2867323f9d5f119740d94a80da4f1a0b5b51 100644
--- a/source/libs/scheduler/inc/schInt.h
+++ b/source/libs/scheduler/inc/schInt.h
@@ -283,8 +283,9 @@ typedef struct SSchJob {
} SSchJob;
typedef struct SSchTaskCtx {
- int64_t jobRid;
+ int64_t jobRid;
SSchTask *pTask;
+ bool asyncLaunch;
} SSchTaskCtx;
extern SSchedulerMgmt schMgmt;
diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c
index 5a64aaaebb3860d2c6729ac8eb1e00be0cc9cda1..c6dc57d752aba9e0bc40fb208584443f2edf57f2 100644
--- a/source/libs/scheduler/src/schRemote.c
+++ b/source/libs/scheduler/src/schRemote.c
@@ -384,8 +384,7 @@ _return:
taosMemoryFreeClear(msg);
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
-}
-
+}
int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
int32_t code = 0;
SSchTaskCallbackParam *pParam = (SSchTaskCallbackParam *)param;
@@ -396,7 +395,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) {
tstrerror(rspCode));
SCH_ERR_JRET(schProcessOnCbBegin(&pJob, &pTask, pParam->queryId, pParam->refId, pParam->taskId));
-
+
code = schHandleResponseMsg(pJob, pTask, pParam->execId, pMsg, rspCode);
pMsg->pData = NULL;
diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c
index 9cab39c30122072207daa9e9639ab92645fc1633..8b3a2016b80bb1c0f96bc312f2043f370f5b7c54 100644
--- a/source/libs/scheduler/src/schTask.c
+++ b/source/libs/scheduler/src/schTask.c
@@ -52,7 +52,7 @@ void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) {
int32_t nodeNum = taosArrayGetSize(pJob->nodeList);
pTask->maxRetryTimes = TMAX(nodeNum, SCH_DEFAULT_MAX_RETRY_NUM);
}
-
+
pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1);
}
@@ -138,14 +138,10 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3
return TSDB_CODE_SUCCESS;
}
- if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it
- SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
- return TSDB_CODE_SUCCESS;
- }
-
SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId));
if (NULL == nodeInfo) { // ignore it
- SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+ SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId,
+ pTask->execId, pTask->waitRetry);
return TSDB_CODE_SUCCESS;
}
@@ -160,11 +156,16 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v
if (dropExecNode) {
SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId));
}
+
+ schUpdateTaskExecNode(pJob, pTask, handle, execId);
+ if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it
+ SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry);
+ SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR);
+ }
+
SCH_SET_TASK_HANDLE(pTask, handle);
- schUpdateTaskExecNode(pJob, pTask, handle, execId);
-
return TSDB_CODE_SUCCESS;
}
@@ -314,7 +315,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) {
if (!schMgmt.cfg.enableReSchedule) {
return TSDB_CODE_SUCCESS;
}
-
+
if (SCH_IS_DATA_BIND_TASK(pTask)) {
return TSDB_CODE_SUCCESS;
}
@@ -341,7 +342,8 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
}
if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) {
- SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, pTask->maxExecTimes, pTask->execId);
+ SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes,
+ pTask->maxExecTimes, pTask->execId);
schHandleJobFailure(pJob, rspCode);
return TSDB_CODE_SUCCESS;
}
@@ -349,7 +351,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
pTask->waitRetry = true;
schDropTaskOnExecNode(pJob, pTask);
taosHashClear(pTask->execNodes);
- SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask));
+ schRemoveTaskFromExecList(pJob, pTask);
schDeregisterTaskHb(pJob, pTask);
atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
taosMemoryFreeClear(pTask->msg);
@@ -427,12 +429,14 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32
code = schDoTaskRedirect(pJob, pTask, pData, rspCode);
taosMemoryFree(pData->pData);
+ taosMemoryFree(pData->pEpSet);
SCH_RET(code);
_return:
taosMemoryFree(pData->pData);
+ taosMemoryFree(pData->pEpSet);
SCH_RET(schProcessOnTaskFailure(pJob, pTask, code));
}
@@ -548,7 +552,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
if ((pTask->retryTimes + 1) > pTask->maxRetryTimes) {
*needRetry = false;
- SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes, pTask->maxRetryTimes);
+ SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes,
+ pTask->maxRetryTimes);
return TSDB_CODE_SUCCESS;
}
@@ -564,25 +569,25 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
return TSDB_CODE_SUCCESS;
}
-/*
- if (SCH_IS_DATA_BIND_TASK(pTask)) {
- if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
- *needRetry = false;
- SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
- SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
- return TSDB_CODE_SUCCESS;
- }
- } else {
- int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
+ /*
+ if (SCH_IS_DATA_BIND_TASK(pTask)) {
+ if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId,
+ SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode));
+ return TSDB_CODE_SUCCESS;
+ }
+ } else {
+ int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs);
- if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
- *needRetry = false;
- SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
- pTask->candidateIdx, candidateNum);
- return TSDB_CODE_SUCCESS;
+ if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) {
+ *needRetry = false;
+ SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d",
+ pTask->candidateIdx, candidateNum);
+ return TSDB_CODE_SUCCESS;
+ }
}
- }
-*/
+ */
*needRetry = true;
SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->execId + 1, errCode, tstrerror(errCode));
@@ -593,7 +598,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo
int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) {
atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1);
- SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask));
+ schRemoveTaskFromExecList(pJob, pTask);
SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT);
if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) {
@@ -630,8 +635,9 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) {
SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY);
}
- SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, naddr->epSet.inUse, naddr->epSet.numOfEps,
- SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port);
+ SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId,
+ naddr->epSet.inUse, naddr->epSet.numOfEps, SCH_GET_CUR_EP(naddr)->fqdn,
+ SCH_GET_CUR_EP(naddr)->port);
++addNum;
}
@@ -711,10 +717,10 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
if (candidateNum <= 1) {
goto _return;
}
-
+
switch (schMgmt.cfg.schPolicy) {
case SCH_LOAD_SEQ:
- case SCH_ALL:
+ case SCH_ALL:
default:
if (++pTask->candidateIdx >= candidateNum) {
pTask->candidateIdx = 0;
@@ -732,15 +738,14 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) {
_return:
SCH_TASK_DLOG("switch task candiateIdx to %d/%d", pTask->candidateIdx, candidateNum);
-
+
return TSDB_CODE_SUCCESS;
}
int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) {
int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId));
if (code) {
- SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code);
- SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR);
+ SCH_TASK_WLOG("task already not in execTask list, code:%x", code);
}
return TSDB_CODE_SUCCESS;
@@ -759,7 +764,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) {
return;
}
- int32_t i = 0;
+ int32_t i = 0;
SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL);
while (nodeInfo) {
if (nodeInfo->handle) {
@@ -821,16 +826,21 @@ int32_t schProcessOnTaskStatusRsp(SQueryNodeEpId *pEpId, SArray *pStatusList) {
int32_t schLaunchTaskImpl(void *param) {
SSchTaskCtx *pCtx = (SSchTaskCtx *)param;
- SSchJob *pJob = schAcquireJob(pCtx->jobRid);
+ SSchJob *pJob = schAcquireJob(pCtx->jobRid);
if (NULL == pJob) {
- taosMemoryFree(param);
qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid);
+ taosMemoryFree(param);
SCH_RET(TSDB_CODE_SCH_JOB_IS_DROPPING);
}
-
+
SSchTask *pTask = pCtx->pTask;
- int8_t status = 0;
- int32_t code = 0;
+
+ if (pCtx->asyncLaunch) {
+ SCH_LOCK_TASK(pTask);
+ }
+
+ int8_t status = 0;
+ int32_t code = 0;
atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1);
pTask->execId++;
@@ -855,7 +865,7 @@ int32_t schLaunchTaskImpl(void *param) {
SSubplan *plan = pTask->plan;
if (NULL == pTask->msg) { // TODO add more detailed reason for failure
- code = qSubPlanToString(plan, &pTask->msg, &pTask->msgLen);
+ code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen);
if (TSDB_CODE_SUCCESS != code) {
SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg,
pTask->msgLen);
@@ -875,8 +885,6 @@ int32_t schLaunchTaskImpl(void *param) {
_return:
- taosMemoryFree(param);
-
if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {
if (code) {
code = schProcessOnTaskFailure(pJob, pTask, code);
@@ -886,27 +894,33 @@ _return:
}
}
+ if (pCtx->asyncLaunch) {
+ SCH_UNLOCK_TASK(pTask);
+ }
+
schReleaseJob(pJob->refId);
+ taosMemoryFree(param);
+
SCH_RET(code);
}
-int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
-
+int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) {
SSchTaskCtx *param = taosMemoryCalloc(1, sizeof(SSchTaskCtx));
if (NULL == param) {
SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY);
}
-
+
param->jobRid = pJob->refId;
param->pTask = pTask;
if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) {
+ param->asyncLaunch = true;
taosAsyncExec(schLaunchTaskImpl, param, NULL);
} else {
SCH_ERR_RET(schLaunchTaskImpl(param));
}
-
+
return TSDB_CODE_SUCCESS;
}
diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c
index a2a45938e4f0091b0588bcb3ca4ac653045d27f7..7cdb7c0db95cd582fad03174d0fa6927cb1fd668 100644
--- a/source/libs/stream/src/streamDispatch.c
+++ b/source/libs/stream/src/streamDispatch.c
@@ -248,9 +248,12 @@ int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, S
char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId);
SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos;
- // TODO: get hash function by hashMethod
- uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));
+ /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/
+ SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo;
+ uint32_t hashValue =
+ taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix);
taosMemoryFree(ctbName);
+
bool found = false;
// TODO: optimize search
int32_t j;
diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c
index dfd6f012cc4f64d252f75a20f761c6f87fc05b78..6cd5132bb9d9021d5c0fe590224414430d36af40 100644
--- a/source/libs/stream/src/streamState.c
+++ b/source/libs/stream/src/streamState.c
@@ -35,6 +35,10 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask) {
goto _err;
}
+ if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) {
+ goto _err;
+ }
+
if (streamStateBegin(pState) < 0) {
goto _err;
}
@@ -44,8 +48,9 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask) {
return pState;
_err:
- if (pState->pStateDb) tdbTbClose(pState->pStateDb);
- if (pState->db) tdbClose(pState->db);
+ tdbTbClose(pState->pStateDb);
+ tdbTbClose(pState->pFuncStateDb);
+ tdbClose(pState->db);
taosMemoryFree(pState);
return NULL;
}
@@ -53,6 +58,7 @@ _err:
void streamStateClose(SStreamState* pState) {
tdbCommit(pState->db, &pState->txn);
tdbTbClose(pState->pStateDb);
+ tdbTbClose(pState->pFuncStateDb);
tdbClose(pState->db);
taosMemoryFree(pState);
@@ -101,6 +107,17 @@ int32_t streamStateAbort(SStreamState* pState) {
return 0;
}
+int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) {
+ return tdbTbUpsert(pState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->txn);
+}
+int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) {
+ return tdbTbGet(pState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen);
+}
+
+int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) {
+ return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn);
+}
+
int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) {
return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn);
}
@@ -112,6 +129,29 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key) {
return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn);
}
+int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) {
+ // todo refactor
+ int32_t size = *pVLen;
+ if (streamStateGet(pState, key, pVal, pVLen) == 0) {
+ return 0;
+ }
+ void* tmp = taosMemoryCalloc(1, size);
+ if (streamStatePut(pState, key, &tmp, size) == 0) {
+ taosMemoryFree(tmp);
+ int32_t code = streamStateGet(pState, key, pVal, pVLen);
+ ASSERT(code == 0);
+ return code;
+ }
+ taosMemoryFree(tmp);
+ return -1;
+}
+
+int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) {
+ // todo refactor
+ streamFreeVal(pVal);
+ return 0;
+}
+
SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) {
SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur));
if (pCur == NULL) return NULL;
diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c
index 51098374b03531142c9c12443fa5b02efddc3aca..6f29b54f806f1113ec69dface7bdcbb4b0c42afc 100644
--- a/source/libs/sync/src/syncMain.c
+++ b/source/libs/sync/src/syncMain.c
@@ -2766,8 +2766,6 @@ const char* syncStr(ESyncState state) {
}
int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) {
- SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg);
-
if (ths->state != TAOS_SYNC_STATE_FOLLOWER) {
syncNodeEventLog(ths, "I am not follower, can not do leader transfer");
return 0;
@@ -2799,6 +2797,8 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p
}
*/
+ SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg);
+
do {
char logBuf[128];
snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index);
diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c
index b42aba560fa1c26ef9426b55729c1d39cafa8a24..faebe5bbecb16012831ed103de520c14accc81d5 100644
--- a/source/libs/sync/src/syncMessage.c
+++ b/source/libs/sync/src/syncMessage.c
@@ -1992,6 +1992,313 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) {
}
}
+// ---- message process SyncHeartbeat----
+SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) {
+ uint32_t bytes = sizeof(SyncHeartbeat);
+ SyncHeartbeat* pMsg = taosMemoryMalloc(bytes);
+ memset(pMsg, 0, bytes);
+ pMsg->bytes = bytes;
+ pMsg->vgId = vgId;
+ pMsg->msgType = TDMT_SYNC_HEARTBEAT;
+ return pMsg;
+}
+
+void syncHeartbeatDestroy(SyncHeartbeat* pMsg) {
+ if (pMsg != NULL) {
+ taosMemoryFree(pMsg);
+ }
+}
+
+void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen) {
+ ASSERT(pMsg->bytes <= bufLen);
+ memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg) {
+ memcpy(pMsg, buf, len);
+ ASSERT(len == pMsg->bytes);
+}
+
+char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len) {
+ char* buf = taosMemoryMalloc(pMsg->bytes);
+ ASSERT(buf != NULL);
+ syncHeartbeatSerialize(pMsg, buf, pMsg->bytes);
+ if (len != NULL) {
+ *len = pMsg->bytes;
+ }
+ return buf;
+}
+
+SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len) {
+ uint32_t bytes = *((uint32_t*)buf);
+ SyncHeartbeat* pMsg = taosMemoryMalloc(bytes);
+ ASSERT(pMsg != NULL);
+ syncHeartbeatDeserialize(buf, len, pMsg);
+ ASSERT(len == pMsg->bytes);
+ return pMsg;
+}
+
+void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg) {
+ memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+ pRpcMsg->msgType = pMsg->msgType;
+ pRpcMsg->contLen = pMsg->bytes;
+ pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+ syncHeartbeatSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg) {
+ syncHeartbeatDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+ SyncHeartbeat* pMsg = syncHeartbeatDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+ ASSERT(pMsg != NULL);
+ return pMsg;
+}
+
+cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg) {
+ char u64buf[128] = {0};
+ cJSON* pRoot = cJSON_CreateObject();
+
+ if (pMsg != NULL) {
+ cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+ cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+ cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+ cJSON* pSrcId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+ cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->srcId.addr;
+ cJSON* pTmp = pSrcId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+ cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+ cJSON* pDestId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+ cJSON_AddStringToObject(pDestId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->destId.addr;
+ cJSON* pTmp = pDestId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+ cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+ cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+ cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex);
+ cJSON_AddStringToObject(pRoot, "commitIndex", u64buf);
+ }
+
+ cJSON* pJson = cJSON_CreateObject();
+ cJSON_AddItemToObject(pJson, "SyncHeartbeat", pRoot);
+ return pJson;
+}
+
+char* syncHeartbeat2Str(const SyncHeartbeat* pMsg) {
+ cJSON* pJson = syncHeartbeat2Json(pMsg);
+ char* serialized = cJSON_Print(pJson);
+ cJSON_Delete(pJson);
+ return serialized;
+}
+
+void syncHeartbeatPrint(const SyncHeartbeat* pMsg) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ printf("syncHeartbeatPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ printf("syncHeartbeatPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatLog(const SyncHeartbeat* pMsg) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ sTrace("syncHeartbeatLog | len:%" PRIu64 " | %s", strlen(serialized), serialized);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) {
+ if (gRaftDetailLog) {
+ char* serialized = syncHeartbeat2Str(pMsg);
+ sTrace("syncHeartbeatLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized);
+ taosMemoryFree(serialized);
+ }
+}
+
+// ---- message process SyncHeartbeatReply----
+SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId) {
+ uint32_t bytes = sizeof(SyncHeartbeatReply);
+ SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes);
+ memset(pMsg, 0, bytes);
+ pMsg->bytes = bytes;
+ pMsg->vgId = vgId;
+ pMsg->msgType = TDMT_SYNC_HEARTBEAT_REPLY;
+ return pMsg;
+}
+
+void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg) {
+ if (pMsg != NULL) {
+ taosMemoryFree(pMsg);
+ }
+}
+
+void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen) {
+ ASSERT(pMsg->bytes <= bufLen);
+ memcpy(buf, pMsg, pMsg->bytes);
+}
+
+void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg) {
+ memcpy(pMsg, buf, len);
+ ASSERT(len == pMsg->bytes);
+}
+
+char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len) {
+ char* buf = taosMemoryMalloc(pMsg->bytes);
+ ASSERT(buf != NULL);
+ syncHeartbeatReplySerialize(pMsg, buf, pMsg->bytes);
+ if (len != NULL) {
+ *len = pMsg->bytes;
+ }
+ return buf;
+}
+
+SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len) {
+ uint32_t bytes = *((uint32_t*)buf);
+ SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes);
+ ASSERT(pMsg != NULL);
+ syncHeartbeatReplyDeserialize(buf, len, pMsg);
+ ASSERT(len == pMsg->bytes);
+ return pMsg;
+}
+
+void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg) {
+ memset(pRpcMsg, 0, sizeof(*pRpcMsg));
+ pRpcMsg->msgType = pMsg->msgType;
+ pRpcMsg->contLen = pMsg->bytes;
+ pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen);
+ syncHeartbeatReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen);
+}
+
+void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg) {
+ syncHeartbeatReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg);
+}
+
+SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) {
+ SyncHeartbeatReply* pMsg = syncHeartbeatReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen);
+ ASSERT(pMsg != NULL);
+ return pMsg;
+}
+
+cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg) {
+ char u64buf[128] = {0};
+ cJSON* pRoot = cJSON_CreateObject();
+
+ if (pMsg != NULL) {
+ cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes);
+ cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId);
+ cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType);
+
+ cJSON* pSrcId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr);
+ cJSON_AddStringToObject(pSrcId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->srcId.addr;
+ cJSON* pTmp = pSrcId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId);
+ cJSON_AddItemToObject(pRoot, "srcId", pSrcId);
+
+ cJSON* pDestId = cJSON_CreateObject();
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr);
+ cJSON_AddStringToObject(pDestId, "addr", u64buf);
+ {
+ uint64_t u64 = pMsg->destId.addr;
+ cJSON* pTmp = pDestId;
+ char host[128] = {0};
+ uint16_t port;
+ syncUtilU642Addr(u64, host, sizeof(host), &port);
+ cJSON_AddStringToObject(pTmp, "addr_host", host);
+ cJSON_AddNumberToObject(pTmp, "addr_port", port);
+ }
+ cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId);
+ cJSON_AddItemToObject(pRoot, "destId", pDestId);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm);
+ cJSON_AddStringToObject(pRoot, "privateTerm", u64buf);
+
+ snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term);
+ cJSON_AddStringToObject(pRoot, "term", u64buf);
+
+ cJSON_AddStringToObject(pRoot, "matchIndex", u64buf);
+ snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime);
+ cJSON_AddStringToObject(pRoot, "startTime", u64buf);
+ }
+
+ cJSON* pJson = cJSON_CreateObject();
+ cJSON_AddItemToObject(pJson, "SyncHeartbeatReply", pRoot);
+ return pJson;
+}
+
+char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg) {
+ cJSON* pJson = syncHeartbeatReply2Json(pMsg);
+ char* serialized = cJSON_Print(pJson);
+ cJSON_Delete(pJson);
+ return serialized;
+}
+
+void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ printf("syncHeartbeatReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ printf("syncHeartbeatReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized);
+ fflush(NULL);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ sTrace("syncHeartbeatReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized);
+ taosMemoryFree(serialized);
+}
+
+void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) {
+ if (gRaftDetailLog) {
+ char* serialized = syncHeartbeatReply2Str(pMsg);
+ sTrace("syncHeartbeatReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized);
+ taosMemoryFree(serialized);
+ }
+}
+
// ---- message process SyncApplyMsg----
SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) {
uint32_t bytes = sizeof(SyncApplyMsg) + dataLen;
diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt
index 72845d0c1d1a9378a3a189f4037c6fc646c8a536..b9cc7a391dde35e2569f30000752b3ef175fc824 100644
--- a/source/libs/sync/test/CMakeLists.txt
+++ b/source/libs/sync/test/CMakeLists.txt
@@ -57,6 +57,8 @@ add_executable(syncLeaderTransferTest "")
add_executable(syncReconfigFinishTest "")
add_executable(syncRestoreFromSnapshot "")
add_executable(syncRaftCfgIndexTest "")
+add_executable(syncHeartbeatTest "")
+add_executable(syncHeartbeatReplyTest "")
target_sources(syncTest
@@ -295,6 +297,14 @@ target_sources(syncRaftCfgIndexTest
PRIVATE
"syncRaftCfgIndexTest.cpp"
)
+target_sources(syncHeartbeatTest
+ PRIVATE
+ "syncHeartbeatTest.cpp"
+)
+target_sources(syncHeartbeatReplyTest
+ PRIVATE
+ "syncHeartbeatReplyTest.cpp"
+)
target_include_directories(syncTest
@@ -592,6 +602,16 @@ target_include_directories(syncRaftCfgIndexTest
"${TD_SOURCE_DIR}/include/libs/sync"
"${CMAKE_CURRENT_SOURCE_DIR}/../inc"
)
+target_include_directories(syncHeartbeatTest
+ PUBLIC
+ "${TD_SOURCE_DIR}/include/libs/sync"
+ "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
+target_include_directories(syncHeartbeatReplyTest
+ PUBLIC
+ "${TD_SOURCE_DIR}/include/libs/sync"
+ "${CMAKE_CURRENT_SOURCE_DIR}/../inc"
+)
target_link_libraries(syncTest
@@ -830,6 +850,14 @@ target_link_libraries(syncRaftCfgIndexTest
sync
gtest_main
)
+target_link_libraries(syncHeartbeatTest
+ sync
+ gtest_main
+)
+target_link_libraries(syncHeartbeatReplyTest
+ sync
+ gtest_main
+)
enable_testing()
diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..0ccd7b70bb8f552fa14c0a615993abbab678a8a0
--- /dev/null
+++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp
@@ -0,0 +1,105 @@
+#include
+#include
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+ sTrace("--- sync log test: trace");
+ sDebug("--- sync log test: debug");
+ sInfo("--- sync log test: info");
+ sWarn("--- sync log test: warn");
+ sError("--- sync log test: error");
+ sFatal("--- sync log test: fatal");
+}
+
+SyncHeartbeatReply *createMsg() {
+ SyncHeartbeatReply *pMsg = syncHeartbeatReplyBuild(1000);
+ pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+ pMsg->srcId.vgId = 100;
+ pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+ pMsg->destId.vgId = 100;
+
+ pMsg->term = 33;
+ pMsg->privateTerm = 44;
+ pMsg->startTime = taosGetTimestampMs();
+ return pMsg;
+}
+
+void test1() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ syncHeartbeatReplyLog2((char *)"test1:", pMsg);
+ syncHeartbeatReplyDestroy(pMsg);
+}
+
+void test2() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ uint32_t len = pMsg->bytes;
+ char * serialized = (char *)taosMemoryMalloc(len);
+ syncHeartbeatReplySerialize(pMsg, serialized, len);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000);
+ syncHeartbeatReplyDeserialize(serialized, len, pMsg2);
+ syncHeartbeatReplyLog2((char *)"test2: syncHeartbeatReplySerialize -> syncHeartbeatReplyDeserialize ",
+ pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test3() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ uint32_t len;
+ char * serialized = syncHeartbeatReplySerialize2(pMsg, &len);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyDeserialize2(serialized, len);
+ syncHeartbeatReplyLog2((char *)"test3: syncHeartbeatReplySerialize3 -> syncHeartbeatReplyDeserialize2 ",
+ pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test4() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000);
+ syncHeartbeatReplyFromRpcMsg(&rpcMsg, pMsg2);
+ syncHeartbeatReplyLog2((char *)"test4: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg ",
+ pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+void test5() {
+ SyncHeartbeatReply *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyFromRpcMsg2(&rpcMsg);
+ syncHeartbeatReplyLog2((char *)"test5: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg2 ",
+ pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatReplyDestroy(pMsg);
+ syncHeartbeatReplyDestroy(pMsg2);
+}
+
+int main() {
+ gRaftDetailLog = true;
+
+ tsAsyncLog = 0;
+ sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+ logTest();
+
+ test1();
+ test2();
+ test3();
+ test4();
+ test5();
+
+ return 0;
+}
diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp
new file mode 100644
index 0000000000000000000000000000000000000000..d910c828f1e13f4e6484c7b8ad7ea6758f626750
--- /dev/null
+++ b/source/libs/sync/test/syncHeartbeatTest.cpp
@@ -0,0 +1,99 @@
+#include
+#include
+#include "syncIO.h"
+#include "syncInt.h"
+#include "syncMessage.h"
+#include "syncUtil.h"
+
+void logTest() {
+ sTrace("--- sync log test: trace");
+ sDebug("--- sync log test: debug");
+ sInfo("--- sync log test: info");
+ sWarn("--- sync log test: warn");
+ sError("--- sync log test: error");
+ sFatal("--- sync log test: fatal");
+}
+
+SyncHeartbeat *createMsg() {
+ SyncHeartbeat *pMsg = syncHeartbeatBuild(789);
+ pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234);
+ pMsg->srcId.vgId = 100;
+ pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678);
+ pMsg->destId.vgId = 100;
+ pMsg->term = 8;
+ pMsg->commitIndex = 33;
+ pMsg->privateTerm = 44;
+ return pMsg;
+}
+
+void test1() {
+ SyncHeartbeat *pMsg = createMsg();
+ syncHeartbeatLog2((char *)"test1:", pMsg);
+ syncHeartbeatDestroy(pMsg);
+}
+
+void test2() {
+ SyncHeartbeat *pMsg = createMsg();
+ uint32_t len = pMsg->bytes;
+ char * serialized = (char *)taosMemoryMalloc(len);
+ syncHeartbeatSerialize(pMsg, serialized, len);
+ SyncHeartbeat *pMsg2 = syncHeartbeatBuild(789);
+ syncHeartbeatDeserialize(serialized, len, pMsg2);
+ syncHeartbeatLog2((char *)"test2: syncHeartbeatSerialize -> syncHeartbeatDeserialize ", pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+void test3() {
+ SyncHeartbeat *pMsg = createMsg();
+ uint32_t len;
+ char * serialized = syncHeartbeatSerialize2(pMsg, &len);
+ SyncHeartbeat *pMsg2 = syncHeartbeatDeserialize2(serialized, len);
+ syncHeartbeatLog2((char *)"test3: syncHeartbeatSerialize2 -> syncHeartbeatDeserialize2 ", pMsg2);
+
+ taosMemoryFree(serialized);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+void test4() {
+ SyncHeartbeat *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeat2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeat *pMsg2 = (SyncHeartbeat *)taosMemoryMalloc(rpcMsg.contLen);
+ syncHeartbeatFromRpcMsg(&rpcMsg, pMsg2);
+ syncHeartbeatLog2((char *)"test4: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg ", pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+void test5() {
+ SyncHeartbeat *pMsg = createMsg();
+ SRpcMsg rpcMsg;
+ syncHeartbeat2RpcMsg(pMsg, &rpcMsg);
+ SyncHeartbeat *pMsg2 =syncHeartbeatFromRpcMsg2(&rpcMsg);
+ syncHeartbeatLog2((char *)"test5: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg2 ", pMsg2);
+
+ rpcFreeCont(rpcMsg.pCont);
+ syncHeartbeatDestroy(pMsg);
+ syncHeartbeatDestroy(pMsg2);
+}
+
+int main() {
+ tsAsyncLog = 0;
+ sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE;
+ gRaftDetailLog = true;
+ logTest();
+
+ test1();
+ test2();
+ test3();
+ test4();
+ test5();
+
+ return 0;
+}
diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c
index 1480920f908e34bb7be5b95affe64619ac042289..c6ecd37680c642feb8e961fc4b398dd3f01d0645 100644
--- a/source/libs/tdb/src/db/tdbBtree.c
+++ b/source/libs/tdb/src/db/tdbBtree.c
@@ -489,7 +489,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN
}
// Copy the root page content to the child page
- tdbPageCopy(pRoot, pChild);
+ tdbPageCopy(pRoot, pChild, 0);
// Reinitialize the root page
zArg.flags = TDB_BTREE_ROOT;
@@ -742,7 +742,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
for (int i = 0; i < nOlds; i++) {
tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL);
tdbBtreeInitPage(pOldsCopy[i], &iarg, 0);
- tdbPageCopy(pOlds[i], pOldsCopy[i]);
+ tdbPageCopy(pOlds[i], pOldsCopy[i], 0);
}
iNew = 0;
nNewCells = 0;
@@ -840,7 +840,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx
i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]);
// copy content to the parent page
tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0);
- tdbPageCopy(pNews[0], pParent);
+ tdbPageCopy(pNews[0], pParent, 1);
}
for (int i = 0; i < 3; i++) {
diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c
index a3f376b929291780bdd57cbf99f5db6035e70aff..1e2eefabf42bd97644baf38e19b77595529aab38 100644
--- a/source/libs/tdb/src/db/tdbPage.c
+++ b/source/libs/tdb/src/db/tdbPage.c
@@ -229,7 +229,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) {
return 0;
}
-void tdbPageCopy(SPage *pFromPage, SPage *pToPage) {
+void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) {
int delta, nFree;
pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr);
@@ -250,8 +250,15 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) {
// Copy the overflow cells
for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) {
+ SCell *pNewCell = pFromPage->apOvfl[iOvfl];
+ if (deepCopyOvfl) {
+ int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL);
+ pNewCell = (SCell *)tdbOsMalloc(szCell);
+ memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell);
+ }
+
+ pToPage->apOvfl[iOvfl] = pNewCell;
pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl];
- pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl];
}
pToPage->nOverflow = pFromPage->nOverflow;
}
diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c
index f90c39278888c7838b0c4b1b5b434e4c06fb30a0..2cc62d3d6af466921ef10b9fc5871499d4797467 100644
--- a/source/libs/tdb/src/db/tdbPager.c
+++ b/source/libs/tdb/src/db/tdbPager.c
@@ -34,6 +34,22 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage
static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage);
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage);
+static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) {
+ SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode));
+ SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode));
+
+ SPgno pgnoL = TDB_PAGE_PGNO(pPageL);
+ SPgno pgnoR = TDB_PAGE_PGNO(pPageR);
+
+ if (pgnoL < pgnoR) {
+ return -1;
+ } else if (pgnoL > pgnoR) {
+ return 1;
+ } else {
+ return 0;
+ }
+}
+
int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
uint8_t *pPtr;
SPager *pPager;
@@ -83,6 +99,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) {
ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize));
pPager->dbFileSize = pPager->dbOrigSize;
+ tRBTreeCreate(&pPager->rbt, pageCmpFn);
+
*ppPager = pPager;
return 0;
}
@@ -170,7 +188,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
// Set page as dirty
pPage->isDirty = 1;
-
+ /*
// Add page to dirty list(TODO: NOT use O(n^2) algorithm)
for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage);
ppPage = &((*ppPage)->pDirtyNext)) {
@@ -185,6 +203,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) {
ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage));
pPage->pDirtyNext = *ppPage;
*ppPage = pPage;
+ */
+ tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage);
// Write page to journal if neccessary
if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) {
@@ -228,6 +248,24 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
return 0;
}
+ SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1);
+ SRBTreeNode *pNode = NULL;
+ while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
+ pPage = (SPage *)pNode;
+ ret = tdbPagerWritePageToDB(pPager, pPage);
+ if (ret < 0) {
+ ASSERT(0);
+ return -1;
+ }
+
+ pPage->isDirty = 0;
+
+ // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
+ tdbPCacheRelease(pPager->pCache, pPage, pTxn);
+ }
+
+ tRBTreeCreate(&pPager->rbt, pageCmpFn);
+ /*
// loop to write the dirty pages to file
for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) {
// TODO: update the page footer
@@ -238,9 +276,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
}
}
- tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize);
- pPager->dbOrigSize = pPager->dbFileSize;
-
// release the page
for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
pPager->pDirty = pPage->pDirtyNext;
@@ -250,6 +285,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) {
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
+ */
+ tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize);
+ pPager->dbOrigSize = pPager->dbFileSize;
// sync the db file
tdbOsFSync(pPager->fd);
@@ -308,15 +346,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) {
}
*/
// 3, release the dirty pages
- for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) {
- pPager->pDirty = pPage->pDirtyNext;
- pPage->pDirtyNext = NULL;
+ SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1);
+ SRBTreeNode *pNode = NULL;
+ while ((pNode = tRBTreeIterNext(&iter)) != NULL) {
+ pPage = (SPage *)pNode;
pPage->isDirty = 0;
+ // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage);
tdbPCacheRelease(pPager->pCache, pPage, pTxn);
}
+ tRBTreeCreate(&pPager->rbt, pageCmpFn);
+
// 4, remove the journal file
tdbOsClose(pPager->jfd);
tdbOsRemove(pPager->jFileName);
@@ -497,12 +539,19 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) {
return 0;
}
-
+/*
+struct TdFile {
+ TdThreadRwlock rwlock;
+ int refId;
+ int fd;
+ FILE *fp;
+} TdFile;
+*/
static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
i64 offset;
int ret;
- offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
+ offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1);
if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) {
ASSERT(0);
return -1;
@@ -514,6 +563,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) {
return -1;
}
+ // pwrite(pPager->fd->fd, pPage->pData, pPage->pageSize, offset);
return 0;
}
diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h
index 6a694cf8f1f8cce95e7fa9373e2aa2c01128a6d9..29a9665c156a4514846c5549e7a90d97a8df4984 100644
--- a/source/libs/tdb/src/inc/tdbInt.h
+++ b/source/libs/tdb/src/inc/tdbInt.h
@@ -19,6 +19,7 @@
#include "tdb.h"
#include "tlog.h"
+#include "trbtree.h"
#ifdef __cplusplus
extern "C" {
@@ -256,6 +257,7 @@ typedef struct {
#pragma pack(pop)
struct SPage {
+ SRBTreeNode node; // must be the first field for pageCmpFn to work
tdb_spinlock_t lock;
int pageSize;
u8 *pData;
@@ -280,13 +282,13 @@ struct SPage {
static inline i32 tdbRefPage(SPage *pPage) {
i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1);
- tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef);
+ // tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef);
return nRef;
}
static inline i32 tdbUnrefPage(SPage *pPage) {
i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1);
- tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef);
+ // tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef);
return nRef;
}
@@ -331,7 +333,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell
int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl);
int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt);
int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt);
-void tdbPageCopy(SPage *pFromPage, SPage *pToPage);
+void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells);
int tdbPageCapacity(int pageSize, int amHdrSize);
static inline SCell *tdbPageGetCell(SPage *pPage, int idx) {
@@ -389,6 +391,7 @@ struct SPager {
SPgno dbFileSize;
SPgno dbOrigSize;
SPage *pDirty;
+ SRBTree rbt;
u8 inTran;
SPager *pNext; // used by TDB
SPager *pHashNext; // used by TDB
diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c
index 62aec219df57f1a8edd11456ca631e315d553803..4600e5e568fc5644024c633e08328a54247b28b5 100644
--- a/source/libs/tfs/src/tfs.c
+++ b/source/libs/tfs/src/tfs.c
@@ -113,6 +113,8 @@ SDiskSize tfsGetSize(STfs *pTfs) {
return size;
}
+int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; }
+
int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) {
pDiskId->level = expLevel;
pDiskId->id = -1;
diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c
index 275e7b42cc10ed2a77934da84ed544a1874e0b28..1559c85e23d59fec376890433f924522df8dc761 100644
--- a/source/libs/transport/src/thttp.c
+++ b/source/libs/transport/src/thttp.c
@@ -126,22 +126,22 @@ _OVER:
return code;
}
-static void destroyHttpClient(SHttpClient* cli) {
+static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) {
taosMemoryFree(cli->wbuf);
taosMemoryFree(cli->rbuf);
taosMemoryFree(cli->addr);
taosMemoryFree(cli);
}
-static void clientCloseCb(uv_handle_t* handle) {
+static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) {
SHttpClient* cli = handle->data;
destroyHttpClient(cli);
}
-static void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
+static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) {
SHttpClient* cli = handle->data;
buf->base = cli->rbuf;
buf->len = HTTP_RECV_BUF_SIZE;
}
-static void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
+static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) {
SHttpClient* cli = handle->data;
if (nread < 0) {
uError("http-report recv error:%s", uv_err_name(nread));
@@ -173,7 +173,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) {
uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb);
}
-static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
+static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) {
uint32_t ip = taosGetIpv4FromFqdn(server);
if (ip == 0xffffffff) {
terrno = TAOS_SYSTEM_ERROR(errno);
diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c
index 4b5441f738d303fd8286e4cf856d1feacfee3eb5..191011111d289f2d261f3ac21299c1e2a4493836 100644
--- a/source/libs/transport/src/transCli.c
+++ b/source/libs/transport/src/transCli.c
@@ -69,11 +69,9 @@ typedef struct SCliThrd {
SAsyncPool* asyncPool;
uv_prepare_t* prepare;
void* pool; // conn pool
-
+ // timer handles
SArray* timerList;
-
// msg queue
-
queue msg;
TdThreadMutex msgMtx;
SDelayQueue* delayQueue;
@@ -108,7 +106,7 @@ static void cliReadTimeoutCb(uv_timer_t* handle);
// register timer in each thread to clear expire conn
// static void cliTimeoutCb(uv_timer_t* handle);
// alloc buffer for recv
-static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
+static FORCE_INLINE void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf);
// callback after recv nbytes from socket
static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf);
// callback after send data to socket
@@ -132,10 +130,10 @@ static void cliSend(SCliConn* pConn);
static void cliDestroyConnMsgs(SCliConn* conn, bool destroy);
// cli util func
-static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx);
-static void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
+static FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx);
+static FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr);
-static int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp);
+static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp);
// process data read from server, add decompress etc later
static void cliHandleResp(SCliConn* conn);
@@ -150,13 +148,11 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd);
static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL,
cliHandleUpdate};
-static void cliSendQuit(SCliThrd* thrd);
-static void destroyUserdata(STransMsg* userdata);
+static FORCE_INLINE void destroyUserdata(STransMsg* userdata);
+static FORCE_INLINE void destroyCmsg(void* cmsg);
+static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst);
+static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx);
-static int cliRBChoseIdx(STrans* pTransInst);
-
-static void destroyCmsg(void* cmsg);
-static void transDestroyConnCtx(STransConnCtx* ctx);
// thread obj
static SCliThrd* createThrdObj();
static void destroyThrdObj(SCliThrd* pThrd);
@@ -166,7 +162,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg);
static void cliReleaseUnfinishedMsg(SCliConn* conn) {
for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) {
SCliMsg* msg = transQueueGet(&conn->cliMsgs, i);
- if (msg != NULL && msg->ctx != NULL) {
+ if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) {
if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) {
conn->ctx.freeFunc(msg->ctx->ahandle);
}
@@ -200,22 +196,22 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) {
#define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL)
#define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label)
-#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \
- do { \
- int i = 0, sz = transQueueSize(&conn->cliMsgs); \
- for (; i < sz; i++) { \
- pMsg = transQueueGet(&conn->cliMsgs, i); \
- if (pMsg != NULL && pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \
- break; \
- } \
- } \
- if (i == sz) { \
- pMsg = NULL; \
- tDebug("msg not found, %" PRIu64 "", ahandle); \
- } else { \
- pMsg = transQueueRm(&conn->cliMsgs, i); \
- tDebug("msg found, %" PRIu64 "", ahandle); \
- } \
+#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \
+ do { \
+ int i = 0, sz = transQueueSize(&conn->cliMsgs); \
+ for (; i < sz; i++) { \
+ pMsg = transQueueGet(&conn->cliMsgs, i); \
+ if (pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \
+ break; \
+ } \
+ } \
+ if (i == sz) { \
+ pMsg = NULL; \
+ tDebug("msg not found, %" PRIu64 "", ahandle); \
+ } else { \
+ pMsg = transQueueRm(&conn->cliMsgs, i); \
+ tDebug("msg found, %" PRIu64 "", ahandle); \
+ } \
} while (0)
#define CONN_GET_NEXT_SENDMSG(conn) \
do { \
@@ -293,7 +289,12 @@ bool cliMaySendCachedMsg(SCliConn* conn) {
if (!transQueueEmpty(&conn->cliMsgs)) {
SCliMsg* pCliMsg = NULL;
CONN_GET_NEXT_SENDMSG(conn);
- cliSend(conn);
+ if (pCliMsg == NULL)
+ return false;
+ else {
+ cliSend(conn);
+ return true;
+ }
}
return false;
_RETURN:
@@ -373,15 +374,19 @@ void cliHandleResp(SCliConn* conn) {
if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) {
tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
+ transFreeMsg(transMsg.pCont);
return;
}
if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) {
tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn);
+ transFreeMsg(transMsg.pCont);
return;
}
- if (cliAppCb(conn, &transMsg, pMsg) != 0) {
- return;
+ if (pMsg == NULL || (pMsg && pMsg->type != Release)) {
+ if (cliAppCb(conn, &transMsg, pMsg) != 0) {
+ return;
+ }
}
destroyCmsg(pMsg);
@@ -390,7 +395,7 @@ void cliHandleResp(SCliConn* conn) {
}
if (CONN_NO_PERSIST_BY_APP(conn)) {
- addConnToPool(pThrd->pool, conn);
+ return addConnToPool(pThrd->pool, conn);
}
uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb);
@@ -429,17 +434,20 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) {
transMsg.info.ahandle);
}
} else {
- transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL;
+ transMsg.info.ahandle = (pMsg->type != Release && pCtx) ? pCtx->ahandle : NULL;
}
if (pCtx == NULL || pCtx->pSem == NULL) {
if (transMsg.info.ahandle == NULL) {
+ if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg);
once = true;
continue;
}
}
- if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
- return;
+ if (pMsg == NULL || (pMsg && pMsg->type != Release)) {
+ if (cliAppCb(pConn, &transMsg, pMsg) != 0) {
+ return;
+ }
}
destroyCmsg(pMsg);
tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn));
@@ -705,6 +713,9 @@ static bool cliHandleNoResp(SCliConn* conn) {
if (cliMaySendCachedMsg(conn) == false) {
SCliThrd* thrd = conn->hostThrd;
addConnToPool(thrd->pool, conn);
+ res = false;
+ } else {
+ res = true;
}
}
}
@@ -782,7 +793,13 @@ void cliSend(SCliConn* pConn) {
uv_buf_t wb = uv_buf_init((char*)pHead, msgLen);
uv_write_t* req = transReqQueuePush(&pConn->wreqQueue);
- uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
+
+ int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb);
+ if (status != 0) {
+ tGError("%s conn %p failed to sent msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType),
+ uv_err_name(status));
+ cliHandleExcept(pConn);
+ }
return;
_RETURN:
return;
@@ -885,26 +902,23 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) {
}
return conn;
}
-void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
+FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) {
if (pCvtAddr->cvt == false) {
return;
}
- for (int i = 0; i < pEpSet->numOfEps && pEpSet->numOfEps == 1; i++) {
- if (strncmp(pEpSet->eps[i].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) {
- memset(pEpSet->eps[i].fqdn, 0, TSDB_FQDN_LEN);
- memcpy(pEpSet->eps[i].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN);
- }
+ if (pEpSet->numOfEps == 1 && strncmp(pEpSet->eps[0].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) {
+ memset(pEpSet->eps[0].fqdn, 0, TSDB_FQDN_LEN);
+ memcpy(pEpSet->eps[0].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN);
}
}
-bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
+FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) {
if (code != 0) return false;
if (pCtx->retryCnt == 0) return false;
if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false;
return true;
}
-
-int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
+FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) {
if (pMsg == NULL) return -1;
memset(pResp, 0, sizeof(STransMsg));
@@ -934,7 +948,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
// persist conn already release by server
STransMsg resp;
cliBuildExceptResp(pMsg, &resp);
- pTransInst->cfp(pTransInst->parent, &resp, NULL);
+ if (pMsg->type != Release) {
+ pTransInst->cfp(pTransInst->parent, &resp, NULL);
+ }
destroyCmsg(pMsg);
return;
}
@@ -980,6 +996,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) {
return;
}
}
+ STraceId* trace = &pMsg->msg.info.traceId;
+ tGTrace("%s conn %p ready", pTransInst->label, conn);
}
static void cliAsyncCb(uv_async_t* handle) {
SAsyncItem* item = handle->data;
@@ -1128,14 +1146,15 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads,
return cli;
}
-static void destroyUserdata(STransMsg* userdata) {
+static FORCE_INLINE void destroyUserdata(STransMsg* userdata) {
if (userdata->pCont == NULL) {
return;
}
transFreeMsg(userdata->pCont);
userdata->pCont = NULL;
}
-static void destroyCmsg(void* arg) {
+
+static FORCE_INLINE void destroyCmsg(void* arg) {
SCliMsg* pMsg = arg;
if (pMsg == NULL) {
return;
@@ -1201,7 +1220,7 @@ static void destroyThrdObj(SCliThrd* pThrd) {
taosMemoryFree(pThrd);
}
-static void transDestroyConnCtx(STransConnCtx* ctx) {
+static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx) {
//
taosMemoryFree(ctx);
}
@@ -1220,7 +1239,7 @@ void cliWalkCb(uv_handle_t* handle, void* arg) {
}
}
-int cliRBChoseIdx(STrans* pTransInst) {
+FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst) {
int8_t index = pTransInst->index;
if (pTransInst->numOfThreads == 0) {
return -1;
@@ -1230,7 +1249,7 @@ int cliRBChoseIdx(STrans* pTransInst) {
}
return index % pTransInst->numOfThreads;
}
-static void doDelayTask(void* param) {
+static FORCE_INLINE void doDelayTask(void* param) {
STaskArg* arg = param;
SCliMsg* pMsg = arg->param1;
SCliThrd* pThrd = arg->param2;
@@ -1264,13 +1283,13 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) {
transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL);
}
-void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
+FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) {
if (*val != exp) {
*val = newVal;
}
}
-bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) {
+FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) {
if ((pResp == NULL || pResp->info.hasEpSet == 0)) {
return false;
}
@@ -1300,7 +1319,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) {
STrans* pTransInst = pThrd->pTransInst;
if (pMsg == NULL || pMsg->ctx == NULL) {
- tTrace("%s conn %p handle resp", pTransInst->label, pConn);
+ tDebug("%s conn %p handle resp", pTransInst->label, pConn);
pTransInst->cfp(pTransInst->parent, pResp, NULL);
return 0;
}
@@ -1402,53 +1421,57 @@ void transUnrefCliHandle(void* handle) {
cliDestroyConn((SCliConn*)handle, true);
}
}
-SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) {
+static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t handle) {
SCliThrd* pThrd = NULL;
SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle);
if (exh == NULL) {
return NULL;
}
- *validHandle = true;
+ if (exh->pThrd == NULL && trans != NULL) {
+ int idx = cliRBChoseIdx(trans);
+ if (idx < 0) return NULL;
+ exh->pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
+ }
+
pThrd = exh->pThrd;
transReleaseExHandle(transGetRefMgt(), handle);
return pThrd;
}
-SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) {
+SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) {
if (handle == 0) {
int idx = cliRBChoseIdx(trans);
if (idx < 0) return NULL;
return ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
}
- SCliThrd* pThrd = transGetWorkThrdFromHandle(handle, validHandle);
- if (*validHandle == true && pThrd == NULL) {
- int idx = cliRBChoseIdx(trans);
- if (idx < 0) return NULL;
- pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx];
- }
+ SCliThrd* pThrd = transGetWorkThrdFromHandle(trans, handle);
return pThrd;
}
int transReleaseCliHandle(void* handle) {
int idx = -1;
bool valid = false;
- SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle, &valid);
+ SCliThrd* pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle);
if (pThrd == NULL) {
return -1;
}
- STransMsg tmsg = {.info.handle = handle};
+ STransMsg tmsg = {.info.handle = handle, .info.ahandle = (void*)0x9527};
TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64());
+ STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx));
+ pCtx->ahandle = tmsg.info.ahandle;
+
SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cmsg->msg = tmsg;
cmsg->type = Release;
+ cmsg->ctx = pCtx;
STraceId* trace = &tmsg.info.traceId;
tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid);
if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) {
- taosMemoryFree(cmsg);
+ destroyCmsg(cmsg);
return -1;
}
return 0;
@@ -1461,9 +1484,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran
return -1;
}
- bool valid = false;
- SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid);
- if (pThrd == NULL && valid == false) {
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
transFreeMsg(pReq->pCont);
transReleaseExHandle(transGetInstMgt(), (int64_t)shandle);
return TSDB_CODE_RPC_BROKEN_LINK;
@@ -1476,9 +1498,7 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran
pCtx->ahandle = pReq->info.ahandle;
pCtx->msgType = pReq->msgType;
- if (ctx != NULL) {
- pCtx->appCtx = *ctx;
- }
+ if (ctx != NULL) pCtx->appCtx = *ctx;
SCliMsg* cliMsg = taosMemoryCalloc(1, sizeof(SCliMsg));
cliMsg->ctx = pCtx;
@@ -1506,9 +1526,8 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs
return -1;
}
- bool valid = false;
- SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid);
- if (pThrd == NULL && valid == false) {
+ SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle);
+ if (pThrd == NULL) {
transFreeMsg(pReq->pCont);
transReleaseExHandle(transGetInstMgt(), (int64_t)shandle);
return TSDB_CODE_RPC_BROKEN_LINK;
@@ -1592,6 +1611,7 @@ int64_t transAllocHandle() {
SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle));
exh->refId = transAddExHandle(transGetRefMgt(), exh);
tDebug("pre alloc refId %" PRId64 "", exh->refId);
+
return exh->refId;
}
#endif
diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c
index dea96fa3acf106f2956435dbff9b3511a8db1219..5f3171ee0e840ee7f558a13b4ad315bd9bcfb856 100644
--- a/source/libs/transport/src/transComm.c
+++ b/source/libs/transport/src/transComm.c
@@ -424,7 +424,7 @@ void transQueueDestroy(STransQueue* queue) {
taosArrayDestroy(queue->q);
}
-static int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
+static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) {
SDelayTask* arg1 = container_of(a, SDelayTask, node);
SDelayTask* arg2 = container_of(b, SDelayTask, node);
if (arg1->execTime > arg2->execTime) {
diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c
index 7007079f8760c944c70bbfe0798fb0fc518cf958..70a47fe079ed2d2fbd4638a3273f69c75bb8d60a 100644
--- a/source/libs/transport/src/transSvr.c
+++ b/source/libs/transport/src/transSvr.c
@@ -125,17 +125,17 @@ static void uvWorkAfterTask(uv_work_t* req, int status);
static void uvWalkCb(uv_handle_t* handle, void* arg);
static void uvFreeCb(uv_handle_t* handle);
-static void uvStartSendRespImpl(SSvrMsg* smsg);
+static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg);
+
static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb);
static void uvStartSendResp(SSvrMsg* msg);
static void uvNotifyLinkBrokenToApp(SSvrConn* conn);
-static void destroySmsg(SSvrMsg* smsg);
-// check whether already read complete packet
-static SSvrConn* createConn(void* hThrd);
-static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
-static void destroyConnRegArg(SSvrConn* conn);
+static FORCE_INLINE void destroySmsg(SSvrMsg* smsg);
+static FORCE_INLINE SSvrConn* createConn(void* hThrd);
+static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/);
+static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn);
static int reallocConnRef(SSvrConn* conn);
@@ -413,7 +413,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) {
wb->len = len;
}
-static void uvStartSendRespImpl(SSvrMsg* smsg) {
+static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) {
SSvrConn* pConn = smsg->pConn;
if (pConn->broken) {
return;
@@ -447,7 +447,7 @@ static void uvStartSendResp(SSvrMsg* smsg) {
return;
}
-static void destroySmsg(SSvrMsg* smsg) {
+static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) {
if (smsg == NULL) {
return;
}
@@ -812,7 +812,7 @@ void* transWorkerThread(void* arg) {
return NULL;
}
-static SSvrConn* createConn(void* hThrd) {
+static FORCE_INLINE SSvrConn* createConn(void* hThrd) {
SWorkThrd* pThrd = hThrd;
SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn));
@@ -842,7 +842,7 @@ static SSvrConn* createConn(void* hThrd) {
return pConn;
}
-static void destroyConn(SSvrConn* conn, bool clear) {
+static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear) {
if (conn == NULL) {
return;
}
@@ -854,7 +854,7 @@ static void destroyConn(SSvrConn* conn, bool clear) {
}
}
}
-static void destroyConnRegArg(SSvrConn* conn) {
+static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn) {
if (conn->regArg.init == 1) {
transFreeMsg(conn->regArg.msg.pCont);
conn->regArg.init = 0;
@@ -907,7 +907,7 @@ static void uvDestroyConn(uv_handle_t* handle) {
}
static void uvPipeListenCb(uv_stream_t* handle, int status) {
if (status != 0) {
- tError("server failed to init pipe");
+ tError("server failed to init pipe, errmsg: %s", uv_err_name(status));
return;
}
@@ -945,7 +945,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
uv_loop_init(srv->loop);
int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to init pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
#ifdef WINDOWS
char pipeName[64];
@@ -956,10 +959,16 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads,
taosGetSelfPthreadId());
#endif
ret = uv_pipe_bind(&srv->pipeListen, pipeName);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to bind pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
ret = uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb);
- assert(ret == 0);
+ if (ret != 0) {
+ tError("failed to listen pipe, errmsg: %s", uv_err_name(ret));
+ goto End;
+ }
for (int i = 0; i < srv->numOfThreads; i++) {
SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd));
@@ -1082,12 +1091,12 @@ void transCloseServer(void* arg) {
if (srv->inited) {
uv_async_send(srv->pAcceptAsync);
taosThreadJoin(srv->thread, NULL);
- }
- SRV_RELEASE_UV(srv->loop);
+ SRV_RELEASE_UV(srv->loop);
- for (int i = 0; i < srv->numOfThreads; i++) {
- sendQuitToWorkThrd(srv->pThreadObj[i]);
- destroyWorkThrd(srv->pThreadObj[i]);
+ for (int i = 0; i < srv->numOfThreads; i++) {
+ sendQuitToWorkThrd(srv->pThreadObj[i]);
+ destroyWorkThrd(srv->pThreadObj[i]);
+ }
}
taosMemoryFree(srv->pThreadObj);
@@ -1139,6 +1148,7 @@ int transReleaseSrvHandle(void* handle) {
tTrace("%s conn %p start to release", transLabel(pThrd->pTransInst), exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q);
+
transReleaseExHandle(transGetRefMgt(), refId);
return 0;
_return1:
@@ -1168,8 +1178,10 @@ int transSendResponse(const STransMsg* msg) {
STraceId* trace = (STraceId*)&msg->info.traceId;
tGTrace("conn %p start to send resp (1/2)", exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q);
+
transReleaseExHandle(transGetRefMgt(), refId);
return 0;
+
_return1:
tTrace("handle %p failed to send resp", exh);
rpcFreeCont(msg->pCont);
@@ -1198,6 +1210,7 @@ int transRegisterMsg(const STransMsg* msg) {
STrans* pTransInst = pThrd->pTransInst;
tTrace("%s conn %p start to register brokenlink callback", transLabel(pTransInst), exh->handle);
transAsyncSend(pThrd->asyncPool, &m->q);
+
transReleaseExHandle(transGetRefMgt(), refId);
return 0;
diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c
index 9db7d6c4554e3231399abd5ab470ea71f8207a37..5c437e6f7aeb942e02e79314e4140af0cdfe4323 100644
--- a/source/libs/wal/src/walRead.c
+++ b/source/libs/wal/src/walRead.c
@@ -168,6 +168,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) {
}
pReader->pIdxFile = pIdxFile;
+
+ pReader->curFileFirstVer = fileFirstVer;
+
return 0;
}
@@ -372,7 +375,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) {
int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) {
int64_t code;
-// ASSERT(pRead->curVersion == pHead->head.version);
+ // ASSERT(pRead->curVersion == pHead->head.version);
code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR);
if (code < 0) {
@@ -415,7 +418,8 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) {
}
if (walValidBodyCksum(*ppHead) != 0) {
- wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver);
+ wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId,
+ ver);
pRead->curInvalid = 1;
terrno = TSDB_CODE_WAL_FILE_CORRUPTED;
return -1;
diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c
index 3dfb1458ad2fc802af761f68a4fe4407098fff25..418e86a40a918d3340ef809701d3f1d44cb7bb04 100644
--- a/source/os/src/osDir.c
+++ b/source/os/src/osDir.c
@@ -31,6 +31,15 @@ typedef struct TdDir {
HANDLE hFind;
} TdDir;
+enum
+ {
+ WRDE_NOSPACE = 1, /* Ran out of memory. */
+ WRDE_BADCHAR, /* A metachar appears in the wrong place. */
+ WRDE_BADVAL, /* Undefined var reference with WRDE_UNDEF. */
+ WRDE_CMDSUB, /* Command substitution with WRDE_NOCMD. */
+ WRDE_SYNTAX /* Shell syntax error. */
+ };
+
int wordexp(char *words, wordexp_t *pwordexp, int flags) {
pwordexp->we_offs = 0;
pwordexp->we_wordc = 1;
@@ -265,9 +274,21 @@ void taosRemoveOldFiles(const char *dirname, int32_t keepDays) {
int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen) {
wordexp_t full_path;
- if (0 != wordexp(dirname, &full_path, 0)) {
- printf("failed to expand path:%s since %s", dirname, strerror(errno));
- wordfree(&full_path);
+ switch (wordexp (dirname, &full_path, 0)) {
+ case 0:
+ break;
+ case WRDE_NOSPACE:
+ wordfree (&full_path);
+ // printf("failed to expand path:%s since Out of memory\n", dirname);
+ return -1;
+ case WRDE_BADCHAR:
+ // printf("failed to expand path:%s since illegal occurrence of newline or one of |, &, ;, <, >, (, ), {, }\n", dirname);
+ return -1;
+ case WRDE_SYNTAX:
+ // printf("failed to expand path:%s since Shell syntax error, such as unbalanced parentheses or unmatched quotes\n", dirname);
+ return -1;
+ default:
+ // printf("failed to expand path:%s since %s\n", dirname, strerror(errno));
return -1;
}
diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c
index 8cc6f0ef2e2b436624cc961315e3ecff6db7691b..5baba5af1e8051afd06b6f188795fb7c26e99a7e 100644
--- a/source/os/src/osSemaphore.c
+++ b/source/os/src/osSemaphore.c
@@ -400,6 +400,9 @@ int tsem_init(tsem_t *psem, int flags, unsigned int count) {
}
int tsem_destroy(tsem_t *psem) {
+ if (psem == NULL || *psem == NULL) return -1;
+ dispatch_release(*psem);
+ *psem = NULL;
return 0;
}
@@ -421,13 +424,7 @@ int tsem_timewait(tsem_t *psem, int64_t nanosecs) {
return 0;
}
-bool taosCheckPthreadValid(TdThread thread) {
- int32_t ret = taosThreadKill(thread, 0);
- if (ret == ESRCH) return false;
- if (ret == EINVAL) return false;
- // alive
- return true;
-}
+bool taosCheckPthreadValid(TdThread thread) { return thread != 0; }
int64_t taosGetSelfPthreadId() {
TdThread thread = taosThreadSelf();
diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c
index 19e9568bbebf20a74e5f316bb50056efa4786c1a..6867c1373bdb03ad63c3001291feaad468090dea 100644
--- a/source/os/src/osSysinfo.c
+++ b/source/os/src/osSysinfo.c
@@ -344,30 +344,27 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) {
*numOfCores = si.dwNumberOfProcessors;
return 0;
#elif defined(_TD_DARWIN_64)
- char *line = NULL;
- size_t size = 0;
+ char buf[16];
int32_t done = 0;
int32_t code = -1;
- TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM);
- if (pFile == NULL) return false;
-
- while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) {
- line[size - 1] = '\0';
- if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) {
- const char *v = strchr(line, ':') + 2;
- tstrncpy(cpuModel, v, maxLen);
- code = 0;
- done |= 1;
- } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) {
- const char *v = strchr(line, ':') + 2;
- *numOfCores = atof(v);
- done |= 2;
- }
+ TdCmdPtr pCmd = taosOpenCmd("sysctl -n machdep.cpu.brand_string");
+ if (pCmd == NULL) return code;
+ if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) {
+ code = 0;
+ done |= 1;
}
+ taosCloseCmd(&pCmd);
- if (line != NULL) taosMemoryFree(line);
- taosCloseFile(&pFile);
+ pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count");
+ if (pCmd == NULL) return code;
+ memset(buf, 0, sizeof(buf));
+ if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) {
+ code = 0;
+ done |= 2;
+ *numOfCores = atof(buf);
+ }
+ taosCloseCmd(&pCmd);
return code;
#else
diff --git a/source/util/src/tarray.c b/source/util/src/tarray.c
index 454739348eec9cd4fa59f5ec359890be2c09c816..4f170c203c1f25af55db51c86d1fadacb41deb35 100644
--- a/source/util/src/tarray.c
+++ b/source/util/src/tarray.c
@@ -386,11 +386,12 @@ void* taosArrayDestroy(SArray* pArray) {
}
void taosArrayDestroyP(SArray* pArray, FDelete fp) {
- if(!pArray) return;
- for (int32_t i = 0; i < pArray->size; i++) {
- fp(*(void**)TARRAY_GET_ELEM(pArray, i));
+ if (pArray) {
+ for (int32_t i = 0; i < pArray->size; i++) {
+ fp(*(void**)TARRAY_GET_ELEM(pArray, i));
+ }
+ taosArrayDestroy(pArray);
}
- taosArrayDestroy(pArray);
}
void taosArrayDestroyEx(SArray* pArray, FDelete fp) {
diff --git a/source/util/src/terror.c b/source/util/src/terror.c
index 044cdc86b4d91cbb26e00435cb241f39c0442f00..50c42ff170db351285aca7ef631f0143d3fc5167 100644
--- a/source/util/src/terror.c
+++ b/source/util/src/terror.c
@@ -288,6 +288,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first")
TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer")
+TAOS_DEFINE_ERROR(TSDB_CODE_MND_IN_REBALANCE, "Topic being rebalanced")
// mnode-stream
TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists")
@@ -579,6 +580,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_NUM, "Invalid function par
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid function para type")
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid function para value")
TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION, "Not buildin function")
+TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps not allowed in function")
//udf
TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping")
diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c
index 89471c4347e49152b2b1fff0313ef29da92c70ff..9cf9e2c4316a850ac31b15aaab8a864deb4d32eb 100644
--- a/source/util/src/tsched.c
+++ b/source/util/src/tsched.c
@@ -149,18 +149,18 @@ void *taosProcessSchedQueue(void *scheduler) {
return NULL;
}
-void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) {
+int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) {
SSchedQueue *pSched = (SSchedQueue *)queueScheduler;
int32_t ret = 0;
if (pSched == NULL) {
uError("sched is not ready, msg:%p is dropped", pMsg);
- return;
+ return -1;
}
if (atomic_load_8(&pSched->stop)) {
uError("sched is already stopped, msg:%p is dropped", pMsg);
- return;
+ return -1;
}
if ((ret = tsem_wait(&pSched->emptySem)) != 0) {
@@ -185,6 +185,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) {
uFatal("post %s fullSem failed(%s)", pSched->label, strerror(errno));
ASSERT(0);
}
+ return ret;
}
void taosCleanUpScheduler(void *param) {
@@ -192,11 +193,11 @@ void taosCleanUpScheduler(void *param) {
if (pSched == NULL) return;
uDebug("start to cleanup %s schedQsueue", pSched->label);
-
+
atomic_store_8(&pSched->stop, 1);
taosMsleep(200);
-
+
for (int32_t i = 0; i < pSched->numOfThreads; ++i) {
if (taosCheckPthreadValid(pSched->qthread[i])) {
tsem_post(&pSched->fullSem);
@@ -220,7 +221,7 @@ void taosCleanUpScheduler(void *param) {
if (pSched->queue) taosMemoryFree(pSched->queue);
if (pSched->qthread) taosMemoryFree(pSched->qthread);
- //taosMemoryFree(pSched);
+ // taosMemoryFree(pSched);
}
// for debug purpose, dump the scheduler status every 1min.
diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c
index 9101aec949873eb976517581c6790e1bc2dac3b9..7460ccbc829e7c630d217a39f29a2eafa934cb53 100644
--- a/source/util/src/tuuid.c
+++ b/source/util/src/tuuid.c
@@ -51,11 +51,11 @@ int64_t tGenIdPI64(void) {
int64_t id;
while (true) {
- int64_t ts = taosGetTimestampMs();
+ int64_t ts = taosGetTimestampMs() >> 8;
uint64_t pid = taosGetPId();
int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1);
- id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF);
+ id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0F) << 48) | ((ts & 0x3FFFFFF) << 20) | (val & 0xFFFFF);
if (id) {
break;
}
diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp
index 1a057c5875ee95de2fc3c457ca09314366fff48c..534c17758714820e9a9f2bf6b81d23ac121fcaf4 100644
--- a/source/util/test/pageBufferTest.cpp
+++ b/source/util/test/pageBufferTest.cpp
@@ -23,9 +23,9 @@ void simpleTest() {
ASSERT_EQ(getTotalBufSize(pBuf), 1024);
- SIDList list = getDataBufPagesIdList(pBuf, groupId);
+ SIDList list = getDataBufPagesIdList(pBuf);
ASSERT_EQ(taosArrayGetSize(list), 1);
- ASSERT_EQ(getNumOfBufGroupId(pBuf), 1);
+ //ASSERT_EQ(getNumOfBufGroupId(pBuf), 1);
releaseBufPage(pBuf, pBufPage);
@@ -98,7 +98,7 @@ void writeDownTest() {
SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId));
ASSERT_EQ(*(int32_t*)pBufPagex->data, nx);
- SArray* pa = getDataBufPagesIdList(pBuf, groupId);
+ SArray* pa = getDataBufPagesIdList(pBuf);
ASSERT_EQ(taosArrayGetSize(pa), 5);
destroyDiskbasedBuf(pBuf);
@@ -152,7 +152,7 @@ void recyclePageTest() {
SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1));
- SArray* pa = getDataBufPagesIdList(pBuf, groupId);
+ SArray* pa = getDataBufPagesIdList(pBuf);
ASSERT_EQ(taosArrayGetSize(pa), 6);
destroyDiskbasedBuf(pBuf);
diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c
index 89561e61fe02e4665828bc8336ec79f6df46fc68..5a6790ef1da3b5a6b015c836d09150b06e0a003f 100644
--- a/source/util/test/trefTest.c
+++ b/source/util/test/trefTest.c
@@ -94,7 +94,7 @@ void *openRefSpace(void *param) {
pSpace->rsetId = taosOpenRef(50, myfree);
if (pSpace->rsetId < 0) {
- printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId));
+ printf("failed to open ref, reason:%s\n", tstrerror(pSpace->rsetId));
return NULL;
}
diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py
index 99e8cd36a4cf1dae08baf93ef4d6338bb08dc7bd..a44ad4c1d0b5cc8e4ba181c1bf941454929f2df6 100644
--- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py
+++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py
@@ -22,9 +22,9 @@ from util.dnodes import *
class TDTestCase:
def caseDescription(self):
- '''
+ """
[TD-13823] taosBenchmark test cases
- '''
+ """
return
def init(self, conn, logSql):
@@ -34,19 +34,19 @@ class TDTestCase:
def getPath(self, tool="taosBenchmark"):
selfPath = os.path.dirname(os.path.realpath(__file__))
- if ("community" in selfPath):
- projPath = selfPath[:selfPath.find("community")]
+ if "community" in selfPath:
+ projPath = selfPath[: selfPath.find("community")]
else:
- projPath = selfPath[:selfPath.find("tests")]
+ projPath = selfPath[: selfPath.find("tests")]
paths = []
for root, dirs, files in os.walk(projPath):
- if ((tool) in files):
+ if (tool) in files:
rootRealPath = os.path.dirname(os.path.realpath(root))
- if ("packaging" not in rootRealPath):
+ if "packaging" not in rootRealPath:
paths.append(os.path.join(root, tool))
break
- if (len(paths) == 0):
+ if len(paths) == 0:
tdLog.exit("taosBenchmark not found!")
return
else:
@@ -55,7 +55,7 @@ class TDTestCase:
def run(self):
binPath = self.getPath()
- cmd = "%s -n 100 -t 100 -y" %binPath
+ cmd = "%s -n 100 -t 100 -y" % binPath
tdLog.info("%s" % cmd)
os.system("%s" % cmd)
tdSql.execute("use test")
@@ -77,14 +77,16 @@ class TDTestCase:
tdSql.checkData(4, 3, "TAG")
tdSql.checkData(5, 0, "location")
tdSql.checkData(5, 1, "VARCHAR")
- tdSql.checkData(5, 2, 16)
+ tdSql.checkData(5, 2, 24)
tdSql.checkData(5, 3, "TAG")
tdSql.query("select count(*) from test.meters where groupid >= 0")
tdSql.checkData(0, 0, 10000)
- tdSql.query("select count(*) from test.meters where location = 'San Francisco' or location = 'Los Angles' or location = 'San Diego' or location = 'San Jose' or \
- location = 'Palo Alto' or location = 'Campbell' or location = 'Mountain View' or location = 'Sunnyvale' or location = 'Santa Clara' or location = 'Cupertino' ")
+ tdSql.query(
+ "select count(*) from test.meters where location = 'California.SanFrancisco' or location = 'California.LosAngles' or location = 'California.SanDiego' or location = 'California.SanJose' or \
+ location = 'California.PaloAlto' or location = 'California.Campbell' or location = 'California.MountainView' or location = 'California.Sunnyvale' or location = 'California.SantaClara' or location = 'California.Cupertino' "
+ )
tdSql.checkData(0, 0, 10000)
def stop(self):
diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt
index 46bae734ea72901ef704969045186a10c52a9a72..161c87844058ca6852c6649141e37f6cdf6a202f 100644
--- a/tests/script/jenkins/basic.txt
+++ b/tests/script/jenkins/basic.txt
@@ -221,6 +221,7 @@
./test.sh -f tsim/table/describe.sim
./test.sh -f tsim/table/double.sim
./test.sh -f tsim/table/float.sim
+./test.sh -f tsim/table/hash.sim
./test.sh -f tsim/table/int.sim
./test.sh -f tsim/table/limit.sim
./test.sh -f tsim/table/smallint.sim
@@ -248,6 +249,12 @@
./test.sh -f tsim/stream/windowClose.sim
./test.sh -f tsim/stream/ignoreExpiredData.sim
./test.sh -f tsim/stream/sliding.sim
+#./test.sh -f tsim/stream/partitionbyColumnInterval.sim
+#./test.sh -f tsim/stream/partitionbyColumnSession.sim
+#./test.sh -f tsim/stream/partitionbyColumnState.sim
+#./test.sh -f tsim/stream/deleteInterval.sim
+#./test.sh -f tsim/stream/deleteSession.sim
+#./test.sh -f tsim/stream/deleteState.sim
# ---- transaction ----
./test.sh -f tsim/trans/lossdata1.sim
diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c
deleted file mode 100644
index d623adacf941e26d0de74c0c582beb7ca83c9c13..0000000000000000000000000000000000000000
--- a/tests/script/sh/abs_max.c
+++ /dev/null
@@ -1,88 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-
-#define TSDB_DATA_INT_NULL 0x80000000LL
-#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL
-
-void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 5) {
- r=*(long *)dataOutput;
- *numOfOutput=0;
-
- for(i=0;i r) {
- r = v;
- }
- }
-
- *(long *)dataOutput=r;
-
- printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
- }
-}
-
-
-
-void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf);
- *numOfOutput=1;
- printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
-}
-
-void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int r = 0;
-
- if (numOfRows > 0) {
- r = *((long *)data);
- }
- printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
- for (int i = 1; i < numOfRows; ++i) {
- printf("abs_max_merge %d - %ld\n", i, *((long *)data + i));
- if (*((long*)data + i) > r) {
- r= *((long*)data + i);
- }
- }
-
- *(long*)dataOutput=r;
- if (numOfRows > 0) {
- *numOfOutput=1;
- } else {
- *numOfOutput=0;
- }
-
- printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput);
-}
-
-
-int abs_max_init(SUdfInit* buf) {
- printf("abs_max init\n");
- return 0;
-}
-
-
-void abs_max_destroy(SUdfInit* buf) {
- printf("abs_max destroy\n");
-}
-
diff --git a/tests/script/sh/add_one.c b/tests/script/sh/add_one.c
deleted file mode 100644
index e12cf8f26f6ddad67f9f7b091c033de46a3f6f50..0000000000000000000000000000000000000000
--- a/tests/script/sh/add_one.c
+++ /dev/null
@@ -1,33 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 4) {
- for(i=0;i
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-typedef struct SDemo{
- double sum;
- int num;
- short otype;
-}SDemo;
-
-#define FLOAT_NULL 0x7FF00000 // it is an NAN
-#define DOUBLE_NULL 0x7FFFFF0000000000LL // it is an NAN
-
-
-void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- double r = 0;
- SDemo *p = (SDemo *)interBuf;
- SDemo *q = (SDemo *)dataOutput;
- printf("demo input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, interBUf:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, interBuf, tsOutput, numOfOutput, buf);
-
- for(i=0;isum += r*r;
- }
-
- p->otype = otype;
- p->num += numOfRows;
-
- q->sum = p->sum;
- q->num = p->num;
- q->otype = p->otype;
-
- *numOfOutput=1;
-
- printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
-}
-
-
-void demo_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int i;
- SDemo *p = (SDemo *)data;
- SDemo res = {0};
- printf("demo_merge input data:%p, rows:%d, dataoutput:%p, numOfOutput:%p, buf:%p\n", data, numOfRows, dataOutput, numOfOutput, buf);
-
- for(i=0;isum * p->sum;
- res.num += p->num;
- p++;
- }
-
- p->sum = res.sum;
- p->num = res.num;
-
- *numOfOutput=1;
-
- printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput);
-}
-
-
-
-void demo_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) {
- SDemo *p = (SDemo *)interBuf;
- printf("demo_finalize interbuf:%p, numOfOutput:%p, buf:%p, sum:%f, num:%d\n", interBuf, numOfOutput, buf, p->sum, p->num);
- if (p->otype == 6) {
- if (p->num != 30000) {
- *(unsigned int *)dataOutput = FLOAT_NULL;
- } else {
- *(float *)dataOutput = (float)(p->sum / p->num);
- }
- printf("finalize values:%f\n", *(float *)dataOutput);
- } else if (p->otype == 7) {
- if (p->num != 30000) {
- *(unsigned long long *)dataOutput = DOUBLE_NULL;
- } else {
- *(double *)dataOutput = (double)(p->sum / p->num);
- }
- printf("finalize values:%f\n", *(double *)dataOutput);
- }
-
- *numOfOutput=1;
-
- printf("demo finalize, numOfOutput:%d\n", *numOfOutput);
-}
-
-
-int demo_init(SUdfInit* buf) {
- printf("demo init\n");
- return 0;
-}
-
-
-void demo_destroy(SUdfInit* buf) {
- printf("demo destroy\n");
-}
-
diff --git a/tests/script/sh/demo.lua b/tests/script/sh/demo.lua
deleted file mode 100644
index c5e5582fc30b58db30a5b18faa4ccfd0a5f656d0..0000000000000000000000000000000000000000
--- a/tests/script/sh/demo.lua
+++ /dev/null
@@ -1,43 +0,0 @@
-funcName = "test"
-
-global = {}
-
-function test_init()
- return global
-end
-
-function test_add(rows, ans, key)
- t = {}
- t["sum"] = 0.0
- t["num"] = 0
- for i=1, #rows do
- t["sum"] = t["sum"] + rows[i] * rows[i]
- end
- t["num"] = #rows
-
-
- if (ans[key] ~= nil)
- then
- ans[key]["sum"] = ans[key]["sum"] + t["sum"]
- ans[key]["num"] = ans[key]["num"] + t["num"]
- else
- ans[key] = t
- end
-
- return ans;
-end
-
-function test_finalize(ans, key)
- local ret = 0.0
-
- if (ans[key] ~= nil and ans[key]["num"] == 30000)
- then
- ret = ans[key]["sum"]/ans[key]["num"]
- ans[key]["sum"] = 0.0
- ans[key]["num"] = 0
- else
- ret = inf
- end
-
- return ret, ans
-end
diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c
deleted file mode 100644
index d6eea5d291c4add6bd40ae36492a1366c67f2cfd..0000000000000000000000000000000000000000
--- a/tests/script/sh/sum_double.c
+++ /dev/null
@@ -1,84 +0,0 @@
-#include
-#include
-#include
-
-typedef struct SUdfInit{
- int maybe_null; /* 1 if function can return NULL */
- int decimals; /* for real functions */
- long long length; /* For string functions */
- char *ptr; /* free pointer for function data */
- int const_item; /* 0 if result is independent of arguments */
-} SUdfInit;
-
-#define TSDB_DATA_INT_NULL 0x80000000LL
-
-
-void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput,
- int* numOfOutput, short otype, short obytes, SUdfInit* buf) {
- int i;
- int r = 0;
- printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf);
- if (itype == 4) {
- r=*(int *)dataOutput;
- *numOfOutput=0;
-
- for(i=0;iptr)=*(int*)dataOutput*2;
- *(int*)dataOutput=*(int*)(buf->ptr);
- printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
-}
-
-void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) {
- int r = 0;
- int sum = 0;
-
- printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf);
- for (int i = 0; i < numOfRows; ++i) {
- printf("sum_double_merge %d - %d\n", i, *((int*)data + i));
- sum +=*((int*)data + i);
- }
-
- *(int*)dataOutput+=sum;
- if (numOfRows > 0) {
- *numOfOutput=1;
- } else {
- *numOfOutput=0;
- }
-
- printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput);
-}
-
-
-int sum_double_init(SUdfInit* buf) {
- buf->maybe_null=1;
- buf->ptr = taosMemoryMalloc(sizeof(int));
- printf("sum_double init\n");
- return 0;
-}
-
-
-void sum_double_destroy(SUdfInit* buf) {
- taosMemoryFree(buf->ptr);
- printf("sum_double destroy\n");
-}
-
diff --git a/tests/script/tsim/column/table.sim b/tests/script/tsim/column/table.sim
index 4f1d32c373e3712275deec14a54e7efa0e77de61..03c4799681015a63e1e343168037f6cd4d601b8b 100644
--- a/tests/script/tsim/column/table.sim
+++ b/tests/script/tsim/column/table.sim
@@ -159,6 +159,7 @@ if $data01 != 10 then
return -1
endi
if $data02 != 4.500000000 then
+ print expect 4.500000000, actual: $data02
return -1
endi
if $data03 != 4.500000000 then
diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim
index ae19a4803bc6bd0f75c2a307696d7b0fc6f1ecb6..a9bf8b26ebda178ed789f3119eac7203a6889f9b 100644
--- a/tests/script/tsim/parser/commit.sim
+++ b/tests/script/tsim/parser/commit.sim
@@ -97,6 +97,7 @@ while $loop <= $loops
endw
sql select count(*) from $stb
if $data00 != $totalNum then
+ print expect $totalNum , actual: $data00
return -1
endi
$loop = $loop + 1
diff --git a/tests/script/tsim/parser/union.sim b/tests/script/tsim/parser/union.sim
index 8bafeff4443d4a125071877bd3d7175be60cd925..8dc19912c830cca6b6fdbbed4a7084bf416d4edd 100644
--- a/tests/script/tsim/parser/union.sim
+++ b/tests/script/tsim/parser/union.sim
@@ -116,7 +116,7 @@ sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 l
sql_error (select c1 from union_tb0 limit 1 union all select c1 from union_tb1 limit 1) limit 1
# sql with parenthese
-sql (((select c1 from union_tb0)))
+sql (select c1 from union_tb0)
if $rows != 10000 then
return -1
endi
diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim
index 70ca1179b0ab5d1e54864c6161749967b303f559..8942f7f702787c9e026e0c37e47ce56765f554bf 100644
--- a/tests/script/tsim/stream/basic1.sim
+++ b/tests/script/tsim/stream/basic1.sim
@@ -5,7 +5,7 @@ sleep 50
sql connect
print =============== create database
-sql create database test vgroups 1
+sql create database test vgroups 1;
sql select * from information_schema.ins_databases
if $rows != 3 then
return -1
@@ -13,7 +13,7 @@ endi
print $data00 $data01 $data02
-sql use test
+sql use test;
sql create table t1(ts timestamp, a int, b int , c int, d double);
diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim
new file mode 100644
index 0000000000000000000000000000000000000000..00d10afad99a258d3c931add7e7ceedc653e85df
--- /dev/null
+++ b/tests/script/tsim/stream/deleteInterval.sim
@@ -0,0 +1,495 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,2,2,2,2.0);
+sql insert into t1 values(1648791213002,3,3,3,3.0);
+sql insert into t1 values(1648791213003,4,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,2,2,2,2.0);
+sql insert into t1 values(1648791213006,3,3,3,3.0);
+sql insert into t1 values(1648791213007,4,4,4,4.0);
+
+sql insert into t1 values(1648791223000,1,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,3,3,3,3.0);
+sql insert into t1 values(1648791223003,4,4,4,4.0);
+
+sql insert into t1 values(1648791233000,1,1,1,1.0);
+sql insert into t1 values(1648791233001,2,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,4,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql drop database if exists test;
+sql create database test2 vgroups 4;
+sql create database test vgroups 1;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop7
+endi
+
+sleep 200
+
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop8:
+sleep 200
+
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001;
+
+$loop_count = 0
+
+loop11:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 6 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+sleep 200
+
+sql delete from st where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop12:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop12
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop12
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop12
+endi
+
+sql insert into t1 values(1648791213004,3,2,3,1.0);
+sql insert into t1 values(1648791213005,3,2,3,1.0);
+sql insert into t1 values(1648791213006,3,2,3,1.0);
+sql insert into t1 values(1648791223004,1,2,3,1.0);
+sql insert into t2 values(1648791213004,3,2,3,1.0);
+sql insert into t2 values(1648791213005,3,2,3,1.0);
+sql insert into t2 values(1648791213006,3,2,3,1.0);
+sql insert into t2 values(1648791223004,1,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006;
+
+$loop_count = 0
+
+loop13:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop13
+endi
+
+if $data01 != 4 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 3 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop13
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop13
+endi
+
+sql insert into t1 values(1648791223005,1,2,3,1.0);
+sql insert into t1 values(1648791223006,1,2,3,1.0);
+sql insert into t2 values(1648791223005,1,2,3,1.0);
+sql insert into t2 values(1648791223006,1,2,3,1.0);
+
+sql insert into t1 values(1648791233005,4,2,3,1.0);
+sql insert into t1 values(1648791233006,2,2,3,1.0);
+sql insert into t2 values(1648791233005,5,2,3,1.0);
+sql insert into t2 values(1648791233006,3,2,3,1.0);
+
+sleep 200
+
+sql delete from st where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop14:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop14
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop14
+endi
+
+return 1
+
+sql drop stream if exists streams3;
+sql drop database if exists test3;
+sql drop database if exists test;
+sql create database test3 vgroups 4;
+sql create database test vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+sql delete from t1;
+
+loop15:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop15
+endi
+
+$loop_count = 0
+
+sql delete from t1 where ts > 100;
+
+loop16:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop16
+endi
+
+$loop_count = 0
+
+sql delete from st;
+
+loop17:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop17
+endi
+
+
+
+
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim
new file mode 100644
index 0000000000000000000000000000000000000000..541609633b023611815252cde0109cdc01094198
--- /dev/null
+++ b/tests/script/tsim/stream/deleteSession.sim
@@ -0,0 +1,532 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,2,2,2,2.0);
+sql insert into t1 values(1648791213002,3,3,3,3.0);
+sql insert into t1 values(1648791213003,4,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,2,2,2,2.0);
+sql insert into t1 values(1648791213006,3,3,3,3.0);
+sql insert into t1 values(1648791213007,4,4,4,4.0);
+
+sql insert into t1 values(1648791223000,1,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,3,3,3,3.0);
+sql insert into t1 values(1648791223003,4,4,4,4.0);
+
+sql insert into t1 values(1648791233000,1,1,1,1.0);
+sql insert into t1 values(1648791233001,2,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,4,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+sql drop stream if exists streams2;
+sql drop database if exists test2;
+sql create database test2 vgroups 4;
+sql use test2;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop7:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop7
+endi
+
+sleep 200
+
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop8:
+sleep 200
+
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop8
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop8
+endi
+
+sql insert into t1 values(1648791223000,1,2,3,1.0);
+sql insert into t1 values(1648791223001,1,2,3,1.0);
+sql insert into t1 values(1648791223002,3,2,3,1.0);
+sql insert into t1 values(1648791223003,3,2,3,1.0);
+sql insert into t2 values(1648791223000,1,2,3,1.0);
+sql insert into t2 values(1648791223001,1,2,3,1.0);
+sql insert into t2 values(1648791223002,3,2,3,1.0);
+sql insert into t2 values(1648791223003,3,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001;
+
+$loop_count = 0
+
+loop11:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop11
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop11
+endi
+
+if $data11 != 6 then
+ print =====data11=$data11
+ goto loop11
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop11
+endi
+
+sleep 200
+
+sql delete from st where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop12:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 1 then
+ print =====rows=$rows
+ goto loop12
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop12
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop12
+endi
+
+sql insert into t1 values(1648791213004,3,2,3,1.0);
+sql insert into t1 values(1648791213005,3,2,3,1.0);
+sql insert into t1 values(1648791213006,3,2,3,1.0);
+sql insert into t1 values(1648791223004,1,2,3,1.0);
+sql insert into t2 values(1648791213004,3,2,3,1.0);
+sql insert into t2 values(1648791213005,3,2,3,1.0);
+sql insert into t2 values(1648791213006,3,2,3,1.0);
+sql insert into t2 values(1648791223004,1,2,3,1.0);
+
+sleep 200
+
+sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006;
+
+$loop_count = 0
+
+loop13:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop13
+endi
+
+if $data01 != 4 then
+ print =====data01=$data01
+ goto loop13
+endi
+
+if $data02 != 3 then
+ print =====data02=$data02
+ goto loop13
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop13
+endi
+
+if $data12 != 1 then
+ print =====data12=$data12
+ goto loop13
+endi
+
+sql insert into t1 values(1648791223005,1,2,3,1.0);
+sql insert into t1 values(1648791223006,1,2,3,1.0);
+sql insert into t2 values(1648791223005,1,2,3,1.0);
+sql insert into t2 values(1648791223006,1,2,3,1.0);
+
+sql insert into t1 values(1648791233005,4,2,3,1.0);
+sql insert into t1 values(1648791233006,2,2,3,1.0);
+sql insert into t2 values(1648791233005,5,2,3,1.0);
+sql insert into t2 values(1648791233006,3,2,3,1.0);
+
+sleep 200
+
+sql delete from st where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop14:
+sleep 200
+sql select * from test.streamt2 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop14
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop14
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop14
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop14
+endi
+
+if $data12 != 3 then
+ print =====data12=$data12
+ goto loop14
+endi
+
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop database if exists test3;
+sql drop database if exists test;
+sql create database test3 vgroups 4;
+sql create database test vgroups 1;
+sql use test3;
+sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int);
+sql create table t1 using st tags(1,1,1);
+sql create table t2 using st tags(2,2,2);
+sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s);
+
+sql insert into t1 values(1648791210000,1,1,1,NULL);
+sql insert into t1 values(1648791210001,2,2,2,NULL);
+sql insert into t2 values(1648791213001,3,3,3,NULL);
+sql insert into t2 values(1648791213003,4,4,4,NULL);
+sql insert into t1 values(1648791216000,5,5,5,NULL);
+sql insert into t1 values(1648791216002,6,6,6,NULL);
+sql insert into t1 values(1648791216004,7,7,7,NULL);
+sql insert into t2 values(1648791218001,8,8,8,NULL);
+sql insert into t2 values(1648791218003,9,9,9,NULL);
+sql insert into t1 values(1648791222000,10,10,10,NULL);
+sql insert into t1 values(1648791222003,11,11,11,NULL);
+sql insert into t1 values(1648791222005,12,12,12,NULL);
+
+sql insert into t1 values(1648791232005,13,13,13,NULL);
+sql insert into t2 values(1648791242005,14,14,14,NULL);
+
+$loop_count = 0
+
+loop19:
+sleep 200
+sql select * from test.streamt3 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 3 then
+ print =====rows=$rows
+ goto loop19
+endi
+
+sql delete from t2 where ts >= 1648791213001 and ts <= 1648791218003;
+
+$loop_count = 0
+
+loop20:
+sleep 200
+sql select * from test.streamt3 order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 5 then
+ print =====rows=$rows
+ goto loop20
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop20
+endi
+
+if $data02 != 2 then
+ print =====data02=$data02
+ goto loop20
+endi
+
+if $data11 != 3 then
+ print =====data11=$data11
+ goto loop20
+endi
+
+if $data12 != 7 then
+ print =====data12=$data12
+ goto loop20
+endi
+
+if $data21 != 3 then
+ print =====data21=$data21
+ goto loop20
+endi
+
+if $data22 != 12 then
+ print =====data22=$data22
+ goto loop20
+endi
+
+if $data31 != 1 then
+ print =====data31=$data31
+ goto loop20
+endi
+
+if $data32 != 13 then
+ print =====data32=$data32
+ goto loop20
+endi
+
+if $data41 != 1 then
+ print =====data41=$data41
+ goto loop20
+endi
+
+if $data42 != 14 then
+ print =====data42=$data42
+ goto loop20
+endi
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim
new file mode 100644
index 0000000000000000000000000000000000000000..ecd9f55340edbc79265255848f5240f0c02fd737
--- /dev/null
+++ b/tests/script/tsim/stream/deleteState.sim
@@ -0,0 +1,198 @@
+$loop_all = 0
+looptest:
+
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sleep 200
+sql connect
+
+sql drop stream if exists streams0;
+sql drop stream if exists streams1;
+sql drop stream if exists streams2;
+sql drop stream if exists streams3;
+sql drop stream if exists streams4;
+sql drop database if exists test;
+sql create database test vgroups 1;
+sql use test;
+sql create table t1(ts timestamp, a int, b int , c int, d double);
+sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a);
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+sleep 200
+sql delete from t1 where ts = 1648791213000;
+
+$loop_count = 0
+
+loop0:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 0 then
+ print =====rows=$rows
+ goto loop0
+endi
+
+sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL);
+
+$loop_count = 0
+
+loop1:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop1
+endi
+
+if $data02 != NULL then
+ print =====data02=$data02
+ goto loop1
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213001,1,2,2,2.0);
+sql insert into t1 values(1648791213002,1,3,3,3.0);
+sql insert into t1 values(1648791213003,1,4,4,4.0);
+
+sleep 200
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002;
+
+$loop_count = 0
+
+loop3:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop3
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop3
+endi
+
+sql insert into t1 values(1648791223000,2,2,3,1.0);
+sql insert into t1 values(1648791223001,2,2,3,1.0);
+sql insert into t1 values(1648791223002,2,2,3,1.0);
+sql insert into t1 values(1648791223003,2,2,3,1.0);
+
+$loop_count = 0
+
+loop4:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop4
+endi
+
+sleep 200
+
+sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003;
+
+$loop_count = 0
+
+loop5:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $data01 != 2 then
+ print =====data01=$data01
+ goto loop5
+endi
+
+if $data02 != 4 then
+ print =====data02=$data02
+ goto loop5
+endi
+
+sql insert into t1 values(1648791213000,1,1,1,1.0);
+sql insert into t1 values(1648791213005,1,2,2,2.0);
+sql insert into t1 values(1648791213006,1,3,3,3.0);
+sql insert into t1 values(1648791213007,1,4,4,4.0);
+
+sql insert into t1 values(1648791223000,2,1,1,1.0);
+sql insert into t1 values(1648791223001,2,2,2,2.0);
+sql insert into t1 values(1648791223002,2,3,3,3.0);
+sql insert into t1 values(1648791223003,2,4,4,4.0);
+
+sql insert into t1 values(1648791233000,3,1,1,1.0);
+sql insert into t1 values(1648791233001,3,2,2,2.0);
+sql insert into t1 values(1648791233008,3,3,3,3.0);
+sql insert into t1 values(1648791233009,3,4,4,4.0);
+
+sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005;
+
+$loop_count = 0
+
+loop6:
+sleep 200
+sql select * from streamt order by c1, c2, c3;
+
+$loop_count = $loop_count + 1
+if $loop_count == 10 then
+ return -1
+endi
+
+if $rows != 2 then
+ print =====rows=$rows
+ goto loop6
+endi
+
+if $data01 != 1 then
+ print =====data01=$data01
+ goto loop6
+endi
+
+if $data02 != 1 then
+ print =====data02=$data02
+ goto loop6
+endi
+
+if $data11 != 2 then
+ print =====data11=$data11
+ goto loop6
+endi
+
+if $data12 != 4 then
+ print =====data12=$data12
+ goto loop6
+endi
+
+
+$loop_all = $loop_all + 1
+print ============loop_all=$loop_all
+
+system sh/stop_dnodes.sh
+
+#goto looptest
\ No newline at end of file
diff --git a/tests/script/tsim/stream/partitionbyColumn0.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim
similarity index 100%
rename from tests/script/tsim/stream/partitionbyColumn0.sim
rename to tests/script/tsim/stream/partitionbyColumnInterval.sim
diff --git a/tests/script/tsim/stream/partitionbyColumn1.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim
similarity index 100%
rename from tests/script/tsim/stream/partitionbyColumn1.sim
rename to tests/script/tsim/stream/partitionbyColumnSession.sim
diff --git a/tests/script/tsim/stream/partitionbyColumn2.sim b/tests/script/tsim/stream/partitionbyColumnState.sim
similarity index 100%
rename from tests/script/tsim/stream/partitionbyColumn2.sim
rename to tests/script/tsim/stream/partitionbyColumnState.sim
diff --git a/tests/script/tsim/table/hash.sim b/tests/script/tsim/table/hash.sim
new file mode 100644
index 0000000000000000000000000000000000000000..664f86713720e1ae1969027508e2931d23397f08
--- /dev/null
+++ b/tests/script/tsim/table/hash.sim
@@ -0,0 +1,84 @@
+system sh/stop_dnodes.sh
+system sh/deploy.sh -n dnode1 -i 1
+system sh/exec.sh -n dnode1 -s start
+sql connect
+
+#=========== prepare
+#sql create database d1 vgroups 2
+sql create database d1 vgroups 2 table_prefix 3 table_suffix 2
+sql select * from information_schema.ins_databases
+print $data(d1)[27] $data(d1)[28]
+if $data(d1)[27] != 3 then
+ return -1
+endi
+if $data(d1)[28] != 2 then
+ return -1
+endi
+
+sql use d1;
+sql create table st (ts timestamp, i int) tags (j int);
+sql create table st_ct_1 using st tags(3) st_ct_2 using st tags(4) st_ct_3 using st tags(5) st_ct_4 using st tags(6) st_ct_5 using st tags(7)
+sql insert into st_ct_1 values(now+1s, 1)
+sql insert into st_ct_1 values(now+2s, 2)
+sql insert into st_ct_1 values(now+3s, 3)
+sql insert into st_ct_2 values(now+1s, 1)
+sql insert into st_ct_2 values(now+2s, 2)
+sql insert into st_ct_2 values(now+3s, 3)
+sql insert into st_ct_3 values(now+1s, 1)
+sql insert into st_ct_3 values(now+2s, 2)
+sql insert into st_ct_3 values(now+3s, 2)
+sql insert into st_ct_4 values(now+1s, 1)
+sql insert into st_ct_4 values(now+2s, 2)
+sql insert into st_ct_4 values(now+3s, 2)
+sql insert into st_ct_5 values(now+1s, 1)
+sql insert into st_ct_5 values(now+2s, 2)
+sql insert into st_ct_5 values(now+3s, 2)
+
+# check query
+sql select * from st
+if $rows != 15 then
+ return -1
+endi
+
+# check table vgroup
+sql select * from information_schema.ins_tables where db_name = 'd1'
+if $data(st_ct_1)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_2)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_3)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_4)[6] != 2 then
+ return -1
+endi
+if $data(st_ct_5)[6] != 2 then
+ return -1
+endi
+
+# check invalid table name
+sql create table c1 using st tags(3)
+sql create table c12 using st tags(3)
+sql create table c123 using st tags(3)
+sql create table c1234 using st tags(3)
+sql create table c12345 using st tags(3)
+sql select * from information_schema.ins_tables where db_name = 'd1'
+if $data(c1)[6] != 2 then
+ return -1
+endi
+if $data(c12)[6] != 3 then
+ return -1
+endi
+if $data(c123)[6] != 2 then
+ return -1
+endi
+if $data(c1234)[6] != 3 then
+ return -1
+endi
+if $data(c12345)[6] != 3 then
+ return -1
+endi
+
+system sh/exec.sh -n dnode1 -s stop -x SIGINT
diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py
index dd44852d4902e7d444e9aebc52fa3b4bb186fc08..80f6a6bc3007bdb2f5a2dfaca2b0fe71d9ea2042 100644
--- a/tests/system-test/0-others/taosShellNetChk.py
+++ b/tests/system-test/0-others/taosShellNetChk.py
@@ -231,7 +231,7 @@ class TDTestCase:
if platform.system().lower() == 'windows':
os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9')
else:
- os.system('pkill taos')
+ os.system('pkill -9 taos')
def stop(self):
tdSql.close()
diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py
index 068d212ac4e49d00fe7ff938118352dbe462a988..f620a4b18a0d7ddf5a0d65c902d0411c270e6463 100644
--- a/tests/system-test/1-insert/delete_data.py
+++ b/tests/system-test/1-insert/delete_data.py
@@ -81,6 +81,7 @@ class TDTestCase:
'binary':self.binary_val,
'nchar':self.nchar_val
}
+
def insert_base_data(self,col_type,tbname,rows,base_data):
for i in range(rows):
if col_type.lower() == 'tinyint':
@@ -290,6 +291,9 @@ class TDTestCase:
self.delete_data_ntb()
self.delete_data_ctb()
self.delete_data_stb()
+ tdDnodes.stoptaosd(1)
+ tdDnodes.starttaosd(1)
+ self.delete_data_ntb()
def stop(self):
tdSql.close()
tdLog.success("%s successfully executed" % __file__)
diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py
index 764da1f166cb9bf149002aae27fc6d7c078be446..63317e8036b6455984231bf634b23f7238935642 100644
--- a/tests/system-test/1-insert/mutil_stage.py
+++ b/tests/system-test/1-insert/mutil_stage.py
@@ -1,4 +1,5 @@
from datetime import datetime
+from platform import platform
import time
from typing import List, Any, Tuple
@@ -83,6 +84,8 @@ class TDTestCase:
def del_old_datadir(self, filename):
cmd = f"sed -i '/^dataDir/d' {filename}"
+ if platform.system().lower() == 'darwin':
+ cmd = f"sed -i '' '/^dataDir/d' {filename}"
if os.system(cmd) != 0:
tdLog.exit(cmd)
diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py
index deff4b42a1784b5bc59466d1c067536e83c10d06..417adc39acc306fbd7b2d74518653d0347fb27cb 100644
--- a/tests/system-test/1-insert/update_data.py
+++ b/tests/system-test/1-insert/update_data.py
@@ -154,10 +154,12 @@ class TDTestCase:
up_bool = random.randint(0,100)%2
up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX)
up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300))
- binary_length = random.randint(0,self.str_length)
- nchar_length = random.randint(0,self.str_length)
- up_binary = tdCom.getLongName(binary_length)
- up_nchar = tdCom.getLongName(nchar_length)
+ binary_length = []
+ for i in range(self.str_length+1):
+ binary_length.append(i)
+ nchar_length = []
+ for i in range(self.str_length+1):
+ nchar_length.append(i)
for col_name,col_type in column_dict.items():
if tb_type == 'ntb':
tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})')
@@ -188,9 +190,13 @@ class TDTestCase:
elif col_type.lower() == 'double':
self.update_and_check_data(tbname,col_name,col_type,up_double,dbname)
elif 'binary' in col_type.lower():
- self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname)
+ for i in binary_length:
+ up_binary = tdCom.getLongName(i)
+ self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname)
elif 'nchar' in col_type.lower():
- self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname)
+ for i in nchar_length:
+ up_nchar = tdCom.getLongName(i)
+ self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname)
elif col_type.lower() == 'timestamp':
self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname)
tdSql.execute(f'insert into {tbname} values({self.ts},null)')
diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py
index 169b1c2c387c7158635483f8ce8868891e42e3c2..5cc9a2d9e888425763914c01baeababfeebc6166 100644
--- a/tests/system-test/2-query/max.py
+++ b/tests/system-test/2-query/max.py
@@ -38,18 +38,7 @@ class TDTestCase:
elif i>=9:
tdSql.checkData(0, 0, np.max(floatData))
- tdSql.query(f"select max(now()) from {dbname}.stb_1")
- tdSql.checkRows(1)
-
- tdSql.query(f"select last(ts) from {dbname}.stb_1")
- lastTs = tdSql.getData(0, 0)
- tdSql.query(f"select max(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, lastTs)
-
- tdSql.query(f"select last(ts) from {dbname}.stb")
- lastTs = tdSql.getData(0, 0)
- tdSql.query(f"select max(ts) from {dbname}.stb")
- tdSql.checkData(0, 0, lastTs)
+ tdSql.error(f"select max(now()) from {dbname}.stb_1")
tdSql.query(f"select max(col1) from {dbname}.stb_1 where col2<=5")
tdSql.checkData(0,0,5)
@@ -78,13 +67,7 @@ class TDTestCase:
elif i>=9:
tdSql.checkData(0, 0, np.max(floatData))
- tdSql.query(f"select max(now()) from {dbname}.ntb")
- tdSql.checkRows(1)
-
- tdSql.query(f"select last(ts) from {dbname}.ntb")
- lastTs = tdSql.getData(0, 0)
- tdSql.query(f"select max(ts) from {dbname}.ntb")
- tdSql.checkData(0, 0, lastTs)
+ tdSql.error(f"select max(now()) from {dbname}.ntb")
tdSql.query(f"select max(col1) from {dbname}.ntb where col2<=5")
tdSql.checkData(0,0,5)
diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py
index 01c267724210591e639753c3566c4826a5218813..08bb7675add128f94f6ab5be0b9a7a8c90e49afe 100644
--- a/tests/system-test/2-query/max_partition.py
+++ b/tests/system-test/2-query/max_partition.py
@@ -181,7 +181,7 @@ class TDTestCase:
# bug need fix
tdSql.checkData(0,1,None)
- tdSql.query(f"select c1 , twa(c1) from {dbname}.stb partition by c1 order by c1")
+ tdSql.query(f"select c1 , twa(c1) from {dbname}.sub_stb_1 partition by c1 order by c1")
tdSql.checkRows(11)
tdSql.checkData(0,1,None)
diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py
index 3d46b7b2224f834360c17cdc311dbf1e0d5a4535..d97c4340f4022af6372134892dba25c371af1b51 100644
--- a/tests/system-test/2-query/min.py
+++ b/tests/system-test/2-query/min.py
@@ -37,6 +37,8 @@ class TDTestCase:
floatData.append(i + 0.1)
# max verifacation
+ tdSql.error(f"select min(now()) from {dbname}.stb_1")
+ tdSql.error(f"select min(ts) from {dbname}.stb_1")
tdSql.error(f"select min(col7) from {dbname}.stb_1")
tdSql.error(f"select min(col8) from {dbname}.stb_1")
tdSql.error(f"select min(col9) from {dbname}.stb_1")
@@ -67,20 +69,9 @@ class TDTestCase:
tdSql.query(f"select min(col1) from {dbname}.stb_1 where col2>=5")
tdSql.checkData(0,0,5)
- tdSql.query(f"select min(now()) from {dbname}.stb_1")
- tdSql.checkRows(1)
-
- tdSql.query(f"select first(ts) from {dbname}.stb_1")
- firstTs = tdSql.getData(0, 0)
- tdSql.query(f"select min(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, firstTs)
-
- tdSql.query(f"select first(ts) from {dbname}.stb_1")
- firstTs = tdSql.getData(0, 0)
- tdSql.query(f"select min(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, firstTs)
-
+ tdSql.error(f"select min(now()) from {dbname}.stb_1")
+ tdSql.error(f"select min(ts) from {dbname}.stb_1")
tdSql.error(f"select min(col7) from {dbname}.stb_1")
tdSql.error(f"select min(col8) from {dbname}.stb_1")
tdSql.error(f"select min(col9) from {dbname}.stb_1")
@@ -111,19 +102,8 @@ class TDTestCase:
tdSql.query(f"select min(col1) from {dbname}.stb where col2>=5")
tdSql.checkData(0,0,5)
- tdSql.query(f"select min(now()) from {dbname}.stb_1")
- tdSql.checkRows(1)
-
- tdSql.query(f"select first(ts) from {dbname}.stb_1")
- firstTs = tdSql.getData(0, 0)
- tdSql.query(f"select min(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, firstTs)
-
- tdSql.query(f"select first(ts) from {dbname}.stb_1")
- firstTs = tdSql.getData(0, 0)
- tdSql.query(f"select min(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, firstTs)
-
+ tdSql.error(f"select min(now()) from {dbname}.stb_1")
+ tdSql.error(f"select min(ts) from {dbname}.stb_1")
tdSql.error(f"select min(col7) from {dbname}.ntb")
tdSql.error(f"select min(col8) from {dbname}.ntb")
tdSql.error(f"select min(col9) from {dbname}.ntb")
@@ -154,19 +134,6 @@ class TDTestCase:
tdSql.query(f"select min(col1) from {dbname}.ntb where col2>=5")
tdSql.checkData(0,0,5)
- tdSql.query(f"select min(now()) from {dbname}.stb_1")
- tdSql.checkRows(1)
-
- tdSql.query(f"select first(ts) from {dbname}.stb_1")
- firstTs = tdSql.getData(0, 0)
- tdSql.query(f"select min(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, firstTs)
-
- tdSql.query(f"select first(ts) from {dbname}.stb_1")
- firstTs = tdSql.getData(0, 0)
- tdSql.query(f"select min(ts) from {dbname}.stb_1")
- tdSql.checkData(0, 0, firstTs)
-
def stop(self):
tdSql.close()
diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py
index c26393158cefe46fb054d7bd3e28a621cab73199..187df2e2d33846a2e61a7519cdec8c13b352f9da 100644
--- a/tests/system-test/2-query/ttl_comment.py
+++ b/tests/system-test/2-query/ttl_comment.py
@@ -137,7 +137,8 @@ class TDTestCase:
tdSql.execute(f"create table {dbname}.child_table2 using {dbname}.super_table1 tags(1) comment ''")
tdSql.execute(f"create table {dbname}.child_table3 using {dbname}.super_table1 tags(1) comment 'child'")
tdSql.execute(f"insert into {dbname}.child_table4 using {dbname}.super_table1 tags(1) values(now, 1)")
-
+ tdSql.execute(f"insert into {dbname}.child_table5 using {dbname}.super_table1 tags(1) ttl 23 comment '' values(now, 1)")
+ tdSql.error(f"insert into {dbname}.child_table6 using {dbname}.super_table1 tags(1) ttl -23 comment '' values(now, 1)")
tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table1'")
tdSql.checkData(0, 0, 'child_table1')
@@ -160,6 +161,11 @@ class TDTestCase:
tdSql.checkData(0, 7, 0)
tdSql.checkData(0, 8, None)
+ tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table5'")
+ tdSql.checkData(0, 0, 'child_table5')
+ tdSql.checkData(0, 7, 23)
+ tdSql.checkData(0, 8, '')
+
tdSql.execute(f"alter table {dbname}.child_table1 comment 'nihao'")
tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table1'")
diff --git a/tests/system-test/2-query/twa.py b/tests/system-test/2-query/twa.py
index 62940477cf701d69e8c8e7568ae4b56d68518d81..4c163da485f2790e15fa098b707c67f7ba95360b 100644
--- a/tests/system-test/2-query/twa.py
+++ b/tests/system-test/2-query/twa.py
@@ -124,7 +124,7 @@ class TDTestCase:
tdSql.checkData(0,1,4.500000000)
# mixup with other functions
- tdSql.query(f"select twa(c1),twa(c2),max(c1),elapsed(ts) from {dbname}.stb1 ")
+ tdSql.query(f"select twa(c1),twa(c2),max(c1),elapsed(ts) from {dbname}.ct1 ")
tdSql.checkData(0,0,1.000000000)
tdSql.checkData(0,1,11111.000000000)
tdSql.checkData(0,2,1)
diff --git a/tests/system-test/7-tmq/tmqDelete-multiCtb.py b/tests/system-test/7-tmq/tmqDelete-multiCtb.py
index e59040305a4dbd43a1bb74012c9c6f295770352a..c9d90b45bee55119ecce1f8950ec02461e769c97 100644
--- a/tests/system-test/7-tmq/tmqDelete-multiCtb.py
+++ b/tests/system-test/7-tmq/tmqDelete-multiCtb.py
@@ -325,7 +325,9 @@ class TDTestCase:
expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4))
elif self.snapshot == 1:
consumerId = 5
- expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))
+ # expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4))
+ # fix case: sometimes only 200 rows are deleted
+ expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4))
topicList = topicFromStb1
ifcheckdata = 1
diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh
index 11df13d451f1d63a116fdceea4757da032e1a9ec..0b45b2425826bd6f43942afb635e8fe548bb6a50 100755
--- a/tests/system-test/fulltest.sh
+++ b/tests/system-test/fulltest.sh
@@ -2,7 +2,7 @@
set -e
set -x
-python3 ./test.py -f 0-others/taosShell.py
+#python3 ./test.py -f 0-others/taosShell.py
python3 ./test.py -f 0-others/taosShellError.py
python3 ./test.py -f 0-others/taosShellNetChk.py
python3 ./test.py -f 0-others/telemetry.py
@@ -333,7 +333,7 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py
python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py
python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py
python3 ./test.py -f 7-tmq/tmq_taosx.py
-# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
+python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py
#------------querPolicy 2-----------