diff --git a/cmake/cmake.define b/cmake/cmake.define index 1500858d9f541a061803c7bc76c514927813d5a2..f55a9bdabc79e31f129b2184144c9472572d5454 100644 --- a/cmake/cmake.define +++ b/cmake/cmake.define @@ -1,6 +1,6 @@ cmake_minimum_required(VERSION 3.0) -set(CMAKE_VERBOSE_MAKEFILE ON) +set(CMAKE_VERBOSE_MAKEFILE OFF) set(TD_BUILD_TAOSA_INTERNAL FALSE) #set output directory diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index 4a8f4864b301101fd04214674069e8c3196ba49d..c67918351de7b25d18403314886bd5227f8fa6c6 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG ae8d51c + GIT_TAG 565ca21 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/01-index.md b/docs/en/01-index.md index 296cd968986d5dd7738d6c3b997901614f893763..9fe5e7800abfeaac55c14315bd4fd825b27620b8 100644 --- a/docs/en/01-index.md +++ b/docs/en/01-index.md @@ -5,7 +5,7 @@ description: This website contains the user manuals for TDengine, an open-source slug: / --- -TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) [time-series database](https://tdengine.com/tsdb/) optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It’s written mainly for architects, developers, and system administrators. +TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) [time-series database](https://tdengine.com/tsdb/) optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It's written mainly for architects, developers, and system administrators. To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section. diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index 9324ca9f7e4e3a29fc0e20b49b0c35ee531ec38e..4d65c86371760c9dcc9cb1d10b887bfbdc66f26d 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -57,7 +57,7 @@ By making full use of [characteristics of time series data](https://tdengine.com - **[Easy Data Analytics](https://tdengine.com/tdengine/time-series-data-analytics-made-easy/)**: Through super tables, storage and compute separation, data partitioning by time interval, pre-computation and other means, TDengine makes it easy to explore, format, and get access to data in a highly efficient way. -- **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered over 19k stars on GitHub. There is an active developer community, and over 140k running instances worldwide. +- **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine's core modules, including cluster feature, are all available under open source licenses. It has gathered over 19k stars on GitHub. There is an active developer community, and over 140k running instances worldwide. With TDengine, the total cost of ownership of your time-series data platform can be greatly reduced. @@ -109,8 +109,8 @@ As a high-performance, scalable and SQL supported time-series database, TDengine | **System Performance Requirements** | **Not Applicable** | **Might Be Applicable** | **Very Applicable** | **Description** | | ------------------------------------------------- | ------------------ | ----------------------- | ------------------- | --------------------------------------------------------------------------------------------------------------------------- | -| Very large total processing capacity | | | √ | TDengine’s cluster functions can easily improve processing capacity via multi-server coordination. | -| Extremely high-speed data processing | | | √ | TDengine’s storage and data processing are optimized for IoT, and can process data many times faster than similar products. | +| Very large total processing capacity | | | √ | TDengine's cluster functions can easily improve processing capacity via multi-server coordination. | +| Extremely high-speed data processing | | | √ | TDengine's storage and data processing are optimized for IoT, and can process data many times faster than similar products. | | Extremely fast processing of high resolution data | | | √ | TDengine has achieved the same or better performance than other relational and NoSQL data processing systems. | ### System Maintenance Requirements diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md index 771a061c310bd53c485394fe3a8897567b12e48b..90f933eb2dd13e667520b39106821bae22e50ed5 100644 --- a/docs/en/04-concept/index.md +++ b/docs/en/04-concept/index.md @@ -127,7 +127,7 @@ To make full use of time-series data characteristics, TDengine adopts a strategy If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. **One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.** -TDengine suggests using DCP ID as the table name (like d1001 in the above table). Each DCP may collect one or multiple metrics (like the `current`, `voltage`, `phase` as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the timestamp as the index, and won’t build the index on any metrics stored. Column wise storage is used. +TDengine suggests using DCP ID as the table name (like d1001 in the above table). Each DCP may collect one or multiple metrics (like the `current`, `voltage`, `phase` as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the timestamp as the index, and won't build the index on any metrics stored. Column wise storage is used. Complex devices, such as connected cars, may have multiple DCPs. In this case, multiple tables are created for a single device, one table per DCP. diff --git a/docs/en/07-develop/01-connect/_connect_java.mdx b/docs/en/07-develop/01-connect/_connect_java.mdx index 880d2aa3e489566203fa0f4b8379feb653a98f73..538e27fcc33d888032baecadcedcbc4f42e908e3 100644 --- a/docs/en/07-develop/01-connect/_connect_java.mdx +++ b/docs/en/07-develop/01-connect/_connect_java.mdx @@ -12,4 +12,4 @@ When using REST connection, the feature of bulk pulling can be enabled if the si {{#include docs/examples/java/src/main/java/com/taos/example/WSConnectExample.java:main}} ``` -More configuration about connection，please refer to [Java Connector](/reference/connector/java) +More configuration about connection, please refer to [Java Connector](/reference/connector/java) diff --git a/docs/en/07-develop/01-connect/_connect_php.mdx b/docs/en/07-develop/01-connect/_connect_php.mdx index dbad72bc1988bd5336f1da132dd9e6ba9b8020e6..d01d0c54310fb290bd8980f63a7c3283441f4d21 100644 --- a/docs/en/07-develop/01-connect/_connect_php.mdx +++ b/docs/en/07-develop/01-connect/_connect_php.mdx @@ -1,3 +1,3 @@ -```php title="原生连接" +```php title=""native" {{#include docs/examples/php/connect.php}} ``` diff --git a/docs/en/07-develop/01-connect/index.md b/docs/en/07-develop/01-connect/index.md index 8cf3c463afe663c18a1765f41fa449fc19c05399..76c26bae23a0f546ccd6fbd14f66a5942909f4c2 100644 --- a/docs/en/07-develop/01-connect/index.md +++ b/docs/en/07-develop/01-connect/index.md @@ -33,7 +33,7 @@ There are two ways for a connector to establish connections to TDengine: For REST and native connections, connectors provide similar APIs for performing operations and running SQL statements on your databases. The main difference is the method of establishing the connection, which is not visible to users. -Key differences： +Key differences: 3. The REST connection is more accessible with cross-platform support, however it results in a 30% performance downgrade. 1. The TDengine client driver (taosc) has the highest performance with all the features of TDengine like [Parameter Binding](/reference/connector/cpp#parameter-binding-api), [Subscription](/reference/connector/cpp#subscription-and-consumption-api), etc. @@ -198,7 +198,7 @@ The sample code below are based on dotnet6.0, they may need to be adjusted if yo 1. Download [taos-jdbcdriver-version-dist.jar](https://repo1.maven.org/maven2/com/taosdata/jdbc/taos-jdbcdriver/3.0.0/). -2. Install the dependency package `RJDBC`： +2. Install the dependency package `RJDBC`: ```R install.packages("RJDBC") @@ -213,7 +213,7 @@ If the client driver (taosc) is already installed, then the C connector is alrea -**Download Source Code Package and Unzip：** +**Download Source Code Package and Unzip: ** ```shell curl -L -o php-tdengine.tar.gz https://github.com/Yurunsoft/php-tdengine/archive/refs/tags/v1.0.2.tar.gz \ @@ -223,13 +223,13 @@ curl -L -o php-tdengine.tar.gz https://github.com/Yurunsoft/php-tdengine/archive > Version number `v1.0.2` is only for example, it can be replaced to any newer version, please check available version from [TDengine PHP Connector Releases](https://github.com/Yurunsoft/php-tdengine/releases). -**Non-Swoole Environment：** +**Non-Swoole Environment: ** ```shell phpize && ./configure && make -j && make install ``` -**Specify TDengine Location：** +**Specify TDengine Location: ** ```shell phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/3.0.0.0 && make -j && make install @@ -238,7 +238,7 @@ phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/3.0.0.0 && > `--with-tdengine-dir=` is followed by the TDengine installation location. > This way is useful in case TDengine location can't be found automatically or macOS. -**Swoole Environment：** +**Swoole Environment: ** ```shell phpize && ./configure --enable-swoole && make -j && make install diff --git a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx index c559883d26c7ef5750f3ee1f828912f8d800c3a4..4c850532df6908c2b57e51929b009d34b7e7dd8d 100644 --- a/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx +++ b/docs/en/07-develop/03-insert-data/30-influxdb-line.mdx @@ -69,7 +69,7 @@ For more details please refer to [InfluxDB Line Protocol](https://docs.influxdat ## Query Examples -If you want query the data of `location=California.LosAngeles,groupid=2`，here is the query SQL: +If you want query the data of `location=California.LosAngeles,groupid=2`, here is the query SQL: ```sql SELECT * FROM meters WHERE location = "California.LosAngeles" AND groupid = 2; diff --git a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx index 30bc3b87bf035f563e1a03508a90466e9288f89c..78ced058bf4cd65d84ad2342141be4cc6a1dd1cd 100644 --- a/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx +++ b/docs/en/07-develop/03-insert-data/40-opentsdb-telnet.mdx @@ -84,7 +84,7 @@ Query OK, 4 row(s) in set (0.005399s) ## Query Examples -If you want query the data of `location=California.LosAngeles groupid=3`，here is the query SQL: +If you want query the data of `location=California.LosAngeles groupid=3`, here is the query SQL: ```sql SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3; diff --git a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx index e9db130241236e7224e5965c4c3c651ddab8cb00..b63bf0d93f141fe5117b9547a2f834015d52d246 100644 --- a/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx +++ b/docs/en/07-develop/03-insert-data/50-opentsdb-json.mdx @@ -97,7 +97,7 @@ Query OK, 2 row(s) in set (0.004076s) ## Query Examples -If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1}，here is the query SQL: +If you want query the data of "tags": {"location": "California.LosAngeles", "groupid": 1}, here is the query SQL: ```sql SELECT * FROM `meters.current` WHERE location = "California.LosAngeles" AND groupid = 3; diff --git a/docs/en/07-develop/03-insert-data/60-high-volume.md b/docs/en/07-develop/03-insert-data/60-high-volume.md index d6a00481383035c0babc6fab94e8359b4806ff72..c15b3213bbf38ba48447f6ecee699d70af91478a 100644 --- a/docs/en/07-develop/03-insert-data/60-high-volume.md +++ b/docs/en/07-develop/03-insert-data/60-high-volume.md @@ -49,7 +49,7 @@ If the data source is Kafka, then the application program is a consumer of Kafka On the server side, database configuration parameter `vgroups` needs to be set carefully to maximize the system performance. If it's set too low, the system capability can't be utilized fully; if it's set too big, unnecessary resource competition may be produced. A normal recommendation for `vgroups` parameter is 2 times of the number of CPU cores. However, depending on the actual system resources, it may still need to tuned. -For more configuration parameters, please refer to [Database Configuration](../../../taos-sql/database) and [Server Configuration](../../../reference/config)。 +For more configuration parameters, please refer to [Database Configuration](../../../taos-sql/database) and [Server Configuration](../../../reference/config). ## Sample Programs @@ -98,7 +98,7 @@ The main Program is responsible for: 3. Start reading threads 4. Output writing speed every 10 seconds -The main program provides 4 parameters for tuning： +The main program provides 4 parameters for tuning: 1. The number of reading threads, default value is 1 2. The number of writing threads, default value is 2 @@ -192,7 +192,7 @@ TDENGINE_JDBC_URL="jdbc:TAOS://localhost:6030?user=root&password=taosdata" If you want to launch the sample program on a remote server, please follow below steps: -1. Package the sample programs. Execute below command under directory `TDengine/docs/examples/java` ： +1. Package the sample programs. Execute below command under directory `TDengine/docs/examples/java`: ``` mvn package ``` @@ -385,7 +385,7 @@ SQLWriter class encapsulates the logic of composing SQL and writing data. Please pip3 install faster-fifo ``` -3. Click the "Copy" in the above sample programs to copy `fast_write_example.py` 、 `sql_writer.py` and `mockdatasource.py`. +3. Click the "Copy" in the above sample programs to copy `fast_write_example.py`, `sql_writer.py`, and `mockdatasource.py`. 4. Execute the program diff --git a/docs/en/07-develop/03-insert-data/_py_kafka.mdx b/docs/en/07-develop/03-insert-data/_py_kafka.mdx index c71821dad129fe1a573e6f607eb591f81f928ae1..bacad0e8a91f44c8db7589d7ab5d10bce245922c 100644 --- a/docs/en/07-develop/03-insert-data/_py_kafka.mdx +++ b/docs/en/07-develop/03-insert-data/_py_kafka.mdx @@ -1,4 +1,4 @@ -### python Kafka 客户端 +### python Kafka client For python kafka client, please refer to [kafka client](https://cwiki.apache.org/confluence/display/KAFKA/Clients#Clients-Python). In this document, we use [kafka-python](http://github.com/dpkp/kafka-python). @@ -88,7 +88,7 @@ In addition to python's built-in multithreading and multiprocessing library, we
kafka_example_consumer -`kafka_example_consumer` is `consumer`，which is responsible for consuming data from kafka and writing it to TDengine. +`kafka_example_consumer` is `consumer`, which is responsible for consuming data from kafka and writing it to TDengine. ```py {{#include docs/examples/python/kafka_example_consumer.py}} diff --git a/docs/en/07-develop/04-query-data/index.mdx b/docs/en/07-develop/04-query-data/index.mdx index 7e167bb4f342167c486403259985976357e44496..5a05d22d854d7accf37c8e533bc350e7a2d0f998 100644 --- a/docs/en/07-develop/04-query-data/index.mdx +++ b/docs/en/07-develop/04-query-data/index.mdx @@ -20,10 +20,10 @@ import CAsync from "./_c_async.mdx"; ## Introduction -SQL is used by TDengine as its query language. Application programs can send SQL statements to TDengine through REST API or connectors. TDengine's CLI `taos` can also be used to execute ad hoc SQL queries. Here is the list of major query functionalities supported by TDengine： +SQL is used by TDengine as its query language. Application programs can send SQL statements to TDengine through REST API or connectors. TDengine's CLI `taos` can also be used to execute ad hoc SQL queries. Here is the list of major query functionalities supported by TDengine: - Query on single column or multiple columns -- Filter on tags or data columns：>, <, =, <\>, like +- Filter on tags or data columns: >, <, =, <\>, like - Grouping of results: `Group By` - Sorting of results: `Order By` - Limit the number of results: `Limit/Offset` - Windowed aggregate queries for time windows (interval), session windows (session), and state windows (state_window) - Arithmetic on columns of numeric types or aggregate results @@ -160,7 +160,7 @@ In the section describing [Insert](/develop/insert-data/sql-writing), a database :::note 1. With either REST connection or native connection, the above sample code works well. -2. Please note that `use db` can't be used in case of REST connection because it's stateless. +2. Please note that `use db` can't be used in case of REST connection because it's stateless. You can specify the database name by either the REST endpoint's parameter or . in the SQL command. ::: diff --git a/docs/en/07-develop/09-udf.md b/docs/en/07-develop/09-udf.md index a7753647e3e7733e94b944b4e50047424f3bbf9b..f107512e9c14459453d6424e6f05770bf510bdde 100644 --- a/docs/en/07-develop/09-udf.md +++ b/docs/en/07-develop/09-udf.md @@ -252,9 +252,9 @@ create table battery(ts timestamp, vol1 float, vol2 float, vol3 float, deviceId ``` Create the UDF: ```bash -create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; +create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; ``` -Use the UDF in the query： +Use the UDF in the query: ```bash select max_vol(vol1,vol2,vol3,deviceid) from battery; ``` @@ -271,9 +271,9 @@ select max_vol(vol1,vol2,vol3,deviceid) from battery; ## Implement a UDF in Python Implement the specified interface functions when implementing a UDF in Python. -- implement `process` function for the scalar UDF。 -- implement `start`, `reduce`, `finish` for the aggregate UDF。 -- implement `init` for initialization and `destroy` for termination。 +- implement `process` function for the scalar UDF. +- implement `start`, `reduce`, `finish` for the aggregate UDF. +- implement `init` for initialization and `destroy` for termination. ### Implement a Scalar UDF in Python diff --git a/docs/en/12-taos-sql/01-data-type.md b/docs/en/12-taos-sql/01-data-type.md index 641fd3cbb7c4e2937cc1756a41a522f6eb4b51aa..cca256139d409ac009bb188ccb377bffc0dd42ef 100644 --- a/docs/en/12-taos-sql/01-data-type.md +++ b/docs/en/12-taos-sql/01-data-type.md @@ -11,7 +11,7 @@ When using TDengine to store and query data, the most important part of the data - The format must be `YYYY-MM-DD HH:mm:ss.MS`, the default time precision is millisecond (ms), for example `2017-08-12 18:25:58.128`. - Internal function `NOW` can be used to get the current timestamp on the client side. - The current timestamp of the client side is applied when `NOW` is used to insert data. -- Epoch Time：timestamp can also be a long integer number, which means the number of seconds, milliseconds or nanoseconds, depending on the time precision, from UTC 1970-01-01 00:00:00. +- Epoch Time: timestamp can also be a long integer number, which means the number of seconds, milliseconds or nanoseconds, depending on the time precision, from UTC 1970-01-01 00:00:00. - Add/subtract operations can be carried out on timestamps. For example `NOW-2h` means 2 hours prior to the time at which query is executed. The units of time in operations can be b(nanosecond), u(microsecond), a(millisecond), s(second), m(minute), h(hour), d(day), or w(week). So `SELECT * FROM t1 WHERE ts > NOW-2w AND ts <= NOW-1w` means the data between two weeks ago and one week ago. The time unit can also be n (calendar month) or y (calendar year) when specifying the time window for down sampling operations. Time precision in TDengine can be set by the `PRECISION` parameter when executing `CREATE DATABASE`. The default time precision is millisecond. In the statement below, the precision is set to nanonseconds. @@ -24,24 +24,24 @@ CREATE DATABASE db_name PRECISION 'ns'; In TDengine, the data types below can be used when specifying a column or tag. -| # | **type** | **Bytes** | **Description** | -| --- | :--------------: | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | -| 1 | TIMESTAMP | 8 | Default precision is millisecond, microsecond and nanosecond are also supported. | -| 2 | INT | 4 | Integer, the value range is [-2^31, 2^31-1]. | -| 3 | INT UNSIGNED | 4 | Unsigned integer, the value range is [0, 2^32-1]. | -| 4 | BIGINT | 8 | Long integer, the value range is [-2^63, 2^63-1]. | -| 5 | BIGINT UNSIGNED | 8 | unsigned long integer, the value range is [0, 2^64-1]. | -| 6 | FLOAT | 4 | Floating point number, the effective number of digits is 6-7, the value range is [-3.4E38, 3.4E38]. | -| 7 | DOUBLE | 8 | Double precision floating point number, the effective number of digits is 15-16, the value range is [-1.7E308, 1.7E308]. | -| 8 | BINARY | User Defined | Single-byte string for ASCII visible characters. Length must be specified when defining a column or tag of binary type. | -| 9 | SMALLINT | 2 | Short integer, the value range is [-32768, 32767]. | -| 10 | INT UNSIGNED | 2 | unsigned integer, the value range is [0, 65535]. | -| 11 | TINYINT | 1 | Single-byte integer, the value range is [-128, 127]. | -| 12 | TINYINT UNSIGNED | 1 | unsigned single-byte integer, the value range is [0, 255]. | -| 13 | BOOL | 1 | Bool, the value range is {true, false}. | -| 14 | NCHAR | User Defined | Multi-byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\'`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. | -| 15 | JSON | | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type. | -| 16 | VARCHAR | User-defined | Alias of BINARY | +| # | **type** | **Bytes** | **Description** | +| --- | :---------------: | ------------ | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | +| 1 | TIMESTAMP | 8 | Default precision is millisecond, microsecond and nanosecond are also supported. | +| 2 | INT | 4 | Integer, the value range is [-2^31, 2^31-1]. | +| 3 | INT UNSIGNED | 4 | Unsigned integer, the value range is [0, 2^32-1]. | +| 4 | BIGINT | 8 | Long integer, the value range is [-2^63, 2^63-1]. | +| 5 | BIGINT UNSIGNED | 8 | unsigned long integer, the value range is [0, 2^64-1]. | +| 6 | FLOAT | 4 | Floating point number, the effective number of digits is 6-7, the value range is [-3.4E38, 3.4E38]. | +| 7 | DOUBLE | 8 | Double precision floating point number, the effective number of digits is 15-16, the value range is [-1.7E308, 1.7E308]. | +| 8 | BINARY | User Defined | Single-byte string for ASCII visible characters. Length must be specified when defining a column or tag of binary type. | +| 9 | SMALLINT | 2 | Short integer, the value range is [-32768, 32767]. | +| 10 | SMALLINT UNSIGNED | 2 | unsigned integer, the value range is [0, 65535]. | +| 11 | TINYINT | 1 | Single-byte integer, the value range is [-128, 127]. | +| 12 | TINYINT UNSIGNED | 1 | unsigned single-byte integer, the value range is [0, 255]. | +| 13 | BOOL | 1 | Bool, the value range is {true, false}. | +| 14 | NCHAR | User Defined | Multi-byte string that can include multi byte characters like Chinese characters. Each character of NCHAR type consumes 4 bytes storage. The string value should be quoted with single quotes. Literal single quote inside the string must be preceded with backslash, like `\'`. The length must be specified when defining a column or tag of NCHAR type, for example nchar(10) means it can store at most 10 characters of nchar type and will consume fixed storage of 40 bytes. An error will be reported if the string value exceeds the length defined. | +| 15 | JSON | | JSON type can only be used on tags. A tag of json type is excluded with any other tags of any other type. | +| 16 | VARCHAR | User-defined | Alias of BINARY | :::note diff --git a/docs/en/12-taos-sql/02-database.md b/docs/en/12-taos-sql/02-database.md index 8c579622c9f8040d3eec61f540f6e50d37bb3ddf..af619c11a5c5ca1e72b7ee6edc4b3106da1e0f1c 100644 --- a/docs/en/12-taos-sql/02-database.md +++ b/docs/en/12-taos-sql/02-database.md @@ -72,8 +72,8 @@ database_option: { - 0: The database can contain multiple supertables. - 1: The database can contain only one supertable. - STT_TRIGGER: specifies the number of file merges triggered by flushed files. The default is 8, ranging from 1 to 16. For high-frequency scenarios with few tables, it is recommended to use the default configuration or a smaller value for this parameter; For multi-table low-frequency scenarios, it is recommended to configure this parameter with a larger value. -- TABLE_PREFIX： The prefix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the prefix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "0001" is used if TSDB_PREFIX is set to 2 but "v3" is used if TSDB_PREFIX is set to -2; It can help you to control the distribution of tables. -- TABLE_SUFFIX：The suffix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the suffix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "v300" is used if TSDB_SUFFIX is set to 2 but "01" is used if TSDB_SUFFIX is set to -2; It can help you to control the distribution of tables. +- TABLE_PREFIX: The prefix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the prefix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "0001" is used if TSDB_PREFIX is set to 2 but "v3" is used if TSDB_PREFIX is set to -2; It can help you to control the distribution of tables. +- TABLE_SUFFIX: The suffix in the table name that is ignored when distributing a table to a vgroup when it's a positive number, or only the suffix is used when distributing a table to a vgroup, the default value is 0; For example, if the table name v30001, then "v300" is used if TSDB_SUFFIX is set to 2 but "01" is used if TSDB_SUFFIX is set to -2; It can help you to control the distribution of tables. - TSDB_PAGESIZE: The page size of the data storage engine in a vnode. The unit is KB. The default is 4 KB. The range is 1 to 16384, that is, 1 KB to 16 MB. - WAL_RETENTION_PERIOD: specifies the maximum time of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a time in seconds. The default value 0. A value of 0 indicates that WAL files are not required to keep for consumption. Alter it with a proper value at first to create topics. - WAL_RETENTION_SIZE: specifies the maximum total size of which WAL files are to be kept for consumption. This parameter is used for data subscription. Enter a size in KB. The default value is 0. A value of 0 indicates that the total size of WAL files to keep for consumption has no upper limit. diff --git a/docs/en/12-taos-sql/05-insert.md b/docs/en/12-taos-sql/05-insert.md index c22357abfa1c9e2a4dece880c407cb4bbfdaa055..62c3e4ad8ef10e498adad2ab2afd1004e07d6baa 100644 --- a/docs/en/12-taos-sql/05-insert.md +++ b/docs/en/12-taos-sql/05-insert.md @@ -82,7 +82,7 @@ One or multiple rows can be inserted into multiple tables in a single SQL statem ```sql INSERT INTO d1001 VALUES ('2021-07-13 14:06:34.630', 10.2, 219, 0.32) ('2021-07-13 14:06:35.779', 10.15, 217, 0.33) - d1002 (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31）; + d1002 (ts, current, phase) VALUES ('2021-07-13 14:06:34.255', 10.27, 0.31); ``` ## Automatically Create Table When Inserting diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 070fd416536856d54859c37fcf34392f2887b7e5..ea0d7abc16b18a92147642d86b167d96903ebfda 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -373,7 +373,7 @@ FROM temp_stable t1, temp_stable t2 WHERE t1.ts = t2.ts AND t1.deviceid = t2.deviceid AND t1.status=0; ``` -For sub-table and super table： +For sub-table and super table: ```sql SELECT * diff --git a/docs/en/12-taos-sql/07-tag-index.md b/docs/en/12-taos-sql/07-tag-index.md index af1d0a352ec8950e49a1269c5b2192228fd35d32..7236002d80d10007b635837b603f38ea3153da87 100644 --- a/docs/en/12-taos-sql/07-tag-index.md +++ b/docs/en/12-taos-sql/07-tag-index.md @@ -6,14 +6,14 @@ description: Use Tag Index to Improve Query Performance ## Introduction -Prior to TDengine 3.0.3.0 (excluded)，only one index is created by default on the first tag of each super table, but it's not allowed to dynamically create index on any other tags. From version 3.0.30, you can dynamically create index on any tag of any type. The index created automatically by TDengine is still valid. Query performance can benefit from indexes if you use properly. +Prior to TDengine 3.0.3.0 (excluded), only one index is created by default on the first tag of each super table, but it's not allowed to dynamically create index on any other tags. From version 3.0.30, you can dynamically create index on any tag of any type. The index created automatically by TDengine is still valid. Query performance can benefit from indexes if you use properly. ## Syntax 1. The syntax of creating an index ```sql -CREATE INDEX index_name ON tbl_name (tagColName） +CREATE INDEX index_name ON tbl_name (tagColName) ``` In the above statement, `index_name` if the name of the index, `tbl_name` is the name of the super table,`tagColName` is the name of the tag on which the index is being created. `tagColName` can be any type supported by TDengine. diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 5c0ad8a71df471ce3481cad4895886b835592c77..7e45ffa1df1f8b9814daafeee7a28016eca911e4 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -434,7 +434,7 @@ TO_ISO8601(expr [, timezone]) **More explanations**: -- You can specify a time zone in the following format: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。 For example, TO_ISO8601(1, "+00:00"). +- You can specify a time zone in the following format: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]. For example, TO_ISO8601(1, "+00:00"). - If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp - If the input is a column of TIMESTAMP type, the precision of the returned value is same as the precision set for the current data base in use @@ -626,7 +626,7 @@ algo_type: { **Applicable table types**: standard tables and supertables -**Explanations**： +**Explanations**: - _p_ is in range [0,100], when _p_ is 0, the result is same as using function MIN; when _p_ is 100, the result is same as function MAX. - `algo_type` can only be input as `default` or `t-digest` Enter `default` to use a histogram-based algorithm. Enter `t-digest` to use the t-digest algorithm to calculate the approximation of the quantile. `default` is used by default. - The approximation result of `t-digest` algorithm is sensitive to input data order. For example, when querying STable with different input data order there might be minor differences in calculated results. @@ -672,7 +672,7 @@ If you input a specific column, the number of non-null values in the column is r ELAPSED(ts_primary_key [, time_unit]) ``` -**Description**：`elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calculated time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. +**Description**: `elapsed` function can be used to calculate the continuous time length in which there is valid data. If it's used with `INTERVAL` clause, the returned result is the calculated time length within each time window. If it's used without `INTERVAL` caluse, the returned result is the calculated time length within the specified time range. Please be noted that the return value of `elapsed` is the number of `time_unit` in the calculated time length. **Return value type**: Double if the input value is not NULL; @@ -680,7 +680,7 @@ ELAPSED(ts_primary_key [, time_unit]) **Applicable tables**: table, STable, outer in nested query -**Explanations**： +**Explanations**: - `ts_primary_key` parameter can only be the first column of a table, i.e. timestamp primary key. - The minimum value of `time_unit` is the time precision of the database. If `time_unit` is not specified, the time precision of the database is used as the default time unit. Time unit specified by `time_unit` can be: 1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks) @@ -758,7 +758,7 @@ SUM(expr) HYPERLOGLOG(expr) ``` -**Description**： +**Description**: The cardinal number of a specific column is returned by using hyperloglog algorithm. The benefit of using hyperloglog algorithm is that the memory usage is under control when the data volume is huge. However, when the data volume is very small, the result may be not accurate, it's recommended to use `select count(data) from (select unique(col) as data from table)` in this case. @@ -772,10 +772,10 @@ HYPERLOGLOG(expr) ### HISTOGRAM ```sql -HISTOGRAM(expr，bin_type, bin_description, normalized) +HISTOGRAM(expr, bin_type, bin_description, normalized) ``` -**Description**：Returns count of data points in user-specified ranges. +**Description**: Returns count of data points in user-specified ranges. **Return value type** If normalized is set to 1, a DOUBLE is returned; otherwise a BIGINT is returned @@ -783,18 +783,18 @@ HISTOGRAM(expr，bin_type, bin_description, normalized) **Applicable table types**: table, STable -**Explanations**： -- bin_type: parameter to indicate the bucket type, valid inputs are: "user_input", "linear_bin", "log_bin"。 -- bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively: +**Explanations**: +- bin_type: parameter to indicate the bucket type, valid inputs are: "user_input", "linear_bin", "log_bin". +- bin_description: parameter to describe how to generate buckets can be in the following JSON formats for each bin_type respectively: - "user_input": "[1, 3, 5, 7]": User specified bin values. - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" - "start" - bin starting point. "width" - bin offset. "count" - number of bins generated. "infinity" - whether to add（-inf, inf）as start/end point in generated set of bins. + "start" - bin starting point. "width" - bin offset. "count" - number of bins generated. "infinity" - whether to add (-inf, inf) as start/end point in generated set of bins. The above "linear_bin" descriptor generates a set of bins: [-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]. - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" - "start" - bin starting point. "factor" - exponential factor of bin offset. "count" - number of bins generated. "infinity" - whether to add（-inf, inf）as start/end point in generated range of bins. + "start" - bin starting point. "factor" - exponential factor of bin offset. "count" - number of bins generated. "infinity" - whether to add (-inf, inf) as start/end point in generated range of bins. The above "linear_bin" descriptor generates a set of bins: [-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]. - normalized: setting to 1/0 to turn on/off result normalization. Valid values are 0 or 1. @@ -1107,7 +1107,7 @@ ignore_negative: { **More explanation**: - It can be used together with `PARTITION BY tbname` against a STable. -- It can be used together with a selected column. For example: select \_rowts, DERIVATIVE() from。 +- It can be used together with a selected column. For example: select \_rowts, DERIVATIVE() from. ### DIFF @@ -1131,7 +1131,7 @@ ignore_negative: { **More explanation**: - The number of result rows is the number of rows subtracted by one, no output for the first row -- It can be used together with a selected column. For example: select \_rowts, DIFF() from。 +- It can be used together with a selected column. For example: select \_rowts, DIFF() from. ### IRATE @@ -1183,7 +1183,7 @@ STATECOUNT(expr, oper, val) **Applicable parameter values**: - oper : Can be one of `'LT'` (lower than), `'GT'` (greater than), `'LE'` (lower than or equal to), `'GE'` (greater than or equal to), `'NE'` (not equal to), `'EQ'` (equal to), the value is case insensitive, the value must be in quotes. -- val ： Numeric types +- val: Numeric types **Return value type**: Integer @@ -1210,7 +1210,7 @@ STATEDURATION(expr, oper, val, unit) **Applicable parameter values**: - oper : Can be one of `'LT'` (lower than), `'GT'` (greater than), `'LE'` (lower than or equal to), `'GE'` (greater than or equal to), `'NE'` (not equal to), `'EQ'` (equal to), the value is case insensitive, the value must be in quotes. -- val ： Numeric types +- val: Numeric types - unit: The unit of time interval. Enter one of the following options: 1b (nanoseconds), 1u (microseconds), 1a (milliseconds), 1s (seconds), 1m (minutes), 1h (hours), 1d (days), or 1w (weeks) If you do not enter a unit of time, the precision of the current database is used by default. **Return value type**: Integer diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index 01d7065905359dd312346f54293c5cf8d8f04afb..b082f7b888dc2e09e4a6ae5c38e7ece591d667e3 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -69,19 +69,20 @@ These pseudocolumns occur after the aggregation clause. `FILL` clause is used to specify how to fill when there is data missing in any window, including: 1. NONE: No fill (the default fill mode) -2. VALUE：Fill with a fixed value, which should be specified together, for example `FILL(VALUE, 1.23)` Note: The value filled depends on the data type. For example, if you run FILL(VALUE 1.23) on an integer column, the value 1 is filled. -3. PREV：Fill with the previous non-NULL value, `FILL(PREV)` -4. NULL：Fill with NULL, `FILL(NULL)` -5. LINEAR：Fill with the closest non-NULL value, `FILL(LINEAR)` -6. NEXT：Fill with the next non-NULL value, `FILL(NEXT)` +2. VALUE: Fill with a fixed value, which should be specified together, for example `FILL(VALUE, 1.23)` Note: The value filled depends on the data type. For example, if you run FILL(VALUE 1.23) on an integer column, the value 1 is filled. +3. PREV: Fill with the previous non-NULL value, `FILL(PREV)` +4. NULL: Fill with NULL, `FILL(NULL)` +5. LINEAR: Fill with the closest non-NULL value, `FILL(LINEAR)` +6. NEXT: Fill with the next non-NULL value, `FILL(NEXT)` In the above filling modes, except for `NONE` mode, the `fill` clause will be ignored if there is no data in the defined time range, i.e. no data would be filled and the query result would be empty. This behavior is reasonable when the filling mode is `PREV`, `NEXT`, `LINEAR`, because filling can't be performed if there is not any data. For filling modes `NULL` and `VALUE`, however, filling can be performed even though there is not any data, filling or not depends on the choice of user's application. To accomplish the need of this force filling behavior and not break the behavior of existing filling modes, TDengine added two new filling modes since version 3.0.3.0. 1. NULL_F: Fill `NULL` by force 2. VALUE_F: Fill `VALUE` by force -The detailed beaviors of `NULL`, `NULL_F`, `VALUE`, and VALUE_F are described below： -- When used with `INTERVAL`: `NULL_F` and `VALUE_F` are filling by force；`NULL` and `VALUE` don't fill by force. The behavior of each filling mode is exactly same as what the name suggests. +The detailed beaviors of `NULL`, `NULL_F`, `VALUE`, and VALUE_F are described below: + +- When used with `INTERVAL`: `NULL_F` and `VALUE_F` are filling by force; `NULL` and `VALUE` don't fill by force. The behavior of each filling mode is exactly same as what the name suggests. - When used with `INTERVAL` in stream processing: `NULL_F` and `NULL` are same, i.e. don't fill by force; `VALUE_F` and `VALUE` and same, i.e. don't fill by force. It's suggested that there is no filling by force in stream processing. - When used with `INTERP`: `NULL` and `NULL_F` and same, i.e. filling by force; `VALUE` and `VALUE_F` are same, i.e. filling by force. It's suggested that there is always filling by force when used with `INTERP`. @@ -97,7 +98,7 @@ The detailed beaviors of `NULL`, `NULL_F`, `VALUE`, and VALUE_F are described be There are two kinds of time windows: sliding window and flip time/tumbling window. -The `INTERVAL` clause is used to generate time windows of the same time interval. The `SLIDING` parameter is used to specify the time step for which the time window moves forward. The query is performed on one time window each time, and the time window moves forward with time. When defining a continuous query, both the size of the time window and the step of forward sliding time need to be specified. As shown in the figure blow, [t0s, t0e] ，[t1s , t1e]， [t2s, t2e] are respectively the time ranges of three time windows on which continuous queries are executed. The time step for which time window moves forward is marked by `sliding time`. Query, filter and aggregate operations are executed on each time window respectively. When the time step specified by `SLIDING` is same as the time interval specified by `INTERVAL`, the sliding time window is actually a flip time/tumbling window. +The `INTERVAL` clause is used to generate time windows of the same time interval. The `SLIDING` parameter is used to specify the time step for which the time window moves forward. The query is performed on one time window each time, and the time window moves forward with time. When defining a continuous query, both the size of the time window and the step of forward sliding time need to be specified. As shown in the figure blow, [t0s, t0e], [t1s, t1e], [t2s, t2e] are respectively the time ranges of three time windows on which continuous queries are executed. The time step for which time window moves forward is marked by `sliding time`. Query, filter and aggregate operations are executed on each time window respectively. When the time step specified by `SLIDING` is same as the time interval specified by `INTERVAL`, the sliding time window is actually a flip time/tumbling window. ![TDengine Database Time Window](./timewindow-1.webp) @@ -121,7 +122,7 @@ Please note that the `timezone` parameter should be configured to be the same va ### State Window -In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two state windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12]. +In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two state windows according to status, [2019-04-28 14:22:07, 2019-04-28 14:22:10] and [2019-04-28 14:22:11, 2019-04-28 14:22:12]. ![TDengine Database Status Window](./timewindow-3.webp) @@ -145,7 +146,7 @@ SELECT tbname, _wstart, CASE WHEN voltage >= 205 and voltage <= 235 THEN 1 ELSE ### Session Window -The primary key, i.e. timestamp, is used to determine which session window a row belongs to. As shown in the figure below, if the limit of time interval for the session window is specified as 12 seconds, then the 6 rows in the figure constitutes 2 time windows, [2019-04-28 14:22:10，2019-04-28 14:22:30] and [2019-04-28 14:23:10，2019-04-28 14:23:30] because the time difference between 2019-04-28 14:22:30 and 2019-04-28 14:23:10 is 40 seconds, which exceeds the time interval limit of 12 seconds. +The primary key, i.e. timestamp, is used to determine which session window a row belongs to. As shown in the figure below, if the limit of time interval for the session window is specified as 12 seconds, then the 6 rows in the figure constitutes 2 time windows, [2019-04-28 14:22:10, 2019-04-28 14:22:30] and [2019-04-28 14:23:10, 2019-04-28 14:23:30] because the time difference between 2019-04-28 14:22:30 and 2019-04-28 14:23:10 is 40 seconds, which exceeds the time interval limit of 12 seconds. ![TDengine Database Session Window](./timewindow-2.webp) @@ -178,7 +179,7 @@ select _wstart, _wend, count(*) from t event_window start with c1 > 0 end with c ### Examples -A table of intelligent meters can be created by the SQL statement below： +A table of intelligent meters can be created by the SQL statement below: ``` CREATE TABLE meters (ts TIMESTAMP, current FLOAT, voltage INT, phase FLOAT) TAGS (location BINARY(64), groupId INT); diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index 43c49c03cd84eb741a5088c93bbb1571270e480b..329e98dbe3c8b82fe08f90a977d6612779996c5b 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -112,7 +112,7 @@ SHOW STREAMS; When you create a stream, you can use the TRIGGER parameter to specify triggering conditions for it. -For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering，the default value is AT_ONCE: +For non-windowed processing, triggering occurs in real time. For windowed processing, there are three methods of triggering, the default value is AT_ONCE: 1. AT_ONCE: triggers on write diff --git a/docs/en/12-taos-sql/17-json.md b/docs/en/12-taos-sql/17-json.md index 843858c0c76211346278fc078e7d04de5adecd7f..3ebcf71ffc36de920f2201c712aaf3c9bbda1f28 100644 --- a/docs/en/12-taos-sql/17-json.md +++ b/docs/en/12-taos-sql/17-json.md @@ -67,7 +67,7 @@ description: This document describes the JSON data type in TDengine. - The maximum length of keys in JSON is 256 bytes, and key must be printable ASCII characters. The maximum total length of a JSON is 4,096 bytes. -- JSON format： +- JSON format: - The input string for JSON can be empty, i.e. "", "\t", or NULL, but it can't be non-NULL string, bool or array. - object can be {}, and the entire JSON is empty if so. Key can be "", and it's ignored if so. diff --git a/docs/en/12-taos-sql/18-escape.md b/docs/en/12-taos-sql/18-escape.md index 85e4610e44f1a6f93e9727c32f8333cddf07ecc7..a44b21db43891537ca94978ffc73359b94175c3d 100644 --- a/docs/en/12-taos-sql/18-escape.md +++ b/docs/en/12-taos-sql/18-escape.md @@ -20,7 +20,7 @@ description: This document describes the usage of escape characters in TDengine. 1. If there are escape characters in identifiers (database name, table name, column name) - Identifier without ``: Error will be returned because identifier must be constituted of digits, ASCII characters or underscore and can't be started with digits - - Identifier quoted with ``： Original content is kept, no escaping + - Identifier quoted with ``: Original content is kept, no escaping 2. If there are escape characters in values - The escape characters will be escaped as the above table. If the escape character doesn't match any supported one, the escape character "\" will be ignored. - "%" and "\_" are used as wildcards in `like`. `\%` and `\_` should be used to represent literal "%" and "\_" in `like`,. If `\%` and `\_` are used out of `like` context, the evaluation result is "`\%`"and "`\_`", instead of "%" and "\_". diff --git a/docs/en/12-taos-sql/22-meta.md b/docs/en/12-taos-sql/22-meta.md index e63f682761d435731154b850bce2785976741924..bd745b1c6f9bbe3f3dfad65fe4664c7a136e722d 100644 --- a/docs/en/12-taos-sql/22-meta.md +++ b/docs/en/12-taos-sql/22-meta.md @@ -184,7 +184,7 @@ Provides information about standard tables and subtables. ## INS_COLUMNS -| # | **列名** | **数据类型** | **说明** | +| # | **Column** | **Data Type** | **Description** | | --- | :---------: | ------------- | ---------------------- | | 1 | table_name | BINARY(192) | Table name | | 2 | db_name | BINARY(64) | Database name | diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index 7b3da109ca3472decdcccd1e51dc1fb9b8d7c2be..eb70a7664b857a7971f24ea3ec7bc7f707f9fe38 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -4,7 +4,7 @@ sidebar_label: SHOW Statement description: This document describes how to use the SHOW statement in TDengine. --- -`SHOW` command can be used to get brief system information. To get details about metadata, information, and status in the system, please use `select` to query the tables in database `INFORMATION_SCHEMA`. +`SHOW` command can be used to get brief system information. To get details about metadata, information, and status in the system, please use `select` to query the tables in database `INFORMATION_SCHEMA`. ## SHOW APPS @@ -187,10 +187,10 @@ SHOW TABLE DISTRIBUTED table_name; Shows how table data is distributed. -Examples： Below is an example of this command to display the block distribution of table `d0` in detailed format. +Examples: Below is an example of this command to display the block distribution of table `d0` in detailed format. ```sql -show table distributed d0\G; +show table distributed d0\G; ```
@@ -201,31 +201,31 @@ _block_dist: Total_Blocks=[5] Total_Size=[93.65 KB] Average_size=[18.73 KB] Comp Total_Blocks : Table `d0` contains total 5 blocks -Total_Size: The total size of all the data blocks in table `d0` is 93.65 KB +Total_Size: The total size of all the data blocks in table `d0` is 93.65 KB Average_size: The average size of each block is 18.73 KB Compression_Ratio: The data compression rate is 23.98% - + *************************** 2.row *************************** _block_dist: Total_Rows=[20000] Inmem_Rows=[0] MinRows=[3616] MaxRows=[4096] Average_Rows=[4000] Total_Rows: Table `d0` contains 20,000 rows -Inmem_Rows： The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows +Inmem_Rows: The rows still in memory, i.e. not committed in disk, is 0, i.e. none such rows -MinRows： The minimum number of rows in a block is 3,616 +MinRows: The minimum number of rows in a block is 3,616 -MaxRows： The maximum number of rows in a block is 4,096B +MaxRows: The maximum number of rows in a block is 4,096B -Average_Rows： The average number of rows in a block is 4,000 +Average_Rows: The average number of rows in a block is 4,000 *************************** 3.row *************************** _block_dist: Total_Tables=[1] Total_Files=[2] -Total_Tables: The number of child tables, 1 in this example +Total_Tables: The number of child tables, 1 in this example -Total_Files： The number of files storing the table's data, 2 in this example +Total_Files: The number of files storing the table's data, 2 in this example *************************** 4.row *************************** @@ -361,7 +361,7 @@ SHOW VARIABLES; SHOW DNODE dnode_id VARIABLES; ``` -Shows the working configuration of the parameters that must be the same on each node. You can also specify a dnode to show the working configuration for that node. +Shows the working configuration of the parameters that must be the same on each node. You can also specify a dnode to show the working configuration for that node. ## SHOW VGROUPS @@ -369,7 +369,7 @@ Shows the working configuration of the parameters that must be the same on each SHOW [db_name.]VGROUPS; ``` -Shows information about all vgroups in the current database. +Shows information about all vgroups in the current database. ## SHOW VNODES diff --git a/docs/en/12-taos-sql/index.md b/docs/en/12-taos-sql/index.md index 276f84f21bc8ab92d13f1975b298dbdcf25db249..f1871cc6fb4f161b63fd5ab05b3ba581e4dc1105 100644 --- a/docs/en/12-taos-sql/index.md +++ b/docs/en/12-taos-sql/index.md @@ -13,7 +13,7 @@ Syntax Specifications used in this chapter: - Information that you input is given in lowercase. - \[ \] means optional input, excluding [] itself. - | means one of a few options, excluding | itself. -- … means the item prior to it can be repeated multiple times. +- ... means the item prior to it can be repeated multiple times. To better demonstrate the syntax, usage and rules of TDengine SQL, hereinafter it's assumed that there is a data set of data from electric meters. Each meter collects 3 data measurements: current, voltage, phase. The data model is shown below: diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 19107240bff5942ebb28e9a8353bc227ccb2b00e..80c33dd475d5c8ea1aa1553571c69a7d2406e5fd 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -22,11 +22,11 @@ wget https://github.com/taosdata/grafanaplugin/raw/master/dashboards/TDinsight.s chmod +x TDinsight.sh ``` -Prepare： +Prepare: 1. TDengine Server - - The URL of REST service：for example `http://localhost:6041` if TDengine is deployed locally + - The URL of REST service: for example `http://localhost:6041` if TDengine is deployed locally - User name and password 2. Grafana Alert Notification diff --git a/docs/en/13-operation/17-diagnose.md b/docs/en/13-operation/17-diagnose.md index 9d42b3ebbcfe0b87fa23bccaee12769d465fc24e..33a0a8c28cd011b12ec0380b2e076d228cc598c8 100644 --- a/docs/en/13-operation/17-diagnose.md +++ b/docs/en/13-operation/17-diagnose.md @@ -9,13 +9,13 @@ When a TDengine client is unable to access a TDengine server, the network connec Diagnostics for network connections can be executed between Linux/Windows/macOS. -Diagnostic steps： +Diagnostic steps: 1. If the port range to be diagnosed is being occupied by a `taosd` server process, please first stop `taosd. 2. On the server side, execute command `taos -n server -P -l ` to monitor the port range starting from the port specified by `-P` parameter with the role of "server". 3. On the client side, execute command `taos -n client -h -P -l ` to send a testing package to the specified server and port. --l ： The size of the testing package, in bytes. The value range is [11, 64,000] and default value is 1,000. +-l : The size of the testing package, in bytes. The value range is [11, 64,000] and default value is 1,000. Please note that the package length must be same in the above 2 commands executed on server side and client side respectively. Output of the server side for the example is below: diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index 409e079b92d5c8375ce6b3da60c0b2aa053ef662..ea010f42db8b988c597bcd4c9278c0b5d50a3ca7 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -83,13 +83,13 @@ For example, `http://h1.taos.com:6041/rest/sql/test` is a URL to `h1.taos.com:60 TDengine supports both Basic authentication and custom authentication mechanisms, and subsequent versions will provide a standard secure digital signature mechanism for authentication. -- authentication information is shown below： +- authentication information is shown below: ```text Authorization: Taosd ``` -- Basic authentication information is shown below： +- Basic authentication information is shown below: ```text Authorization: Basic diff --git a/docs/en/14-reference/03-connector/03-cpp.mdx b/docs/en/14-reference/03-connector/03-cpp.mdx index 014109b77e65f435d6d4cfb985daf74d664e12ae..13029dbe916281d671996254f5dc6069df49db95 100644 --- a/docs/en/14-reference/03-connector/03-cpp.mdx +++ b/docs/en/14-reference/03-connector/03-cpp.mdx @@ -12,9 +12,9 @@ C/C++ developers can use TDengine's client driver and the C/C++ connector, to de After TDengine server or client installation, `taos.h` is located at -- Linux：`/usr/local/taos/include` -- Windows：`C:\TDengine\include` -- macOS：`/usr/local/include` +- Linux: usr/local/taos/include` +- Windows: C:\TDengine\include` +- macOS: usr/local/include` The dynamic libraries for the TDengine client driver are located in. @@ -412,7 +412,8 @@ In addition to writing data using the SQL method or the parameter binding API, w Note that the timestamp resolution parameter only takes effect when the protocol type is `SML_LINE_PROTOCOL`. For OpenTSDB's text protocol, timestamp resolution follows its official resolution rules - time precision is confirmed by the number of characters contained in the timestamp. - schemaless 其他相关的接口 + schemaless interfaces: + - `TAOS_RES *taos_schemaless_insert_with_reqid(TAOS *taos, char *lines[], int numLines, int protocol, int precision, int64_t reqid)` - `TAOS_RES *taos_schemaless_insert_raw(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision)` - `TAOS_RES *taos_schemaless_insert_raw_with_reqid(TAOS *taos, char *lines, int len, int32_t *totalRows, int protocol, int precision, int64_t reqid)` diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index 65c7bc9bc557c8b3dba44e27bb766bb3e5fd5253..260b38b24ffd00682b393fc2e3d14142c0e6f3af 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -970,8 +970,8 @@ TaosConsumer consumer = new TaosConsumer<>(config); - group.id: consumer: Specifies the group that the consumer is in. - value.deserializer: To deserialize the results, you can inherit `com.taosdata.jdbc.tmq.ReferenceDeserializer` and specify the result set bean. You can also inherit `com.taosdata.jdbc.tmq.Deserializer` and perform custom deserialization based on the SQL result set. - td.connect.type: Specifies the type connect with TDengine, `jni` or `WebSocket`. default is `jni` -- httpConnectTimeout：WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type. -- messageWaitTimeout：socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type. +- httpConnectTimeout: WebSocket connection timeout in milliseconds, the default value is 5000 ms. It only takes effect when using WebSocket type. +- messageWaitTimeout: socket timeout in milliseconds, the default value is 10000 ms. It only takes effect when using WebSocket type. - For more information, see [Consumer Parameters](../../../develop/tmq). #### Subscribe to consume data @@ -1267,9 +1267,9 @@ The source code of the sample application is under `TDengine/examples/JDBC`: 5. java.lang.NoSuchMethodError: java.nio.ByteBuffer.position(I)Ljava/nio/ByteBuffer; ... taos-jdbcdriver-3.0.1.jar -**Cause**：taos-jdbcdriver 3.0.1 is compiled on JDK 11. +**Cause**: taos-jdbcdriver 3.0.1 is compiled on JDK 11. -**Solution**： Use taos-jdbcdriver 3.0.2. +**Solution**: Use taos-jdbcdriver 3.0.2. For additional troubleshooting, see [FAQ](../../../train-faq/faq). diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 597a9b0f52efcf4dcd54d58a36cd8ced761ecd1c..99c3d2c066d9e0e269ecc5cc03f6efa106b1906f 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -121,7 +121,7 @@ The parameters are described as follows: - **username/password**: Username and password used to create connections. - **host/port**: Specifies the server and port to establish a connection. If you do not specify a hostname or port, native connections default to `localhost:6030` and Websocket connections default to `localhost:6041`. - **database**: Specify the default database to connect to. It's optional. -- **params**：Optional parameters. +- **params**: Optional parameters. A sample DSN description string is as follows: diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index cc5c8f4e69e208c98c773e94c6aa2ca1ace334e7..a83f2047d02f7161d85a2f7ecb1c2a983ecfc22e 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -255,7 +255,7 @@ The `connect()` function returns a `taos.TaosConnection` instance. In client-sid All arguments to the `connect()` function are optional keyword arguments. The following are the connection parameters specified. -- `url`： The URL of taosAdapter REST service. The default is . +- `url`: The URL of taosAdapter REST service. The default is . - `user`: TDengine user name. The default is `root`. - `password`: TDengine user password. The default is `taosdata`. - `timeout`: HTTP request timeout. Enter a value in seconds. The default is `socket._GLOBAL_DEFAULT_TIMEOUT`. Usually, no configuration is needed. diff --git a/docs/en/14-reference/03-connector/08-node.mdx b/docs/en/14-reference/03-connector/08-node.mdx index b7c2e6b76537dc8b2a039684d2d712d91337cecd..a02bcddfd64789a08696ef11aa3853c81ffd66cf 100644 --- a/docs/en/14-reference/03-connector/08-node.mdx +++ b/docs/en/14-reference/03-connector/08-node.mdx @@ -321,18 +321,18 @@ let cursor = conn.cursor(); | package name | version | TDengine version | Description | |------------------|---------|---------------------|------------------------------------------------------------------| | @tdengine/client | 3.0.0 | 3.0.0 | Supports TDengine 3.0. Not compatible with TDengine 2.x. | -| td2.0-connector | 2.0.12 | 2.4.x；2.5.x；2.6.x | Fixed cursor.close() bug. | -| td2.0-connector | 2.0.11 | 2.4.x；2.5.x；2.6.x | Supports parameter binding, JSON tags and schemaless interface | -| td2.0-connector | 2.0.10 | 2.4.x；2.5.x；2.6.x | Supports connection management, standard queries, connection queries, system information, and data subscription | +| td2.0-connector | 2.0.12 | 2.4.x; 2.5.x; 2.6.x | Fixed cursor.close() bug. | +| td2.0-connector | 2.0.11 | 2.4.x; 2.5.x; 2.6.x | Supports parameter binding, JSON tags and schemaless interface | +| td2.0-connector | 2.0.10 | 2.4.x; 2.5.x; 2.6.x | Supports connection management, standard queries, connection queries, system information, and data subscription | ### REST Connector | package name | version | TDengine version | Description | |----------------------|---------|---------------------|---------------------------------------------------------------------------| | @tdengine/rest | 3.0.0 | 3.0.0 | Supports TDengine 3.0. Not compatible with TDengine 2.x. | -| td2.0-rest-connector | 1.0.7 | 2.4.x；2.5.x；2.6.x | Removed default port 6041。 | -| td2.0-rest-connector | 1.0.6 | 2.4.x；2.5.x；2.6.x | Fixed affectRows bug with create, insert, update, and alter. | -| td2.0-rest-connector | 1.0.5 | 2.4.x；2.5.x；2.6.x | Support cloud token | -| td2.0-rest-connector | 1.0.3 | 2.4.x；2.5.x；2.6.x | Supports connection management, standard queries, system information, error information, and continuous queries | +| td2.0-rest-connector | 1.0.7 | 2.4.x; 2.5.x; 2.6.x | Removed default port 6041 | +| td2.0-rest-connector | 1.0.6 | 2.4.x; 2.5.x; 2.6.x | Fixed affectRows bug with create, insert, update, and alter. | +| td2.0-rest-connector | 1.0.5 | 2.4.x; 2.5.x; 2.6.x | Support cloud token | +| td2.0-rest-connector | 1.0.3 | 2.4.x; 2.5.x; 2.6.x | Supports connection management, standard queries, system information, error information, and continuous queries | ## API Reference diff --git a/docs/en/14-reference/03-connector/09-csharp.mdx b/docs/en/14-reference/03-connector/09-csharp.mdx index fdc0824973eb83d3d544be239ea954e29b5ba265..718462295a44e1620c8751d67eaefbf83f7e64c2 100644 --- a/docs/en/14-reference/03-connector/09-csharp.mdx +++ b/docs/en/14-reference/03-connector/09-csharp.mdx @@ -165,7 +165,7 @@ The parameters are described as follows: * **username/password**: Username and password used to create connections. * **host/port**: Specifies the server and port to establish a connection. Websocket connections default to `localhost:6041`. * **database**: Specify the default database to connect to. It's optional. -* **params**：Optional parameters. +* **params**: Optional parameters. A sample DSN description string is as follows: @@ -279,7 +279,7 @@ ws://localhost:6041/test | TDengine.Connector | Description | |--------------------|--------------------------------| | 3.0.2 | Support .NET Framework 4.5 and above. Support .Net standard 2.0. Nuget package includes dynamic library for WebSocket.| -| 3.0.1 | Support WebSocket and Cloud，With function query, insert, and parameter binding| +| 3.0.1 | Support WebSocket and Cloud, With function query, insert, and parameter binding| | 3.0.0 | Supports TDengine 3.0.0.0. TDengine 2.x is not supported. Added `TDengine.Impl.GetData()` interface to deserialize query results. | | 1.0.7 | Fixed TDengine.Query() memory leak. | | 1.0.6 | Fix schemaless bug in 1.0.4 and 1.0.5. | diff --git a/docs/en/14-reference/03-connector/10-php.mdx b/docs/en/14-reference/03-connector/10-php.mdx index 6cc45ded49b90edbbea2d83ff91da7cf1ad89ce7..b6a31b6de366898485ebb0ab553adcc0d5edcc17 100644 --- a/docs/en/14-reference/03-connector/10-php.mdx +++ b/docs/en/14-reference/03-connector/10-php.mdx @@ -8,23 +8,23 @@ description: This document describes the TDengine PHP connector. PHP Connector relies on TDengine client driver. -Project Repository： +Project Repository: After TDengine client or server is installed, `taos.h` is located at: -- Linux：`/usr/local/taos/include` -- Windows：`C:\TDengine\include` -- macOS：`/usr/local/include` +- Linux: `/usr/local/taos/include` +- Windows: `C:\TDengine\include` +- macOS: `/usr/local/include` TDengine client driver is located at: - Linux: `/usr/local/taos/driver/libtaos.so` - Windows: `C:\TDengine\taos.dll` -- macOS：`/usr/local/lib/libtaos.dylib` +- macOS: `/usr/local/lib/libtaos.dylib` ## Supported Platforms -- Windows、Linux、MacOS +- Windows, Linux, and macOS - PHP >= 7.4 @@ -44,7 +44,7 @@ Regarding how to install TDengine client driver please refer to [Install Client ### Install php-tdengine -**Download Source Code Package and Unzip：** +**Download Source Code Package and Unzip: ** ```shell curl -L -o php-tdengine.tar.gz https://github.com/Yurunsoft/php-tdengine/archive/refs/tags/v1.0.2.tar.gz \ @@ -54,13 +54,13 @@ curl -L -o php-tdengine.tar.gz https://github.com/Yurunsoft/php-tdengine/archive > Version number `v1.0.2` is only for example, it can be replaced to any newer version, please find available versions in [TDengine PHP Connector Releases](https://github.com/Yurunsoft/php-tdengine/releases). -**Non-Swoole Environment：** +**Non-Swoole Environment: ** ```shell phpize && ./configure && make -j && make install ``` -**Specify TDengine location：** +**Specify TDengine location: ** ```shell phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/3.0.0.0 && make -j && make install @@ -69,7 +69,7 @@ phpize && ./configure --with-tdengine-dir=/usr/local/Cellar/tdengine/3.0.0.0 && > `--with-tdengine-dir=` is followed by TDengine location. > It's useful in case TDengine installatio location can't be found automatically or MacOS. -**Swoole Environment：** +**Swoole Environment: ** ```shell phpize && ./configure --enable-swoole && make -j && make install diff --git a/docs/en/14-reference/05-taosbenchmark.md b/docs/en/14-reference/05-taosbenchmark.md index f49424d3ba817be1d338ab78fb5738f859a5a235..8fc20c149f5f5dbaf56ed7fc7d065d42b8af81c5 100644 --- a/docs/en/14-reference/05-taosbenchmark.md +++ b/docs/en/14-reference/05-taosbenchmark.md @@ -245,7 +245,7 @@ The parameters listed in this section apply to all function modes. - ** trying_interval ** : Specify interval between keep trying insert. Valid value is a positive number. Only valid when keep trying be enabled. Available with v3.0.9+. - ** childtable_from and childtable_to ** : specify the child table range to create. The range is [childtable_from, childtable_to). -  + - ** continue_if_fail ** : allow the user to specify the reaction if the insertion failed. - "continue_if_fail" : "no" // means taosBenchmark will exit if it fails to insert as default reaction behavior. diff --git a/docs/en/14-reference/07-tdinsight/index.md b/docs/en/14-reference/07-tdinsight/index.md index 8d97dafa7c8362a694ca8d025e1035b74ec20e8b..965a38168ad93691b37a1b7e74aa995e6346987a 100644 --- a/docs/en/14-reference/07-tdinsight/index.md +++ b/docs/en/14-reference/07-tdinsight/index.md @@ -233,7 +233,7 @@ After the importing is done, `TDinsight for 3.x` dashboard is available on the p In the `TDinsight for 3.x` dashboard, choose the database used by taosKeeper to store monitoring data, you can see the monitoring result. -![TDengine Database TDinsight 选择数据库](./assets/select_dashboard_db.webp) +![TDengine Database TDinsight select database](./assets/select_dashboard_db.webp) ## TDinsight dashboard details diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 715a68696b8433c2f3447c079622d92d53bd36bc..c3d28fa48e0bde79bae873519edc4d1ea059a364 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -151,7 +151,7 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | -------- | -------------------------------------------- | | Applicable | Server Only | | Meaning |Switch for allowing TDengine to collect and report crash related information | -| Value Range | 0,1 0: Not allowed；1：allowed | +| Value Range | 0,1 0: Not allowed; 1: allowed | | Default Value | 1 | @@ -183,7 +183,7 @@ Please note the `taoskeeper` needs to be installed and running to create the `lo | -------- | -------------------------------- | | Applicable | Server only | | Meaning | count()/hyperloglog() return value or not if the input data is empty or NULL | -| Vlue Range | 0：Return empty line，1：Return 0 | +| Vlue Range | 0: Return empty line, 1: Return 0 | | Default | 1 | | Notes | When this parameter is setting to 1, for queries containing GROUP BY, PARTITION BY and INTERVAL clause, and input data in certain groups or windows is empty or NULL, the corresponding groups or windows have no return values | @@ -661,7 +661,7 @@ The charset that takes effect is UTF-8. ## 3.0 Parameters -| # | **参数** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | +| # | **Parameter** | **Applicable to 2.x ** | **Applicable to 3.0 ** | Current behavior in 3.0 | | --- | :---------------------: | --------------- | --------------- | ------------------------------------------------- | | 1 | firstEp | Yes | Yes | | | 2 | secondEp | Yes | Yes | | diff --git a/docs/en/20-third-party/01-grafana.mdx b/docs/en/20-third-party/01-grafana.mdx index b0e9dea707fb2fb326329895921850cd6d721b6d..64acefe6b81bb4b74b59e00a4fb41283598fef1a 100644 --- a/docs/en/20-third-party/01-grafana.mdx +++ b/docs/en/20-third-party/01-grafana.mdx @@ -200,11 +200,16 @@ As shown above, select the `TDengine` data source in the `Query` and enter the c - Group by column name(s): `group by` or `partition by` columns name split by comma. By setting `Group by column name(s)`, it can show multi-dimension data if Sql is `group by` or `partition by`. Such as, it can show data by `dnode_ep` if sql is `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)` and `Group by column name(s)` is `dnode_ep`. - Format to: format legend for `group by` or `partition by`. Such as it can display series data by `dnode_ep` if sql is `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)` and `Group by column name(s)` is `dnode_ep` and `Format to` is `mem_system_{{dnode_ep}}`. +:::note + +Since the REST connection because is stateless. Grafana plugin can use . in the SQL command to specify the database name. + +::: + Follow the default prompt to query the average system memory usage for the specified interval on the server where the current TDengine deployment is located as follows. ![TDengine Database TDinsight plugin create dashboard 2](./grafana/create_dashboard2.webp) -查询每台 TDengine 服务器指定间隔系统内存平均使用量如下. The example to query the average system memory usage for the specified interval on each server as follows. ![TDengine Database TDinsight plugin create dashboard 2](./grafana/create_dashboard3.webp) @@ -217,7 +222,7 @@ You can install TDinsight dashboard in data source configuration page (like `htt ![TDengine Database Grafana plugine import dashboard](./import_dashboard.webp) -A dashboard for TDengine 2.x has been published on Grafana: [Dashboard 15167 - TDinsight](https://grafana.com/grafana/dashboards/15167)) 。 Check the [TDinsight User Manual](/reference/tdinsight/) for the details. +A dashboard for TDengine 2.x has been published on Grafana: [Dashboard 15167 - TDinsight](https://grafana.com/grafana/dashboards/15167)). Check the [TDinsight User Manual](/reference/tdinsight/) for the details. For more dashboards using TDengine data source, [search here in Grafana](https://grafana.com/grafana/dashboards/?dataSource=tdengine-datasource). Here is a sub list: diff --git a/docs/en/20-third-party/09-emq-broker.md b/docs/en/20-third-party/09-emq-broker.md index 10ce0174ed5d362a113acd7ade59741a10155657..9dab6e52c9ce42d4965f824e28d3b338cc93f5a1 100644 --- a/docs/en/20-third-party/09-emq-broker.md +++ b/docs/en/20-third-party/09-emq-broker.md @@ -47,7 +47,7 @@ Select "Rule" in the "Rule Engine" on the left and click the "Create" button: ! ### Edit SQL fields -Copy SQL bellow and paste it to the SQL edit area： +Copy SQL bellow and paste it to the SQL edit area: ```sql SELECT @@ -76,7 +76,8 @@ Select "WebHook" and fill in the request URL as the address and port of the serv ### Edit "action" -Edit the resource configuration to add the key/value pairing for Authorization. If you use the default TDengine username and password then the value of key Authorization is： +Edit the resource configuration to add the key/value pairing for Authorization. If you use the default TDengine username and password then the value of key Authorization is: + ``` Basic cm9vdDp0YW9zZGF0YQ== ``` diff --git a/docs/en/20-third-party/12-google-data-studio.md b/docs/en/20-third-party/12-google-data-studio.md index ea6431fa5a81b3b9e51fe04e8e404cf52e9c0ad9..4e71bcfc89e615c7de5b79d576ec5f123e96b895 100644 --- a/docs/en/20-third-party/12-google-data-studio.md +++ b/docs/en/20-third-party/12-google-data-studio.md @@ -10,7 +10,7 @@ TDengine is a high-performance, scalable time-series database that supports SQL. The TDengine team immediately saw the benefits of using TDengine to process time-series data with Data Studio to analyze it, and they got to work to create a connector for Data Studio. -With the release of the TDengine connector in Data Studio, you can now get even more out of your data. To obtain the connector, first go to the Data Studio Connector Gallery, click Connect to Data, and search for “TDengine”. +With the release of the TDengine connector in Data Studio, you can now get even more out of your data. To obtain the connector, first go to the Data Studio Connector Gallery, click Connect to Data, and search for "TDengine". ![02](gds/gds-02.png.webp) @@ -30,8 +30,8 @@ After the connection is established, you can use Data Studio to process your dat ![06](gds/gds-06.png.webp) -In Data Studio, TDengine timestamps and tags are considered dimensions, and all other items are considered metrics. You can create all kinds of custom charts with your data – some examples are shown below. +In Data Studio, TDengine timestamps and tags are considered dimensions, and all other items are considered metrics. You can create all kinds of custom charts with your data - some examples are shown below. ![07](gds/gds-07.png.webp) -With the ability to process petabytes of data per day and provide monitoring and alerting in real time, TDengine is a great solution for time-series data management. Now, with the Data Studio connector, we’re sure you’ll be able to gain new insights and obtain even more value from your data. +With the ability to process petabytes of data per day and provide monitoring and alerting in real time, TDengine is a great solution for time-series data management. Now, with the Data Studio connector, we're sure you'll be able to gain new insights and obtain even more value from your data. diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index cef05dcc56793846fd36515121f6688f74999bbf..c2ef0a190051c96802aeca781f8776c6cec53a2d 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -26,9 +26,9 @@ A complete TDengine system runs on one or more physical nodes. Logically, a comp **Management node (mnode)**: A virtual logical unit (M in the figure) responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes. At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 3) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). mnode adopts RAFT protocol to guarantee high data availability and high data reliability. Any data operation can only be performed through the Leader in the RAFT group. The first mnode in the mnode RAFT group is created automatically when the first dnode of the cluster is deployed. Other two follower mnodes need to be created through SQL command in TDengine CLI. There can be at most one mnode in a single dnode, and the mnode is identified by the EP of the dnode where it's located. Each dnode can communicate with each other to automatically get the EP of all mnodes. -**Computation node (qnode)**： A virtual logical unit (Q in the figure) responsible for executing query and computing tasks including the `show` commands based on system built-in tables. There can be multiple qnodes configured in a TDengine cluster to share the query and computing tasks. A qnode is not coupled with a specific database, that means each qnode can execute the query tasks for multiple databases in parallel. There can be at most one qnode in a single dnode, and the qnode is identified by the EP of the dnode. TDengine client driver can get the list of qnodes through the communication with mnode. If there is no qnode available in the system, query and computing tasks are executed by vnodes. When a query task is executed, according to the execution plan, one or more qnodes may be scheduled by the scheduler to execute the task. qnode can get data from vnode, and send the execution result to other qnodes for further processing. With introducing qnodes, TDengine achieves the separation between storage and computing. +**Computation node (qnode)**: A virtual logical unit (Q in the figure) responsible for executing query and computing tasks including the `show` commands based on system built-in tables. There can be multiple qnodes configured in a TDengine cluster to share the query and computing tasks. A qnode is not coupled with a specific database, that means each qnode can execute the query tasks for multiple databases in parallel. There can be at most one qnode in a single dnode, and the qnode is identified by the EP of the dnode. TDengine client driver can get the list of qnodes through the communication with mnode. If there is no qnode available in the system, query and computing tasks are executed by vnodes. When a query task is executed, according to the execution plan, one or more qnodes may be scheduled by the scheduler to execute the task. qnode can get data from vnode, and send the execution result to other qnodes for further processing. With introducing qnodes, TDengine achieves the separation between storage and computing. -**Stream Processing node (snode)**： A virtual logical unit (S in the figure) responsible for stream processing tasks is introduced in TDengine. There can be multiple snodes configured in a TDengine cluster to share the burden of stream processing tasks. snode is not coupled with a specific stream, that means a single snode can execute the tasks of multiple streams. There can be at most one snode in a single dnode, it's identified by the EP of the dnode. mnode schedules available snodes to perform the stream processing tasks. If there is no snode available in the system, stream processing tasks are executed in vnodes. +**Stream Processing node (snode)**: A virtual logical unit (S in the figure) responsible for stream processing tasks is introduced in TDengine. There can be multiple snodes configured in a TDengine cluster to share the burden of stream processing tasks. snode is not coupled with a specific stream, that means a single snode can execute the tasks of multiple streams. There can be at most one snode in a single dnode, it's identified by the EP of the dnode. mnode schedules available snodes to perform the stream processing tasks. If there is no snode available in the system, stream processing tasks are executed in vnodes. **Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed using RAFT protocol. Write operations can only be performed on the leader vnode, and then replicated to follower vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `replica` when creating a DB, and the default is 1. Using the multiple replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID, to each vgroup. Virtual nodes with the same vnode group ID belong to the same vgroup. If `replica` is set to 1, it means no data replication. The number of replication for a database can be dynamically changed to 3 for high data reliability. Even if a virtual node group is deleted, its ID will not be reused. @@ -59,7 +59,7 @@ After obtaining the mnode EP list, the data node initiates the connection. It wi - Step : Connect to the existing working data node using TDengine CLI, and then add the End Point of the new data node with the command "create dnode" - Step 2: In the system configuration parameter file `taos.cfg` of the new data node, set the `firstEp` and `secondEp` parameters to the EP of any two data nodes in the existing cluster. If there is only one existing data node in the system, skip parameter `secondEp`. Please refer to the user tutorial for detailed steps. In this way, the cluster will be established step by step. -**Redirection**: Regardless of dnode or TAOSC, the connection to the mnode is initiated first. The mnode is automatically created and maintained by the system, so the user does not know which dnode is running the mnode. TDengine only requires a connection to any working dnode in the system. Because any running dnode maintains the currently running mnode EP List, when receiving a connecting request from the newly started dnode or TAOSC, if it’s not an mnode itself, it will reply to the connection initiator with the mnode EP List. After receiving this list, TAOSC or the newly started dnode will try to establish the connection again with mnode. When the mnode EP List changes, each data node quickly obtains the latest list and notifies TAOSC through messaging interaction among nodes. +**Redirection**: Regardless of dnode or TAOSC, the connection to the mnode is initiated first. The mnode is automatically created and maintained by the system, so the user does not know which dnode is running the mnode. TDengine only requires a connection to any working dnode in the system. Because any running dnode maintains the currently running mnode EP List, when receiving a connecting request from the newly started dnode or TAOSC, if it's not an mnode itself, it will reply to the connection initiator with the mnode EP List. After receiving this list, TAOSC or the newly started dnode will try to establish the connection again with mnode. When the mnode EP List changes, each data node quickly obtains the latest list and notifies TAOSC through messaging interaction among nodes. ### A Typical Data Writing Process @@ -107,7 +107,7 @@ For large-scale data management, to achieve scale-out, it is generally necessary VNode (Virtual Data Node) is responsible for providing writing, query and computing functions for collected time-series data. To facilitate load balancing, data recovery and support heterogeneous environments, TDengine splits a data node into multiple vnodes according to its computing and storage resources. The management of these vnodes is done automatically by TDengine and is completely transparent to the application. -For a single data collection point, regardless of the amount of data, a vnode (or vnode group, if the number of replicas is greater than 1) has enough computing resource and storage resource to process (if a 16-byte record is generated per second, the original data generated in one year will be less than 0.5 G). So TDengine stores all the data of a table (a data collection point) in one vnode instead of distributing the data to two or more dnodes. Moreover, a vnode can store data from multiple data collection points (tables), and the upper limit of the tables’ quantity for a vnode is one million. By design, all tables in a vnode belong to the same DB. On a data node, unless specially configured, the number of vnodes owned by a DB will not exceed the number of system cores. +For a single data collection point, regardless of the amount of data, a vnode (or vnode group, if the number of replicas is greater than 1) has enough computing resource and storage resource to process (if a 16-byte record is generated per second, the original data generated in one year will be less than 0.5 G). So TDengine stores all the data of a table (a data collection point) in one vnode instead of distributing the data to two or more dnodes. Moreover, a vnode can store data from multiple data collection points (tables), and the upper limit of the tables' quantity for a vnode is one million. By design, all tables in a vnode belong to the same DB. On a data node, unless specially configured, the number of vnodes owned by a DB will not exceed the number of system cores. When creating a DB, the system does not allocate resources immediately. However, when creating a table, the system will check if there is an allocated vnode with free tablespace. If so, the table will be created in the vacant vnode immediately. If not, the system will create a new vnode on a dnode from the cluster according to the current workload, and then a table. If there are multiple replicas of a DB, the system does not create only one vnode, but a vgroup (virtual data node group). The system has no limit on the number of vnodes, which is just limited by the computing and storage resources of physical nodes. @@ -132,9 +132,9 @@ Leader Vnode uses a writing process as follows:
Figure 3: TDengine Leader writing process
1. Leader vnode receives the application data insertion request, verifies, and moves to next step; -2. Leader vnode will write the original request packet into database log file WAL. If the database configuration parameter `“wal_level”` is set to 1, vnode doesn't invoked fsync. If `wal_level` is set to 2, fsync is invoked according to another database parameter `wal_fsync_period`. +2. Leader vnode will write the original request packet into database log file WAL. If the database configuration parameter `"wal_level"` is set to 1, vnode doesn't invoked fsync. If `wal_level` is set to 2, fsync is invoked according to another database parameter `wal_fsync_period`. 3. If there are multiple replicas, the leader vnode will forward data packet to follower vnodes in the same virtual node group, and the forwarded packet has a version number with data; -4. Leader vnode Writes the data into memory and add the record to “skip list”; +4. Leader vnode Writes the data into memory and add the record to "skip list"; 5. Leader vnode returns a confirmation message to the application, indicating a successful write. 6. If any of Step 2, 3 or 4 fails, the error will directly return to the application. @@ -148,7 +148,7 @@ For a follower vnode, the write process as follows: 1. Follower vnode receives a data insertion request forwarded by Leader vnode; 2. The behavior regarding `wal_level` and `wal_fsync_period` in a follower vnode is same as the leader vnode. -3. Write into memory and add the record to “skip list”. +3. Write into memory and add the record to "skip list". Compared with Leader vnode, follower vnode has no forwarding or reply confirmation step. But writing into memory and WAL is exactly the same. @@ -156,7 +156,7 @@ Compared with Leader vnode, follower vnode has no forwarding or reply confirmati Vnode maintains a version number. When memory data is persisted, the version number is also persisted. For each data update operation, whether it is time-series data or metadata, this version number will be increased by one. -When a vnode starts, its role (leader, follower) is uncertain, and the data is in an unsynchronized state. It’s necessary to establish TCP connections with other vnodes in the virtual node group and exchange status, including version and its own role. Through the exchange, the system implements a leader-selection process according to standard RAFT protocol. +When a vnode starts, its role (leader, follower) is uncertain, and the data is in an unsynchronized state. It's necessary to establish TCP connections with other vnodes in the virtual node group and exchange status, including version and its own role. Through the exchange, the system implements a leader-selection process according to standard RAFT protocol. ### Synchronous Replication @@ -192,7 +192,7 @@ When data is written to disk, the system decides whether to compress the data ba ### Tiered Storage -By default, TDengine saves all data in /var/lib/taos directory, and the data files of each vnode are saved in a different directory under this directory. In order to expand the storage space, minimize the bottleneck of file reading and improve the data throughput rate, TDengine can configure the system parameter “dataDir” to allow multiple mounted hard disks to be used by system at the same time. In addition, TDengine also provides the function of tiered data storage, i.e. storage on different storage media according to the time stamps of data files. For example, the latest data is stored on SSD, the data older than a week is stored on local hard disk, and data older than four weeks is stored on network storage device. This reduces storage costs and ensures efficient data access. The movement of data on different storage media is automatically done by the system and is completely transparent to applications. Tiered storage of data is also configured through the system parameter “dataDir”. +By default, TDengine saves all data in /var/lib/taos directory, and the data files of each vnode are saved in a different directory under this directory. In order to expand the storage space, minimize the bottleneck of file reading and improve the data throughput rate, TDengine can configure the system parameter "dataDir" to allow multiple mounted hard disks to be used by system at the same time. In addition, TDengine also provides the function of tiered data storage, i.e. storage on different storage media according to the time stamps of data files. For example, the latest data is stored on SSD, the data older than a week is stored on local hard disk, and data older than four weeks is stored on network storage device. This reduces storage costs and ensures efficient data access. The movement of data on different storage media is automatically done by the system and is completely transparent to applications. Tiered storage of data is also configured through the system parameter "dataDir". dataDir format is as follows: @@ -202,7 +202,7 @@ dataDir data_path [tier_level] Where data_path is the folder path of mount point and tier_level is the media storage-tier. The higher the media storage-tier, means the older the data file. Multiple hard disks can be mounted at the same storage-tier, and data files on the same storage-tier are distributed on all hard disks within the tier. TDengine supports up to 3 tiers of storage, so tier_level values are 0, 1, and 2. When configuring dataDir, there must be only one mount path without specifying tier_level, which is called special mount disk (path). The mount path defaults to level 0 storage media and contains special file links, which cannot be removed, otherwise it will have a devastating impact on the written data. -Suppose there is a physical node with six mountable hard disks/mnt/disk1,/mnt/disk2, …,/mnt/disk6, where disk1 and disk2 need to be designated as level 0 storage media, disk3 and disk4 are level 1 storage media, and disk5 and disk6 are level 2 storage media. Disk1 is a special mount disk, you can configure it in/etc/taos/taos.cfg as follows: +Suppose there is a physical node with six mountable hard disks/mnt/disk1,/mnt/disk2, ..., /mnt/disk6, where disk1 and disk2 need to be designated as level 0 storage media, disk3 and disk4 are level 1 storage media, and disk5 and disk6 are level 2 storage media. Disk1 is a special mount disk, you can configure it in/etc/taos/taos.cfg as follows: ``` dataDir /mnt/disk1/taos diff --git a/docs/en/25-application/_03-immigrate.md b/docs/en/25-application/_03-immigrate.md index 5f4a86937e305c517bca36f6fcf3be6ae791dd20..f78042353249a29f7ee634cfc544c6c0914e3251 100644 --- a/docs/en/25-application/_03-immigrate.md +++ b/docs/en/25-application/_03-immigrate.md @@ -200,7 +200,7 @@ After migrating via DataX, we found that we can significantly improve the effici ### 2. Manual data migration -Suppose you need to use the multi-value model for data writing. In that case, you need to develop a tool to export data from OpenTSDB, confirm which timelines can be merged and imported into the same timeline, and then pass the time to import simultaneously through the SQL statement—written to the database. +Suppose you need to use the multi-value model for data writing. In that case, you need to develop a tool to export data from OpenTSDB, confirm which timelines can be merged and imported into the same timeline, and then pass the time to import simultaneously through the SQL statement-written to the database. Manual migration of data requires attention to the following two issues: @@ -258,7 +258,7 @@ Equivalent function: apercentile Example: ```sql -Select apercentile(col1, 50, “t-digest”) from table_name +select apercentile(col1, 50, "t-digest") from table_name ``` Remark: diff --git a/docs/zh/07-develop/04-query-data/index.mdx b/docs/zh/07-develop/04-query-data/index.mdx index d6156c8a59a70af80f2632cdf3801ef7281b69d5..d4d6d8192d7907a9d5f6fc25d00a057c620965f3 100644 --- a/docs/zh/07-develop/04-query-data/index.mdx +++ b/docs/zh/07-develop/04-query-data/index.mdx @@ -161,7 +161,7 @@ Query OK, 6 rows in database (0.005515s) :::note 1. 无论是使用 REST 连接还是原生连接的连接器，以上示例代码都能正常工作。 -2. 唯一需要注意的是：由于 REST 接口无状态， 不能使用 `use db` 语句来切换数据库。 +2. 唯一需要注意的是：由于 REST 接口无状态， 不能使用 `use db` 语句来切换数据库。除了在 REST 参数中指定数据库以外也可以在 SQL 语句中使用 . 来指定数据库。 ::: diff --git a/docs/zh/20-third-party/01-grafana.mdx b/docs/zh/20-third-party/01-grafana.mdx index 7980736a1c62150df1c2742c788aabde9b1dbc3b..00850bebb29212cc630c83d339aff8755b2fb38a 100644 --- a/docs/zh/20-third-party/01-grafana.mdx +++ b/docs/zh/20-third-party/01-grafana.mdx @@ -200,6 +200,12 @@ docker run -d \ - Group by column name(s)： **半角**逗号分隔的 `group by` 或 `partition by` 列名。如果是 `group by` or `partition by` 查询语句，设置 `Group by` 列，可以展示多维数据。例如：INPUT SQL 为 `select _wstart as ts, avg(mem_system), dnode_ep from log.dnodes_info where ts>=$from and ts<=$to partition by dnode_ep interval($interval)`，设置 Group by 列名为 `dnode_ep`，可以按 `dnode_ep` 展示数据。 - Format to： Group by 或 Partition by 场景下多维数据 legend 格式化格式。例如上述 INPUT SQL，将 Format to 设置为 `mem_system_{{dnode_ep}}`，展示的 legend 名字为格式化的列名。 +:::note + +由于 REST 接口无状态， 不能使用 `use db` 语句来切换数据库。Grafana 插件中 SQL 语句中可以使用 . 来指定数据库。 + +::: + 按照默认提示查询当前 TDengine 部署所在服务器指定间隔系统内存平均使用量如下： ![TDengine Database Grafana plugin create dashboard](./create_dashboard2.webp) diff --git a/include/client/taos.h b/include/client/taos.h index d9fd1ca1b81e72b4d08150422a49c7e3051f7f89..8811c4ab64e3ae65085c26ccd791705a98541423 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -310,6 +310,7 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); +DLL_EXPORT int64_t tmq_get_vgroup_offset(TAOS_RES* res); /* ------------------------------ TAOSX -----------------------------------*/ // note: following apis are unstable diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index 63e9e3799a4f418b792ec4a61182b12bce30b21c..5b125b42d4bc67ed6625d38f9baf4fb356ad041e 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -20,13 +20,13 @@ #include "tsimplehash.h" #include "tstreamFileState.h" +#ifndef _STREAM_STATE_H_ +#define _STREAM_STATE_H_ + #ifdef __cplusplus extern "C" { #endif -#ifndef _STREAM_STATE_H_ -#define _STREAM_STATE_H_ - // void* streamBackendInit(const char* path); // void streamBackendCleanup(void* arg); // SListNode* streamBackendAddCompare(void* backend, void* arg); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 865977d62b212d5e621ab3fce05ca7dbedd979d4..c7e55650cd7a422b2af9680a79660e64328deeb0 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,16 +13,13 @@ * along with this program. If not, see . */ -#include "executor.h" #include "os.h" -#include "query.h" #include "streamState.h" #include "tdatablock.h" #include "tdbInt.h" #include "tmsg.h" #include "tmsgcb.h" #include "tqueue.h" -#include "trpc.h" #ifdef __cplusplus extern "C" { @@ -340,7 +337,7 @@ typedef struct SStreamMeta { TTB* pTaskDb; TTB* pCheckpointDb; SHashObj* pTasks; - SArray* pTaskList; // SArray + SArray* pTaskList; // SArray void* ahandle; TXN* txn; FTaskExpand* expandFunc; @@ -568,6 +565,8 @@ int32_t streamAggRecoverPrepare(SStreamTask* pTask); // int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask); int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); +void streamMetaInit(); +void streamMetaCleanup(); SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index c8f3feb2d425858a76779cd2b94fcede72849e39..cae5c8715daafff8f58f82579af2577b9a6a323b 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -449,6 +449,7 @@ static void *tscCrashReportThreadFp(void *param) { tscError("failed to send crash report"); if (pFile) { taosReleaseCrashLogFile(pFile, false); + pFile = NULL; continue; } } else { @@ -468,6 +469,7 @@ static void *tscCrashReportThreadFp(void *param) { if (pFile) { taosReleaseCrashLogFile(pFile, truncateFile); + pFile = NULL; truncateFile = false; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 10c42bb67d2961ac0530333fbb1e9fd4a2294062..0c60592fc50c087454f1a924ef778cc9d09117fa 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -24,6 +24,8 @@ typedef struct { struct { int64_t clusterId; int32_t passKeyCnt; + int32_t passVer; + int32_t reqCnt; }; }; } SHbParam; @@ -536,14 +538,20 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } -static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { +static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SHbParam *param, SClientHbReq *req) { STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); if (!pTscObj) { tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); return TSDB_CODE_APP_ERROR; } - int32_t code = 0; + int32_t code = 0; + + if (param && (param->passVer != INT32_MIN) && (param->passVer <= pTscObj->passInfo.ver)) { + tscDebug("hb got user basic info, no need since passVer %d <= %d", param->passVer, pTscObj->passInfo.ver); + goto _return; + } + SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); if (!user) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -570,6 +578,11 @@ static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { goto _return; } + // assign the passVer + if (param) { + param->passVer = pTscObj->passInfo.ver; + } + _return: releaseTscObj(connKey->tscRid); if (code) { @@ -714,13 +727,16 @@ int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { } int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { - SHbParam *hbParam = (SHbParam *)param; - struct SCatalog *pCatalog = NULL; - - int32_t code = catalogGetHandle(hbParam->clusterId, &pCatalog); - if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code)); - return code; + int32_t code = 0; + SHbParam *hbParam = (SHbParam *)param; + SCatalog *pCatalog = NULL; + + if (hbParam->reqCnt == 0) { + code = catalogGetHandle(hbParam->clusterId, &pCatalog); + if (code != TSDB_CODE_SUCCESS) { + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code)); + return code; + } } hbGetAppInfo(hbParam->clusterId, req); @@ -728,24 +744,28 @@ int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req hbGetQueryBasicInfo(connKey, req); if (hbParam->passKeyCnt > 0) { - hbGetUserBasicInfo(connKey, req); + hbGetUserBasicInfo(connKey, hbParam, req); } - code = hbGetExpiredUserInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; - } + if (hbParam->reqCnt == 0) { + code = hbGetExpiredUserInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } - code = hbGetExpiredDBInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; - } + code = hbGetExpiredDBInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } - code = hbGetExpiredStbInfo(connKey, pCatalog, req); - if (TSDB_CODE_SUCCESS != code) { - return code; + code = hbGetExpiredStbInfo(connKey, pCatalog, req); + if (TSDB_CODE_SUCCESS != code) { + return code; + } } + ++hbParam->reqCnt; // success to get catalog info + return TSDB_CODE_SUCCESS; } @@ -766,6 +786,10 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { } int32_t connKeyCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); pBatchReq->reqs = taosArrayInit(connKeyCnt, sizeof(SClientHbReq)); + if (!pBatchReq->reqs) { + tFreeClientHbBatchReq(pBatchReq); + return NULL; + } int64_t rid = -1; int32_t code = 0; @@ -782,12 +806,18 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { return NULL; } + SHbParam param = {0}; + while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - SHbParam param; + switch (pOneReq->connKey.connType) { case CONN_TYPE__QUERY: { - param.clusterId = pOneReq->clusterId; + if (param.clusterId == 0) { + // init + param.clusterId = pOneReq->clusterId; + param.passVer = INT32_MIN; + } param.passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); break; } @@ -801,9 +831,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pOneReq->connKey.tscRid, pOneReq->connKey.connType); } } - break; -#if 0 if (code) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; @@ -812,7 +840,6 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; -#endif } releaseTscObj(rid); @@ -885,7 +912,6 @@ static void *hbThreadFunc(void *param) { hbGatherAppInfo(); } - SArray *mgr = taosArrayInit(sz, sizeof(void *)); for (int i = 0; i < sz; i++) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); if (pAppHbMgr == NULL) { @@ -894,7 +920,6 @@ static void *hbThreadFunc(void *param) { int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); if (connCnt == 0) { - taosArrayPush(mgr, &pAppHbMgr); continue; } SClientHbBatchReq *pReq = hbGatherAllInfo(pAppHbMgr); @@ -908,7 +933,6 @@ static void *hbThreadFunc(void *param) { terrno = TSDB_CODE_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); - taosArrayPush(mgr, &pAppHbMgr); break; } @@ -920,7 +944,6 @@ static void *hbThreadFunc(void *param) { tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); - taosArrayPush(mgr, &pAppHbMgr); break; } pInfo->fp = hbAsyncCallBack; @@ -941,12 +964,8 @@ static void *hbThreadFunc(void *param) { // hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); - taosArrayPush(mgr, &pAppHbMgr); } - taosArrayDestroy(clientHbMgr.appHbMgrs); - clientHbMgr.appHbMgrs = mgr; - taosThreadMutexUnlock(&clientHbMgr.lock); taosMsleep(HEARTBEAT_INTERVAL); diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 87aee4a8a3f8b95afb62338dc2fcd4700212029c..63e8b3097c3cd7a8199e5001723515dafdb2bb24 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -2109,6 +2109,29 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { } } +int64_t tmq_get_vgroup_offset(TAOS_RES* res) { + if (TD_RES_TMQ(res)) { + SMqRspObj* pRspObj = (SMqRspObj*) res; + STqOffsetVal* pOffset = &pRspObj->rsp.rspOffset; + if (pOffset->type == TMQ_OFFSET__LOG) { + return pRspObj->rsp.rspOffset.version; + } + } else if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pRspObj = (SMqMetaRspObj*)res; + if (pRspObj->metaRsp.rspOffset.type == TMQ_OFFSET__LOG) { + return pRspObj->metaRsp.rspOffset.version; + } + } else if (TD_RES_TMQ_METADATA(res)) { + SMqTaosxRspObj* pRspObj = (SMqTaosxRspObj*) res; + if (pRspObj->rsp.rspOffset.type == TMQ_OFFSET__LOG) { + return pRspObj->rsp.rspOffset.version; + } + } + + // data from tsdb, no valid offset info + return -1; +} + const char* tmq_get_table_name(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 06b6221940a4f0991fe520a58e2918409139380b..89c394fdd0889a30b737b43f534aec96e3fb3afa 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -91,6 +91,7 @@ static void *dmCrashReportThreadFp(void *param) { dError("failed to send crash report"); if (pFile) { taosReleaseCrashLogFile(pFile, false); + pFile = NULL; continue; } } else { @@ -110,6 +111,7 @@ static void *dmCrashReportThreadFp(void *param) { if (pFile) { taosReleaseCrashLogFile(pFile, truncateFile); + pFile = NULL; truncateFile = false; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c index d8841201478b948f396e8d62a64c714290559c96..544512233e47e0b6a7f29dcbaf056f2e6ad48195 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmMgmt.c +++ b/source/dnode/mgmt/node_mgmt/src/dmMgmt.c @@ -18,6 +18,7 @@ #include "dmNodes.h" #include "index.h" #include "qworker.h" +#include "tstream.h" static bool dmRequireNode(SDnode *pDnode, SMgmtWrapper *pWrapper) { SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); @@ -153,6 +154,7 @@ int32_t dmInitDnode(SDnode *pDnode) { } indexInit(tsNumOfCommitThreads); + streamMetaInit(); dmReportStartup("dnode-transport", "initialized"); dDebug("dnode is created, ptr:%p", pDnode); @@ -175,6 +177,7 @@ void dmCleanupDnode(SDnode *pDnode) { dmCleanupServer(pDnode); dmClearVars(pDnode); rpcCleanup(); + streamMetaCleanup(); indexCleanup(); taosConvDestroy(); dDebug("dnode is closed, ptr:%p", pDnode); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 002407ce8ab5b93206077113da7ee09452f2e196..d4cbcaaacda393b52df4d9fc27d10de55be7ae03 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -751,7 +751,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SDnodeObj *pDnode = NULL; SCreateDnodeReq createReq = {0}; - if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0) { + if ((terrno = grantCheck(TSDB_GRANT_DNODE)) != 0 || (terrno = grantCheck(TSDB_GRANT_CPU_CORES)) != 0) { code = terrno; goto _OVER; } diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index df7955771d799b35030bc745df7f220e6920ed43..68b8dd72019df1ed623170249272e3cda3c5ad1a 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -1306,7 +1306,7 @@ int32_t mndPauseAllStreamTasks(STrans *pTrans, SStreamObj *pStream) { int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (mndPauseStreamTask(pTrans, pTask) < 0) { + if (pTask->taskLevel == TASK_LEVEL__SOURCE && mndPauseStreamTask(pTrans, pTask) < 0) { return -1; } } @@ -1430,7 +1430,7 @@ int32_t mndResumeAllStreamTasks(STrans *pTrans, SStreamObj *pStream, int8_t igUn int32_t sz = taosArrayGetSize(pTasks); for (int32_t j = 0; j < sz; j++) { SStreamTask *pTask = taosArrayGetP(pTasks, j); - if (mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { + if (pTask->taskLevel == TASK_LEVEL__SOURCE && mndResumeStreamTask(pTrans, pTask, igUntreated) < 0) { return -1; } } diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index d56595dae93c00e884d70c91bdcf9cff44aa53f6..c59669fc53c9006ae9c535fcb9cb671369adb61f 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -894,8 +894,10 @@ void destroyTimeSliceOperatorInfo(void* param) { } taosArrayDestroy(pInfo->pLinearInfo); - taosMemoryFree(pInfo->pPrevGroupKey->pData); - taosMemoryFree(pInfo->pPrevGroupKey); + if (pInfo->pPrevGroupKey) { + taosMemoryFree(pInfo->pPrevGroupKey->pData); + taosMemoryFree(pInfo->pPrevGroupKey); + } cleanupExprSupp(&pInfo->scalarSup); diff --git a/source/libs/stream/inc/streamBackendRocksdb.h b/source/libs/stream/inc/streamBackendRocksdb.h index 5d2970a4b70cb2ec68cb6c069d393513757871d4..0f39cf817b1bbd191d9ab49d3456be9e1bfa1c66 100644 --- a/source/libs/stream/inc/streamBackendRocksdb.h +++ b/source/libs/stream/inc/streamBackendRocksdb.h @@ -16,8 +16,6 @@ #ifndef _STREAM_BACKEDN_ROCKSDB_H_ #define _STREAM_BACKEDN_ROCKSDB_H_ -#include "executor.h" - #include "rocksdb/c.h" // #include "streamInc.h" #include "streamState.h" @@ -112,14 +110,6 @@ int32_t streamStateGetParName_rocksdb(SStreamState* pState, int64_t groupId, voi void streamStateDestroy_rocksdb(SStreamState* pState, bool remove); -void* streamStateCreateBatch(); -int32_t streamStateGetBatchSize(void* pBatch); -void streamStateClearBatch(void* pBatch); -void streamStateDestroyBatch(void* pBatch); -int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, - void* val, int32_t vlen); -int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch); - // default cf int32_t streamDefaultPut_rocksdb(SStreamState* pState, const void* key, void* pVal, int32_t pVLen); int32_t streamDefaultGet_rocksdb(SStreamState* pState, const void* key, void** pVal, int32_t* pVLen); @@ -138,7 +128,7 @@ int32_t streamStateGetBatchSize(void* pBatch); void streamStateClearBatch(void* pBatch); void streamStateDestroyBatch(void* pBatch); int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, - void* val, int32_t vlen); + void* val, int32_t vlen, int64_t ttl); int32_t streamStatePutBatch_rocksdb(SStreamState* pState, void* pBatch); // int32_t streamDefaultIter_rocksdb(SStreamState* pState, const void* start, const void* end, SArray* result); #endif \ No newline at end of file diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 71fbe5e086c9e0ae49ab78ab57f618ed87fd0c62..c471bc2bd8f41723067de1d0ae98d504aa4a1aa9 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -16,9 +16,12 @@ #ifndef _STREAM_INC_H_ #define _STREAM_INC_H_ -//#include "executor.h" +#include "executor.h" +#include "query.h" #include "tstream.h" +#include "trpc.h" + #ifdef __cplusplus extern "C" { #endif diff --git a/source/libs/stream/src/streamBackendRocksdb.c b/source/libs/stream/src/streamBackendRocksdb.c index db4ec17b19294b2396f10d24cc10baa0e8df9950..16ba81c74a610c761d6ad697b14a2db2ff5cba24 100644 --- a/source/libs/stream/src/streamBackendRocksdb.c +++ b/source/libs/stream/src/streamBackendRocksdb.c @@ -13,8 +13,9 @@ * along with this program. If not, see . */ -// #include "streamStateRocksdb.h" #include "streamBackendRocksdb.h" +#include "executor.h" +#include "query.h" #include "tcommon.h" typedef struct SCompactFilteFactory { @@ -110,6 +111,9 @@ void* streamBackendInit(const char* path) { taosMemoryFreeClear(err); } } else { + /* + list all cf and get prefix + */ int64_t streamId; int32_t taskId, dummpy = 0; SHashObj* tbl = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -649,18 +653,7 @@ const char* compactFilteFactoryName(void* arg) { void destroyCompactFilte(void* arg) { (void)arg; } unsigned char compactFilte(void* arg, int level, const char* key, size_t klen, const char* val, size_t vlen, char** newval, size_t* newvlen, unsigned char* value_changed) { - // int64_t unixTime = taosGetTimestampMs(); - if (streamStateValueIsStale((char*)val)) { - return 1; - } - // SStreamValue value; - // memset(&value, 0, sizeof(value)); - // streamValueDecode(&value, (char*)val); - // taosMemoryFree(value.data); - // if (value.unixTimestamp != 0 && value.unixTimestamp < unixTime) { - // return 1; - // } - return 0; + return streamStateValueIsStale((char*)val) ? 1 : 0; } const char* compactFilteName(void* arg) { return "stream_filte"; } @@ -703,7 +696,6 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids) { memcpy(cfNames[0], "default", strlen("default")); continue; } - qError("cf name %s", idstr); GEN_COLUMN_FAMILY_NAME(cfNames[i], idstr, ginitDict[(i - 1) % (cfLen)].key); if (i % cfLen == 0) { @@ -711,9 +703,6 @@ int32_t streamStateOpenBackendCf(void* backend, char* name, SHashObj* ids) { if (pIter != NULL) idstr = taosHashGetKey(pIter, &keyLen); } } - for (int i = 0; i < nSize * cfLen + 1; i++) { - qError("cf name %s", cfNames[i]); - } rocksdb_options_t** cfOpts = taosMemoryCalloc(nSize * cfLen + 1, sizeof(rocksdb_options_t*)); RocksdbCfParam* params = taosMemoryCalloc(nSize * cfLen + 1, sizeof(RocksdbCfParam*)); for (int i = 0; i < nSize * cfLen + 1; i++) { @@ -858,7 +847,6 @@ int streamStateOpenBackend(void* backend, SStreamState* pState) { if (err != NULL) { qError("failed to create cf:%s_%s, reason:%s", pState->pTdbState->idstr, ginitDict[i].key, err); taosMemoryFreeClear(err); - // return -1; } } pState->pTdbState->rocksdb = handle->db; @@ -1012,53 +1000,51 @@ rocksdb_iterator_t* streamStateIterCreate(SStreamState* pState, const char* cfNa taosMemoryFree(ttlV); \ } while (0); -#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ - do { \ - code = 0; \ - char buf[128] = {0}; \ - char* err = NULL; \ - int i = streamGetInit(funcname); \ - if (i < 0) { \ - qWarn("streamState failed to get cf name: %s", funcname); \ - code = -1; \ - break; \ - } \ - char toString[128] = {0}; \ - if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ - int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ - rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ - rocksdb_t* db = pState->pTdbState->rocksdb; \ - rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ - size_t len = 0; \ - char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ - if (val == NULL) { \ - qDebug("streamState str: %s failed to read from %s, err: not exist", toString, funcname); \ - if (err != NULL) taosMemoryFree(err); \ - code = -1; \ - } else { \ - char * p = NULL, *end = NULL; \ - int32_t len = ginitDict[i].deValueFunc(val, len, NULL, &p); \ - if (len < 0) { \ - qDebug("streamState str: %s failed to read from %s, err: %s, timeout", toString, funcname, err); \ - code = -1; \ - } else { \ - qDebug("streamState str: %s succ to read from %s, valLen:%d", toString, funcname, len); \ - } \ - if (pVal != NULL) { \ - *pVal = p; \ - } else { \ - taosMemoryFree(p); \ - } \ - taosMemoryFree(val); \ - if (vLen != NULL) *vLen = len; \ - } \ - if (err != NULL) { \ - taosMemoryFree(err); \ - qDebug("streamState str: %s failed to read from %s, err: %s", toString, funcname, err); \ - code = -1; \ - } else { \ - if (code == 0) qDebug("streamState str: %s succ to read from %s", toString, funcname); \ - } \ +#define STREAM_STATE_GET_ROCKSDB(pState, funcname, key, pVal, vLen) \ + do { \ + code = 0; \ + char buf[128] = {0}; \ + char* err = NULL; \ + int i = streamGetInit(funcname); \ + if (i < 0) { \ + qWarn("streamState failed to get cf name: %s", funcname); \ + code = -1; \ + break; \ + } \ + char toString[128] = {0}; \ + if (qDebugFlag & DEBUG_TRACE) ginitDict[i].toStrFunc((void*)key, toString); \ + int32_t klen = ginitDict[i].enFunc((void*)key, buf); \ + rocksdb_column_family_handle_t* pHandle = pState->pTdbState->pHandle[ginitDict[i].idx]; \ + rocksdb_t* db = pState->pTdbState->rocksdb; \ + rocksdb_readoptions_t* opts = pState->pTdbState->readOpts; \ + size_t len = 0; \ + char* val = rocksdb_get_cf(db, opts, pHandle, (const char*)buf, klen, (size_t*)&len, &err); \ + if (val == NULL) { \ + if (err == NULL) { \ + qDebug("streamState str: %s failed to read from %s_%s, err: not exist", toString, pState->pTdbState->idstr, \ + funcname); \ + } else { \ + qDebug("streamState str: %s failed to read from %s_%s, err: %s", toString, pState->pTdbState->idstr, funcname, \ + err); \ + taosMemoryFreeClear(err); \ + } \ + code = -1; \ + } else { \ + char* p = NULL; \ + int32_t len = ginitDict[i].deValueFunc(val, len, NULL, (char**)pVal); \ + if (len < 0) { \ + qDebug("streamState str: %s failed to read from %s_%s, err: already ttl ", toString, pState->pTdbState->idstr, \ + funcname); \ + code = -1; \ + } else { \ + qDebug("streamState str: %s succ to read from %s_%s, valLen:%d", toString, pState->pTdbState->idstr, funcname, \ + len); \ + } \ + taosMemoryFree(val); \ + if (vLen != NULL) *vLen = len; \ + } \ + if (code == 0) \ + qDebug("streamState str: %s succ to read from %s_%s", toString, pState->pTdbState->idstr, funcname); \ } while (0); #define STREAM_STATE_DEL_ROCKSDB(pState, funcname, key) \ @@ -1133,10 +1119,7 @@ int32_t streamStateClear_rocksdb(SStreamState* pState) { // rocksdb_compact_range_cf(pState->pTdbState->rocksdb, pState->pTdbState->pHandle[0], sKeyStr, sLen, eKeyStr, // eLen); if (err != NULL) { - qWarn( - "failed to delete range cf(state) err: %s, " - "start: %s, end:%s", - err, toStringStart, toStringEnd); + qWarn("failed to delete range cf(state) start: %s, end:%s, reason:%s", toStringStart, toStringEnd, err); taosMemoryFree(err); } @@ -1588,20 +1571,17 @@ int32_t streamStateFillGetKVByCur_rocksdb(SStreamStateCur* pCur, SWinKey* pKey, if (!rocksdb_iter_valid(pCur->iter) || iterValueIsStale(pCur->iter)) { return -1; } - size_t tlen; - char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &tlen); + size_t klen, vlen; + char* keyStr = (char*)rocksdb_iter_key(pCur->iter, &klen); winKeyDecode(&winKey, keyStr); - size_t vlen = 0; const char* valStr = rocksdb_iter_value(pCur->iter, &vlen); - char* dst = NULL; - int32_t len = decodeValueFunc((void*)valStr, vlen, NULL, &dst); + // char* dst = NULL; + int32_t len = decodeValueFunc((void*)valStr, vlen, NULL, (char**)pVal); if (len < 0) { return -1; } - - if (pVal != NULL) *pVal = (char*)dst; - if (pVLen != NULL) *pVLen = vlen; + if (pVLen != NULL) *pVLen = len; *pKey = winKey; return 0; @@ -1999,7 +1979,7 @@ int32_t streamStateGetBatchSize(void* pBatch) { void streamStateClearBatch(void* pBatch) { rocksdb_writebatch_clear((rocksdb_writebatch_t*)pBatch); } void streamStateDestroyBatch(void* pBatch) { rocksdb_writebatch_destroy((rocksdb_writebatch_t*)pBatch); } int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_writebatch_t* pBatch, void* key, - void* val, int32_t vlen) { + void* val, int32_t vlen, int64_t ttl) { int i = streamGetInit(cfName); if (i < 0) { @@ -2010,7 +1990,7 @@ int32_t streamStatePutBatch(SStreamState* pState, const char* cfName, rocksdb_wr int32_t klen = ginitDict[i].enFunc((void*)key, buf); char* ttlV = NULL; - int32_t ttlVLen = ginitDict[i].enValueFunc(val, vlen, 0, &ttlV); + int32_t ttlVLen = ginitDict[i].enValueFunc(val, vlen, ttl, &ttlV); rocksdb_column_family_handle_t* pCf = pState->pTdbState->pHandle[ginitDict[i].idx]; rocksdb_writebatch_put_cf((rocksdb_writebatch_t*)pBatch, pCf, buf, (size_t)klen, ttlV, (size_t)ttlVLen); taosMemoryFree(ttlV); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 0fb78fb589e60d8ebb1a562f589715351bb576cf..f4d8522f317ee4c0247edfcce134039d39674d8b 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -20,12 +20,12 @@ #define MIN_STREAM_EXEC_BATCH_NUM 16 bool streamTaskShouldStop(const SStreamStatus* pStatus) { - int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus); + int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); return (status == TASK_STATUS__STOP) || (status == TASK_STATUS__DROPPING); } bool streamTaskShouldPause(const SStreamStatus* pStatus) { - int32_t status = atomic_load_8((int8_t*) &pStatus->taskStatus); + int32_t status = atomic_load_8((int8_t*)&pStatus->taskStatus); return (status == TASK_STATUS__PAUSE); } @@ -35,7 +35,7 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* while (pTask->taskLevel == TASK_LEVEL__SOURCE) { int8_t status = atomic_load_8(&pTask->status.taskStatus); - if (status != TASK_STATUS__NORMAL) { + if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__PAUSE) { qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, atomic_load_8(&pTask->status.taskStatus)); taosMsleep(2); @@ -53,14 +53,14 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); - qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr, - pSubmit->submit.msgLen, pSubmit->submit.ver); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, + pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; SArray* pBlockList = pBlock->blocks; int32_t numOfBlocks = taosArrayGetSize(pBlockList); - qDebug("s-task:%s set sdata blocks as input num:%d, ver:%"PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%" PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; @@ -202,7 +202,8 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { qRes->blocks = pRes; code = streamTaskOutput(pTask, qRes); if (code == TSDB_CODE_UTIL_QUEUE_OUT_OF_MEMORY) { - taosFreeQitem(pRes); + taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); + taosFreeQitem(qRes); return code; } @@ -332,12 +333,12 @@ int32_t streamExecForAll(SStreamTask* pTask) { int64_t ckId = 0; int64_t dataVer = 0; qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); - if (ckId > pTask->chkInfo.id) { // save it since the checkpoint is updated + if (ckId > pTask->chkInfo.id) { // save it since the checkpoint is updated qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 ", checkPoint id:%" PRId64 " -> %" PRId64, pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); - pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer}; + pTask->chkInfo = (SCheckpointInfo){.version = dataVer, .id = ckId, .currentVer = pTask->chkInfo.currentVer}; taosWLockLatch(&pTask->pMeta->lock); @@ -407,7 +408,7 @@ int32_t streamTryExec(SStreamTask* pTask) { atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); qDebug("s-task:%s exec completed", pTask->id.idStr); - if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status))) { + if (!taosQueueEmpty(pTask->inputQueue->queue) && (!streamTaskShouldStop(&pTask->status)) && (!streamTaskShouldPause(&pTask->status))) { streamSchedExec(pTask); } } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 939e5910e75eb07233e510e81408a32e8abc20ad..a284984fe35f779595602659a03e976dee5799b5 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -19,6 +19,13 @@ #include "tref.h" #include "ttimer.h" +static TdThreadOnce streamMetaModuleInit = PTHREAD_ONCE_INIT; +static int32_t streamBackendId = 0; +static void streamMetaEnvInit() { streamBackendId = taosOpenRef(20, streamBackendCleanup); } + +void streamMetaInit() { taosThreadOnce(&streamMetaModuleInit, streamMetaEnvInit); } +void streamMetaCleanup() { taosCloseRef(streamBackendId); } + SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId) { int32_t code = -1; SStreamMeta* pMeta = taosMemoryCalloc(1, sizeof(SStreamMeta)); @@ -32,18 +39,16 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF sprintf(streamPath, "%s/%s", path, "stream"); pMeta->path = taosStrdup(streamPath); if (tdbOpen(pMeta->path, 16 * 1024, 1, &pMeta->db, 0) < 0) { - taosMemoryFree(streamPath); goto _err; } + memset(streamPath, 0, len); sprintf(streamPath, "%s/%s", pMeta->path, "checkpoints"); code = taosMulModeMkDir(streamPath, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); - taosMemoryFree(streamPath); goto _err; } - taosMemoryFree(streamPath); if (tdbTbOpen("task.db", sizeof(int32_t), -1, NULL, pMeta->db, &pMeta->pTaskDb, 0) < 0) { goto _err; @@ -74,26 +79,26 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF pMeta->vgId = vgId; pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; + pMeta->streamBackendId = streamBackendId; - char* statePath = taosMemoryCalloc(1, len); - sprintf(statePath, "%s/%s", pMeta->path, "state"); - code = taosMulModeMkDir(statePath, 0755); + memset(streamPath, 0, len); + sprintf(streamPath, "%s/%s", pMeta->path, "state"); + code = taosMulModeMkDir(streamPath, 0755); if (code != 0) { terrno = TAOS_SYSTEM_ERROR(code); - taosMemoryFree(streamPath); goto _err; } - pMeta->streamBackend = streamBackendInit(statePath); - pMeta->streamBackendId = taosOpenRef(20, streamBackendCleanup); - pMeta->streamBackendRid = taosAddRef(pMeta->streamBackendId, pMeta->streamBackend); + pMeta->streamBackend = streamBackendInit(streamPath); + pMeta->streamBackendRid = taosAddRef(streamBackendId, pMeta->streamBackend); - taosMemoryFree(statePath); + taosMemoryFree(streamPath); taosInitRWLatch(&pMeta->lock); return pMeta; _err: + taosMemoryFree(streamPath); taosMemoryFree(pMeta->path); if (pMeta->pTasks) taosHashCleanup(pMeta->pTasks); if (pMeta->pTaskList) taosArrayDestroy(pMeta->pTaskList); @@ -128,8 +133,7 @@ void streamMetaClose(SStreamMeta* pMeta) { } taosHashCleanup(pMeta->pTasks); - taosRemoveRef(pMeta->streamBackendId, pMeta->streamBackendRid); - taosCloseRef(pMeta->streamBackendId); + taosRemoveRef(streamBackendId, pMeta->streamBackendRid); pMeta->pTaskList = taosArrayDestroy(pMeta->pTaskList); taosMemoryFree(pMeta->path); taosMemoryFree(pMeta); @@ -263,13 +267,9 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); if (ppTask) { SStreamTask* pTask = *ppTask; - - // taosWLockLatch(&pMeta->lock); - taosHashRemove(pMeta->pTasks, &taskId, sizeof(int32_t)); tdbTbDelete(pMeta->pTaskDb, &taskId, sizeof(int32_t), pMeta->txn); - // atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); int32_t num = taosArrayGetSize(pMeta->pTaskList); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 1cca4d55cf34339810cc63a48565b27d122e629d..373cb27941f56abf55f8bcfb0486a67225703d3b 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -115,7 +115,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int pState->taskId = pTask->id.taskId; pState->streamId = pTask->id.streamId; #ifdef USE_ROCKSDB - qWarn("open stream state1"); + // qWarn("open stream state1"); taosAcquireRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); int code = streamStateOpenBackend(pTask->pMeta->streamBackend, pState); if (code == -1) { @@ -220,6 +220,7 @@ void streamStateClose(SStreamState* pState, bool remove) { #ifdef USE_ROCKSDB // streamStateCloseBackend(pState); streamStateDestroy(pState, remove); + taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); #else tdbCommit(pState->pTdbState->db, pState->pTdbState->txn); tdbPostCommit(pState->pTdbState->db, pState->pTdbState->txn); @@ -231,7 +232,6 @@ void streamStateClose(SStreamState* pState, bool remove) { tdbTbClose(pState->pTdbState->pParTagDb); tdbClose(pState->pTdbState->db); #endif - taosReleaseRef(pTask->pMeta->streamBackendId, pTask->pMeta->streamBackendRid); } int32_t streamStateBegin(SStreamState* pState) { @@ -399,7 +399,7 @@ int32_t streamStateSaveInfo(SStreamState* pState, void* pKey, int32_t keyLen, vo int32_t code = 0; void* batch = streamStateCreateBatch(); - code = streamStatePutBatch(pState, "default", batch, pKey, pVal, vLen); + code = streamStatePutBatch(pState, "default", batch, pKey, pVal, vLen, 0); if (code != 0) { return code; } diff --git a/source/libs/stream/src/tstreamFileState.c b/source/libs/stream/src/tstreamFileState.c index b7401ec5d97a3f73733483828c6977ce4d636b62..67835e77b8847d45d1015923bdc6bc3b035fdf5d 100644 --- a/source/libs/stream/src/tstreamFileState.c +++ b/source/libs/stream/src/tstreamFileState.c @@ -15,6 +15,7 @@ #include "tstreamFileState.h" +#include "query.h" #include "streamBackendRocksdb.h" #include "taos.h" #include "tcommon.h" @@ -154,9 +155,7 @@ void streamFileStateClear(SStreamFileState* pFileState) { clearExpiredRowBuff(pFileState, 0, true); } -bool needClearDiskBuff(SStreamFileState* pFileState) { - return pFileState->flushMark > 0; -} +bool needClearDiskBuff(SStreamFileState* pFileState) { return pFileState->flushMark > 0; } void popUsedBuffs(SStreamFileState* pFileState, SStreamSnapshot* pFlushList, uint64_t max, bool used) { uint64_t i = 0; @@ -325,7 +324,9 @@ bool hasRowBuff(SStreamFileState* pFileState, void* pKey, int32_t keyLen) { void releaseRowBuffPos(SRowBuffPos* pBuff) { pBuff->beUsed = false; } SStreamSnapshot* getSnapshot(SStreamFileState* pFileState) { - clearExpiredRowBuff(pFileState, pFileState->maxTs - pFileState->deleteMark, false); + int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN + : pFileState->maxTs - pFileState->deleteMark; + clearExpiredRowBuff(pFileState, mark, false); return pFileState->usedBuffs; } @@ -356,7 +357,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, } SStateKey sKey = {.key = *((SWinKey*)pPos->pKey), .opNum = ((SStreamState*)pFileState->pFileStore)->number}; - code = streamStatePutBatch(pFileState->pFileStore, "state", batch, &sKey, pPos->pRowBuff, pFileState->rowSize); + code = streamStatePutBatch(pFileState->pFileStore, "state", batch, &sKey, pPos->pRowBuff, pFileState->rowSize, 0); qDebug("===stream===put %" PRId64 " to disc, res %d", sKey.key.ts, code); } if (streamStateGetBatchSize(batch) > 0) { @@ -372,7 +373,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = 0; sprintf(keyBuf, "%s:%" PRId64 "", taskKey, ((SStreamState*)pFileState->pFileStore)->checkPointId); streamFileStateEncode(&pFileState->flushMark, &valBuf, &len); - code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len); + code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0); taosMemoryFree(valBuf); } { @@ -381,7 +382,7 @@ int32_t flushSnapshot(SStreamFileState* pFileState, SStreamSnapshot* pSnapshot, int32_t len = 0; memcpy(keyBuf, taskKey, strlen(taskKey)); len = sprintf(valBuf, "%" PRId64 "", ((SStreamState*)pFileState->pFileStore)->checkPointId); - code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len); + code = streamStatePutBatch(pFileState->pFileStore, "default", batch, keyBuf, valBuf, len, 0); } streamStatePutBatch_rocksdb(pFileState->pFileStore, batch); } @@ -440,7 +441,9 @@ int32_t deleteExpiredCheckPoint(SStreamFileState* pFileState, TSKEY mark) { int32_t recoverSnapshot(SStreamFileState* pFileState) { int32_t code = TSDB_CODE_SUCCESS; - deleteExpiredCheckPoint(pFileState, pFileState->maxTs - pFileState->deleteMark); + int64_t mark = (INT64_MIN + pFileState->deleteMark >= pFileState->maxTs) ? INT64_MIN + : pFileState->maxTs - pFileState->deleteMark; + deleteExpiredCheckPoint(pFileState, mark); void* pStVal = NULL; int32_t len = 0; diff --git a/source/libs/stream/test/CMakeLists.txt b/source/libs/stream/test/CMakeLists.txt index 5a97ba45f684dc444f3e7b5cb0cfdeade8728fd1..a0c171769025f62b0a70a3677d9faeab2a4f6693 100644 --- a/source/libs/stream/test/CMakeLists.txt +++ b/source/libs/stream/test/CMakeLists.txt @@ -10,7 +10,7 @@ ADD_EXECUTABLE(streamUpdateTest "tstreamUpdateTest.cpp") TARGET_LINK_LIBRARIES( streamUpdateTest - PUBLIC os util common gtest stream + PUBLIC os util common gtest gtest_main stream ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/stream/test/tstreamUpdateTest.cpp b/source/libs/stream/test/tstreamUpdateTest.cpp index c6981878744510b13a18c5795a3b9f05b968dc58..18c60aff284414e5ba5044d50000a9bd45718965 100644 --- a/source/libs/stream/test/tstreamUpdateTest.cpp +++ b/source/libs/stream/test/tstreamUpdateTest.cpp @@ -1,11 +1,28 @@ #include +#include "streamBackendRocksdb.h" +#include "tstream.h" #include "tstreamUpdate.h" #include "ttime.h" using namespace std; #define MAX_NUM_SCALABLE_BF 100000 +class StreamStateEnv : public ::testing::Test { + protected: + virtual void SetUp() { + streamMetaInit(); + backend = streamBackendInit(path); + } + virtual void TearDown() { + streamMetaCleanup(); + // indexClose(index); + } + + const char *path = TD_TMP_DIR_PATH "stream"; + void *backend; +}; + bool equalSBF(SScalableBf *left, SScalableBf *right) { if (left->growth != right->growth) return false; if (left->numBits != right->numBits) return false; @@ -191,8 +208,9 @@ TEST(TD_STREAM_UPDATE_TEST, update) { // updateInfoDestroy(pSU6); // updateInfoDestroy(pSU7); } - -int main(int argc, char *argv[]) { - testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); -} \ No newline at end of file +// TEST() +TEST(StreamStateEnv, test1) {} +// int main(int argc, char *argv[]) { +// testing::InitGoogleTest(&argc, argv); +// return RUN_ALL_TESTS(); +// } \ No newline at end of file