diff --git a/README.md b/README.md index 611d97aac9436bdcc732efcf98822f2dd11d74ab..a8126d74d44b2615f0d0a3e80fc330eaac6a2950 100644 --- a/README.md +++ b/README.md @@ -15,11 +15,11 @@ [![Coverage Status](https://coveralls.io/repos/github/taosdata/TDengine/badge.svg?branch=develop)](https://coveralls.io/github/taosdata/TDengine?branch=develop) [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/4201/badge)](https://bestpractices.coreinfrastructure.org/projects/4201) -English | [简体中文](README-CN.md) | We are hiring, check [here](https://tdengine.com/careers) +English | [简体中文](README-CN.md) | [Lean more about TSDB](https://tdengine.com/tsdb) # What is TDengine？ -TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/what-is-a-time-series-database/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages: +TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. TDengine differentiates itself from other time-seires databases with the following advantages: - **[High-Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. @@ -33,6 +33,8 @@ TDengine is an open source, high-performance, cloud native [time-series database - **[Open Source](https://tdengine.com/tdengine/open-source-time-series-database/)**: TDengine’s core modules, including cluster feature, are all available under open source licenses. It has gathered 18.8k stars on GitHub. There is an active developer community, and over 139k running instances worldwide. +For a full list of TDengine competitive advantages, please [check here](https://tdengine.com/tdengine/) + # Documentation For user manual, system design and architecture, please refer to [TDengine Documentation](https://docs.tdengine.com) ([TDengine 文档](https://docs.taosdata.com)) @@ -319,6 +321,7 @@ TDengine provides abundant developing tools for users to develop on TDengine. Fo Please follow the [contribution guidelines](CONTRIBUTING.md) to contribute to the project. -# Join TDengine WeChat Group +# Join TDengine User Community -Add WeChat “tdengine” to join the group，you can communicate with other users. +- Join [TDengine Discord Channel](https://discord.com/invite/VZdSuUg4pS?utm_id=discord) +- Join wechat group by adding WeChat “tdengine” diff --git a/cmake/cmake.version b/cmake/cmake.version index 7c895b12ff3397a221be5d2cadd95f4e0e64897d..d5721f767125721ef2d7ae897789b64debb6d0a1 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.1.0") + SET(TD_VER_NUMBER "3.0.1.1") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 0f7a450920b1a60a0af508062d104c26f592a15f..6fb2a69847316f97ba016ac2d71481c0f18d7b4c 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 3588b3d + GIT_TAG e7270c9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/01-index.md b/docs/en/01-index.md index 5265be42f81c4f43fa73e5b7d603d8989c2a5671..13552ea9dcfbae71a368cad9d55b4a0030ac266a 100644 --- a/docs/en/01-index.md +++ b/docs/en/01-index.md @@ -4,7 +4,7 @@ sidebar_label: Documentation Home slug: / --- -TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) time-series database optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It’s written mainly for architects, developers, and system administrators. +TDengine is an [open-source](https://tdengine.com/tdengine/open-source-time-series-database/), [cloud-native](https://tdengine.com/tdengine/cloud-native-time-series-database/) [time-series database](https://tdengine.com/tsdb/) optimized for the Internet of Things (IoT), Connected Cars, and Industrial IoT. It enables efficient, real-time data ingestion, processing, and monitoring of TB and even PB scale data per day, generated by billions of sensors and data collectors. This document is the TDengine user manual. It introduces the basic, as well as novel concepts, in TDengine, and also talks in detail about installation, features, SQL, APIs, operation, maintenance, kernel design, and other topics. It’s written mainly for architects, developers, and system administrators. To get an overview of TDengine, such as a feature list, benchmarks, and competitive advantages, please browse through the [Introduction](./intro) section. @@ -22,6 +22,8 @@ If you want to know more about TDengine tools, the REST API, and connectors for If you are very interested in the internal design of TDengine, please read the chapter [Inside TDengine](./tdinternal), which introduces the cluster design, data partitioning, sharding, writing, and reading processes in detail. If you want to study TDengine code or even contribute code, please read this chapter carefully. +To get more general introduction about time series database, please read through [a series of articles](https://tdengine.com/tsdb/). To lean more competitive advantages about TDengine, please read through [a series of blogs](https://tdengine.com/tdengine/). + TDengine is an open-source database, and we would love for you to be a part of TDengine. If you find any errors in the documentation or see parts where more clarity or elaboration is needed, please click "Edit this page" at the bottom of each page to edit it directly. Together, we make a difference! diff --git a/docs/en/02-intro/index.md b/docs/en/02-intro/index.md index d385845d7c57203d6e1cc8ddb8d53307f2655914..20840883523607512bd8c21643d49c5b50d3a528 100644 --- a/docs/en/02-intro/index.md +++ b/docs/en/02-intro/index.md @@ -3,7 +3,7 @@ title: Introduction toc_max_heading_level: 2 --- -TDengine is an open source, high-performance, cloud native [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](../develop/cache), [stream processing](../develop/stream), [data subscription](../develop/tmq) and other functionalities to reduce the system complexity and cost of development and operation. +TDengine is an [open source](https://tdengine.com/tdengine/open-source-time-series-database/), [high-performance](https://tdengine.com/tdengine/high-performance-time-series-database/), [cloud native](https://tdengine.com/tdengine/cloud-native-time-series-database/) [time-series database](https://tdengine.com/tsdb/) optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. Its code, including its cluster feature is open source under GNU AGPL v3.0. Besides the database engine, it provides [caching](../develop/cache), [stream processing](../develop/stream), [data subscription](../develop/tmq) and other functionalities to reduce the system complexity and cost of development and operation. This section introduces the major features, competitive advantages, typical use-cases and benchmarks to help you get a high level overview of TDengine. @@ -43,7 +43,7 @@ For more details on features, please read through the entire documentation. ## Competitive Advantages -By making full use of [characteristics of time series data](https://tdengine.com/tsdb/characteristics-of-time-series-data/), TDengine differentiates itself from other time series databases, with the following advantages. +By making full use of [characteristics of time series data](https://tdengine.com/tsdb/characteristics-of-time-series-data/), TDengine differentiates itself from other [time series databases](https://tdengine.com/tsdb), with the following advantages. - **[High-Performance](https://tdengine.com/tdengine/high-performance-time-series-database/)**: TDengine is the only time-series database to solve the high cardinality issue to support billions of data collection points while out performing other time-series databases for data ingestion, querying and data compression. @@ -127,3 +127,8 @@ As a high-performance, scalable and SQL supported time-series database, TDengine - [TDengine vs OpenTSDB](https://tdengine.com/2019/09/12/710.html) - [TDengine vs Cassandra](https://tdengine.com/2019/09/12/708.html) - [TDengine vs InfluxDB](https://tdengine.com/2019/09/12/706.html) + +## More readings +- [Introduction to Time-Series Database](https://tdengine.com/tsdb/) +- [Introduction to TDengine competitive advantages](https://tdengine.com/tdengine/) + diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index b0400de67388a433355358d827aa41f35598655c..292fa82183b5fb6d8c3449a5282e360f4841710a 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -111,7 +111,7 @@ Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the W :::info -For information about TDengine releases, see [Release History](../../releases). +For information about TDengine releases, see [Release History](../../releases/tdengine). ::: :::note diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 1f861119250e9e32fcac3344a5bdfb760e6191a7..118a2483737743ddbca7eece14f0a8a9a88993c4 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -348,19 +348,15 @@ SELECT ... FROM (SELECT ... FROM ...) ...; :::info -- Only one layer of nesting is allowed, that means no sub query is allowed within a sub query -- The result set returned by the inner query will be used as a "virtual table" by the outer query. The "virtual table" can be renamed using `AS` keyword for easy reference in the outer query. -- Sub query is not allowed in continuous query. +- The result of a nested query is returned as a virtual table used by the outer query. It's recommended to give an alias to this table for the convenience of using it in the outer query. - JOIN operation is allowed between tables/STables inside both inner and outer queries. Join operation can be performed on the result set of the inner query. -- UNION operation is not allowed in either inner query or outer query. -- The functions that can be used in the inner query are the same as those that can be used in a non-nested query. +- The features that can be used in the inner query are the same as those that can be used in a non-nested query. - `ORDER BY` inside the inner query is unnecessary and will slow down the query performance significantly. It is best to avoid the use of `ORDER BY` inside the inner query. - Compared to the non-nested query, the functionality that can be used in the outer query has the following restrictions: - Functions - - If the result set returned by the inner query doesn't contain timestamp column, then functions relying on timestamp can't be used in the outer query, like `TOP`, `BOTTOM`, `FIRST`, `LAST`, `DIFF`. - - Functions that need to scan the data twice can't be used in the outer query, like `STDDEV`, `PERCENTILE`. - - `IN` operator is not allowed in the outer query but can be used in the inner query. - - `GROUP BY` is not supported in the outer query. + - If the result set returned by the inner query doesn't contain timestamp column, then functions relying on timestamp can't be used in the outer query, like INTERP,DERIVATIVE, IRATE, LAST_ROW, FIRST, LAST, TWA, STATEDURATION, TAIL, UNIQUE. + - If the result set returned by the inner query are not sorted in order by timestamp, then functions relying on data ordered by timestamp can't be used in the outer query, like LEASTSQUARES, ELAPSED, INTERP, DERIVATIVE, IRATE, TWA, DIFF, STATECOUNT, STATEDURATION, CSUM, MAVG, TAIL, UNIQUE. + - Functions that need to scan the data twice can't be used in the outer query, like PERCENTILE. ::: diff --git a/docs/en/12-taos-sql/12-distinguished.md b/docs/en/12-taos-sql/12-distinguished.md index 707089abe54fc12bb09de47c1c51af1a32b8cbcd..98b782c3eb8cca43d14c78f321a4f2232930f30b 100644 --- a/docs/en/12-taos-sql/12-distinguished.md +++ b/docs/en/12-taos-sql/12-distinguished.md @@ -5,11 +5,11 @@ title: Time-Series Extensions As a purpose-built database for storing and processing time-series data, TDengine provides time-series-specific extensions to standard SQL. -These extensions include tag-partitioned queries and windowed queries. +These extensions include partitioned queries and windowed queries. -## Tag-Partitioned Queries +## Partitioned Queries -When you query a supertable, you may need to partition the supertable by tag and perform additional operations on a specific partition. In this case, you can use the following SQL clause: +When you query a supertable, you may need to partition the supertable by some dimensions and perform additional operations on a specific partition. In this case, you can use the following SQL clause: ```sql PARTITION BY part_list @@ -17,22 +17,24 @@ PARTITION BY part_list part_list can be any scalar expression, such as a column, constant, scalar function, or a combination of the preceding items. -A PARTITION BY clause with a tag is processed as follows: +A PARTITION BY clause is processed as follows: -- The PARTITION BY clause must occur after the WHERE clause and cannot be used with a JOIN clause. -- The PARTITION BY clause partitions the super table by the specified tag group, and the specified calculation is performed on each partition. The calculation performed is determined by the rest of the statement - a window clause, GROUP BY clause, or SELECT clause. -- You can use PARTITION BY together with a window clause or GROUP BY clause. In this case, the window or GROUP BY clause takes effect on every partition. For example, the following statement partitions the table by the location tag, performs downsampling over a 10 minute window, and returns the maximum value: +- The PARTITION BY clause must occur after the WHERE clause +- The PARTITION BY caluse partitions the data according to the specified dimentions, then perform computation on each partition. The performed computation is determined by the rest of the statement - a window clause, GROUP BY clause, or SELECT clause. +- The PARTITION BY clause can be used together with a window clause or GROUP BY clause. In this case, the window or GROUP BY clause takes effect on every partition. For example, the following statement partitions the table by the location tag, performs downsampling over a 10 minute window, and returns the maximum value: ```sql select max(current) from meters partition by location interval(10m) ``` +The most common usage of PARTITION BY is partitioning the data in subtables by tags then perform computation when querying data in a supertable. More specifically, `PARTITION BY TBNAME` partitions the data of each subtable into a single timeline, and this method facilitates the statistical analysis in many use cases of processing timeseries data. + ## Windowed Queries Aggregation by time window is supported in TDengine. For example, in the case where temperature sensors report the temperature every seconds, the average temperature for every 10 minutes can be retrieved by performing a query with a time window. Window related clauses are used to divide the data set to be queried into subsets and then aggregation is performed across the subsets. There are three kinds of windows: time window, status window, and session window. There are two kinds of time windows: sliding window and flip time/tumbling window. The query syntax is as follows: ```sql -SELECT function_list FROM tb_name +SELECT select_list FROM tb_name [WHERE where_condition] [SESSION(ts_col, tol_val)] [STATE_WINDOW(col)] @@ -42,12 +44,6 @@ SELECT function_list FROM tb_name The following restrictions apply: -### Restricted Functions - -- Aggregate functions and select functions can be used in `function_list`, with each function having only one output. For example COUNT, AVG, SUM, STDDEV, LEASTSQUARES, PERCENTILE, MIN, MAX, FIRST, LAST. Functions having multiple outputs, such as DIFF or arithmetic operations can't be used. -- `LAST_ROW` can't be used together with window aggregate. -- Scalar functions, like CEIL/FLOOR, can't be used with window aggregate. - ### Other Rules - The window clause must occur after the PARTITION BY clause and before the GROUP BY clause. It cannot be used with a GROUP BY clause. @@ -82,7 +78,7 @@ These pseudocolumns occur after the aggregation clause. 1. A huge volume of interpolation output may be returned using `FILL`, so it's recommended to specify the time range when using `FILL`. The maximum number of interpolation values that can be returned in a single query is 10,000,000. 2. The result set is in ascending order of timestamp when you aggregate by time window. -3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `GROUP BY` is not used in the query, the result set will be returned in ascending order of timestamp; otherwise the result set is not exactly in the order of ascending timestamp in each group. +3. If aggregate by window is used on STable, the aggregate function is performed on all the rows matching the filter conditions. If `PARTITION BY` is not used in the query, the result set will be returned in strict ascending order of timestamp; otherwise the result set is not exactly in the order of ascending timestamp in each group. ::: @@ -112,9 +108,9 @@ When using time windows, note the following: Please note that the `timezone` parameter should be configured to be the same value in the `taos.cfg` configuration file on client side and server side. - The result set is in ascending order of timestamp when you aggregate by time window. -### Status Window +### State Window -In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two status windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12]. Status window is not applicable to STable for now. +In case of using integer, bool, or string to represent the status of a device at any given moment, continuous rows with the same status belong to a status window. Once the status changes, the status window closes. As shown in the following figure, there are two state windows according to status, [2019-04-28 14:22:07，2019-04-28 14:22:10] and [2019-04-28 14:22:11，2019-04-28 14:22:12]. ![TDengine Database Status Window](./timewindow-3.webp) @@ -130,7 +126,7 @@ The primary key, i.e. timestamp, is used to determine which session window a row ![TDengine Database Session Window](./timewindow-2.webp) -If the time interval between two continuous rows are within the time interval specified by `tol_value` they belong to the same session window; otherwise a new session window is started automatically. Session window is not supported on STable for now. +If the time interval between two continuous rows are within the time interval specified by `tol_value` they belong to the same session window; otherwise a new session window is started automatically. ``` diff --git a/docs/en/12-taos-sql/25-grant.md b/docs/en/12-taos-sql/25-grant.md index b9a3fa2321c8d073845d0cf9157ce335c930e06f..f895567c621f123778c8a711a68357a889ca0098 100644 --- a/docs/en/12-taos-sql/25-grant.md +++ b/docs/en/12-taos-sql/25-grant.md @@ -9,15 +9,54 @@ This document describes how to manage permissions in TDengine. ## Create a User ```sql -CREATE USER use_name PASS 'password'; +CREATE USER user_name PASS 'password' [SYSINFO {1|0}]; ``` This statement creates a user account. -The maximum length of use_name is 23 bytes. +The maximum length of user_name is 23 bytes. The maximum length of password is 128 bytes. The password can include leters, digits, and special characters excluding single quotation marks, double quotation marks, backticks, backslashes, and spaces. The password cannot be empty. +`SYSINFO` indicates whether the user is allowed to view system information. `1` means allowed, `0` means not allowed. System information includes server configuration, dnode, vnode, storage. The default value is `1`. + +For example, we can create a user whose password is `123456` and is able to view system information. + +```sql +taos> create user test pass '123456' sysinfo 1; +Query OK, 0 of 0 rows affected (0.001254s) +``` + +## View Users + +To show the users in the system, please use + +```sql +SHOW USERS; +``` + +This is an example: + +```sql +taos> show users; + name | super | enable | sysinfo | create_time | +================================================================================ + test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 | + root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 | +Query OK, 2 rows in database (0.001657s) +``` + +Alternatively, you can get the user information by querying a built-in table, INFORMATION_SCHEMA.INS_USERS. For example: + +```sql +taos> select * from information_schema.ins_users; + name | super | enable | sysinfo | create_time | +================================================================================ + test | 0 | 1 | 1 | 2022-08-29 15:10:27.315 | + root | 1 | 1 | 1 | 2022-08-29 15:03:34.710 | +Query OK, 2 rows in database (0.001953s) +``` + ## Delete a User ```sql @@ -40,6 +79,13 @@ alter_user_clause: { - ENABLE: Specify whether the user is enabled or disabled. 1 indicates enabled and 0 indicates disabled. - SYSINFO: Specify whether the user can query system information. 1 indicates that the user can query system information and 0 indicates that the user cannot query system information. +For example, you can use below command to disable user `test`: + +```sql +taos> alter user test enable 0; +Query OK, 0 of 0 rows affected (0.001160s) +``` + ## Grant Permissions @@ -62,7 +108,7 @@ priv_level : { } ``` -Grant permissions to a user. +Grant permissions to a user, this feature is only available in enterprise edition. Permissions are granted on the database level. You can grant read or write permissions. @@ -92,4 +138,4 @@ priv_level : { ``` -Revoke permissions from a user. +Revoke permissions from a user, this feature is only available in enterprise edition. diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index a4679983f2bc77bb4e438f5d43fa1b8beb39b120..74a5564a2a7b82009e6b2406290a5c15afbf3c1d 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -2,7 +2,7 @@ title: TDengine Monitoring --- -After TDengine is started, a database named `log` is created automatically to help with monitoring. Information that includes CPU, memory and disk usage, bandwidth, number of requests, disk I/O speed, slow queries, is written into the `log` database at a predefined interval. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console. +After TDengine is started, it automatically writes monitoring data including CPU, memory and disk usage, bandwidth, number of requests, disk I/O speed, slow queries, into a designated database at a predefined interval through taosKeeper. Additionally, some important system operations, like logon, create user, drop database, and alerts and warnings generated in TDengine are written into the `log` database too. A system operator can view the data in `log` database from TDengine CLI or from a web console. The collection of the monitoring information is enabled by default, but can be disabled by parameter `monitor` in the configuration file. @@ -10,7 +10,7 @@ The collection of the monitoring information is enabled by default, but can be d TDinsight is a complete solution which uses the monitoring database `log` mentioned previously, and Grafana, to monitor a TDengine cluster. -From version 2.3.3.0, more monitoring data has been added in the `log` database. Please refer to [TDinsight Grafana Dashboard](https://grafana.com/grafana/dashboards/15167) to learn more details about using TDinsight to monitor TDengine. +Please refer to [TDinsight Grafana Dashboard](../../reference/tdinsight) to learn more details about using TDinsight to monitor TDengine. A script `TDinsight.sh` is provided to deploy TDinsight automatically. @@ -30,31 +30,14 @@ Prepare： 2. Grafana Alert Notification -There are two ways to setup Grafana alert notification. +You can use below command to setup Grafana alert notification. -- An existing Grafana Notification Channel can be specified with parameter `-E`, the notifier uid of the channel can be obtained by `curl -u admin:admin localhost:3000/api/alert-notifications |jq` +An existing Grafana Notification Channel can be specified with parameter `-E`, the notifier uid of the channel can be obtained by `curl -u admin:admin localhost:3000/api/alert-notifications |jq` ```bash sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -E ``` -- The AliCloud SMS alert built in TDengine data source plugin can be enabled with parameter `-s`, the parameters of enabling this plugin are listed below: - - - `-I`: AliCloud SMS Key ID - - `-K`: AliCloud SMS Key Secret - - `-S`: AliCloud SMS Signature - - `-C`: SMS notification template - - `-T`: Input parameters in JSON format for the SMS notification template, for example`{"alarm_level":"%s","time":"%s","name":"%s","content":"%s"}` - - `-B`: List of mobile numbers to be notified - - Below is an example of the full command using the AliCloud SMS alert. - - ```bash - sudo ./TDinsight.sh -a http://localhost:6041 -u root -p taosdata -s \ - -I XXXXXXX -K XXXXXXXX -S taosdata -C SMS_1111111 -B 18900000000 \ - -T '{"alarm_level":"%s","time":"%s","name":"%s","content":"%s"}' - ``` - Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`. For more use cases and restrictions please refer to [TDinsight](/reference/tdinsight/). diff --git a/docs/en/14-reference/03-connector/_linux_install.mdx b/docs/en/14-reference/03-connector/_linux_install.mdx index 07f8fb5c7118d84c53017f44d9811a3357944cfc..97f6cd333961d24bd015a2ebcf7223f89fecca2c 100644 --- a/docs/en/14-reference/03-connector/_linux_install.mdx +++ b/docs/en/14-reference/03-connector/_linux_install.mdx @@ -4,7 +4,7 @@ import PkgListV3 from "/components/PkgListV3"; - [All Downloads](../../releases) + [All Downloads](../../releases/tdengine) 2. Unzip diff --git a/docs/en/14-reference/03-connector/_windows_install.mdx b/docs/en/14-reference/03-connector/_windows_install.mdx index ea638ed1ed6c64c3ec4ceaea436f65dd1f09a27e..81fc5573bb85c6fd1b501f1ff4ec072df8f23db1 100644 --- a/docs/en/14-reference/03-connector/_windows_install.mdx +++ b/docs/en/14-reference/03-connector/_windows_install.mdx @@ -4,7 +4,7 @@ import PkgListV3 from "/components/PkgListV3"; - [All Downloads](../../releases) + [All Downloads](../../releases/tdengine) 2. Execute the installer, select the default value as prompted, and complete the installation 3. Installation path diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index e7ea620d0bed3aee3ff0acf8063120acca33c335..78c4febb92f11225ad2aa82630f4892f6e786a4d 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -30,7 +30,7 @@ taosAdapter provides the following features. ### Install taosAdapter -If you use the TDengine server, you don't need additional steps to install taosAdapter. You can download taosAdapter from [TDengine 3.0 released versions](../../releases) to download the TDengine server installation package. If you need to deploy taosAdapter separately on another server other than the TDengine server, you should install the full TDengine server package on that server to install taosAdapter. If you need to build taosAdapter from source code, you can refer to the [Building taosAdapter]( https://github.com/taosdata/taosadapter/blob/3.0/BUILD.md) documentation. +If you use the TDengine server, you don't need additional steps to install taosAdapter. You can download taosAdapter from [TDengine 3.0 released versions](../../releases/tdengine) to download the TDengine server installation package. If you need to deploy taosAdapter separately on another server other than the TDengine server, you should install the full TDengine server package on that server to install taosAdapter. If you need to build taosAdapter from source code, you can refer to the [Building taosAdapter]( https://github.com/taosdata/taosadapter/blob/3.0/BUILD.md) documentation. ### Start/Stop taosAdapter diff --git a/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json b/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json index 54dc1062d6440cc0fc7b8c69d9e4c6b53e4cd01e..f651983528ca824b4e6b14586aac5a5bfb4ecab8 100644 --- a/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json +++ b/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json @@ -211,7 +211,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Leader MNode", + "title": "Master MNode", "transformations": [ { "id": "filterByValue", @@ -221,7 +221,7 @@ "config": { "id": "regex", "options": { - "value": "leader" + "value": "master" } }, "fieldName": "role" @@ -300,7 +300,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Leader MNode Create Time", + "title": "Master MNode Create Time", "transformations": [ { "id": "filterByValue", @@ -310,7 +310,7 @@ "config": { "id": "regex", "options": { - "value": "leader" + "value": "master" } }, "fieldName": "role" diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-1-cluster-status.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-1-cluster-status.webp index a78e18028a94c2f6a783b08d992a25c791527407..3bc0d960f1db45ee8d2adcee26de89334e681956 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-1-cluster-status.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-1-cluster-status.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-2-dnodes.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-2-dnodes.webp index b152418d0902b8ebdf62ebce6705c10dd5ab4fbf..f5a602d3f9dcecb64ded5e1f463ba460daab0024 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-2-dnodes.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-2-dnodes.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-3-mnodes.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-3-mnodes.webp index f58f48b7f17375cb8e62e7c0126ca3aea56a13f6..f155fa42a0fb5df71ee48c8c65a8c7d8851ddc3e 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-3-mnodes.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-3-mnodes.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-4-requests.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-4-requests.webp index 00afcce013602dce0da17bfd033f65aaa8e43bb7..dc0b85e262bd4340e986a42105e0ff9838d12fa6 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-4-requests.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-4-requests.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-5-database.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-5-database.webp index 567e5694f9d7a035a3eb354493d3df8ed64db251..342c8cfc0a8e852e7cd092aff453ed1fd2ec85a2 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-5-database.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-5-database.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp b/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp index 8666193f59497180574fd2786266e5baabbe9761..942130d4fabf7944c7add10acb3bb42ca7f51e0f 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp and b/docs/en/14-reference/07-tdinsight/assets/TDinsight-8-taosadapter.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/howto-add-datasource.webp b/docs/en/14-reference/07-tdinsight/assets/howto-add-datasource.webp index 06d0ff6ed50091a6340508bc5b2b3f78b65dcb18..d7fc9e233acd1a4b1bbb940b13bc4296c261a33a 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/howto-add-datasource.webp and b/docs/en/14-reference/07-tdinsight/assets/howto-add-datasource.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/import_dashboard.webp b/docs/en/14-reference/07-tdinsight/assets/import_dashboard.webp index fb7958f1b9fbd43c8f63136024842790e711c490..ae2a1e8e9b7b63a68d56dfcd2187eca614da9a3d 100644 Binary files a/docs/en/14-reference/07-tdinsight/assets/import_dashboard.webp and b/docs/en/14-reference/07-tdinsight/assets/import_dashboard.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/import_dashboard_view.webp b/docs/en/14-reference/07-tdinsight/assets/import_dashboard_view.webp new file mode 100644 index 0000000000000000000000000000000000000000..1b10e41c75fbbb9a30bce4aa8d1adb8216fbe127 Binary files /dev/null and b/docs/en/14-reference/07-tdinsight/assets/import_dashboard_view.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/select_dashboard_db.webp b/docs/en/14-reference/07-tdinsight/assets/select_dashboard_db.webp new file mode 100644 index 0000000000000000000000000000000000000000..956132e37e9df255d3ff82654fd357bec001e695 Binary files /dev/null and b/docs/en/14-reference/07-tdinsight/assets/select_dashboard_db.webp differ diff --git a/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json b/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json index 1add8522a712aa2cfef6187e577c42d205432b66..b4254c428b28a0084e54b5e3c509dd2e0ec651b9 100644 --- a/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json +++ b/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json @@ -153,7 +153,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Leader MNode", + "title": "Master MNode", "transformations": [ { "id": "filterByValue", @@ -163,7 +163,7 @@ "config": { "id": "regex", "options": { - "value": "leader" + "value": "master" } }, "fieldName": "role" @@ -246,7 +246,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Leader MNode Create Time", + "title": "Master MNode Create Time", "transformations": [ { "id": "filterByValue", @@ -256,7 +256,7 @@ "config": { "id": "regex", "options": { - "value": "leader" + "value": "master" } }, "fieldName": "role" diff --git a/docs/en/14-reference/07-tdinsight/index.md b/docs/en/14-reference/07-tdinsight/index.md index 2e562035254311f2caa0b6d4512842080aab64d5..d03c16a8bc25a39fb751ded4b5c12df4b69a8e4e 100644 --- a/docs/en/14-reference/07-tdinsight/index.md +++ b/docs/en/14-reference/07-tdinsight/index.md @@ -5,15 +5,23 @@ sidebar_label: TDinsight TDinsight is a solution for monitoring TDengine using the builtin native monitoring database and [Grafana]. -After TDengine starts, it will automatically create a monitoring database `log`. TDengine will automatically write many metrics in specific intervals into the `log` database. The metrics may include the server's CPU, memory, hard disk space, network bandwidth, number of requests, disk read/write speed, slow queries, other information like important system operations (user login, database creation, database deletion, etc.), and error alarms. With [Grafana] and [TDengine Data Source Plugin](https://github.com/taosdata/grafanaplugin/releases), TDinsight can visualize cluster status, node information, insertion and query requests, resource usage, vnode, dnode, and mnode status, exception alerts and many other metrics. This is very convenient for developers who want to monitor TDengine cluster status in real-time. This article will guide users to install the Grafana server, automatically install the TDengine data source plug-in, and deploy the TDinsight visualization panel using the `TDinsight.sh` installation script. +After TDengine starts, it automatically writes many metrics in specific intervals into a designated database. The metrics may include the server's CPU, memory, hard disk space, network bandwidth, number of requests, disk read/write speed, slow queries, other information like important system operations (user login, database creation, database deletion, etc.), and error alarms. With [Grafana] and [TDengine Data Source Plugin](https://github.com/taosdata/grafanaplugin/releases), TDinsight can visualize cluster status, node information, insertion and query requests, resource usage, vnode, dnode, and mnode status, exception alerts and many other metrics. This is very convenient for developers who want to monitor TDengine cluster status in real-time. This article will guide users to install the Grafana server, automatically install the TDengine data source plug-in, and deploy the TDinsight visualization panel using the `TDinsight.sh` installation script. ## System Requirements -To deploy TDinsight, a single-node TDengine server or a multi-node TDengine cluster and a [Grafana] server are required. This dashboard requires TDengine 2.3.3.0 and above, with the `log` database enabled (`monitor = 1`). +To deploy TDinsight, we need +- a single-node TDengine server or a multi-node TDengine cluster and a [Grafana] server are required. This dashboard requires TDengine 3.0.1.0 and above, with the monitoring feature enabled. For detailed configuration, please refer to [TDengine monitoring configuration](../config/#monitoring-parameters). +- taosAdapter has been instaleld and running, please refer to [taosAdapter](../taosadapter). +- taosKeeper has been installed and running, please refer to [taosKeeper](../taoskeeper). + +Please record +- The endpoint of taosAdapter REST service, for example `http://tdengine.local:6041` +- Authentication of taosAdapter, e.g. user name and password +- The database name used by taosKeeper to store monitoring data ## Installing Grafana -We recommend using the latest [Grafana] version 7 or 8 here. You can install Grafana on any [supported operating system](https://grafana.com/docs/grafana/latest/installation/requirements/#supported-operating-systems) by following the [official Grafana documentation Instructions](https://grafana.com/docs/grafana/latest/installation/) to install [Grafana]. +We recommend using the latest [Grafana] version 8 or 9 here. You can install Grafana on any [supported operating system](https://grafana.com/docs/grafana/latest/installation/requirements/#supported-operating-systems) by following the [official Grafana documentation Instructions](https://grafana.com/docs/grafana/latest/installation/) to install [Grafana]. ### Installing Grafana on Debian or Ubuntu @@ -71,7 +79,7 @@ chmod +x TDinsight.sh ./TDinsight.sh ``` -This script will automatically download the latest [Grafana TDengine data source plugin](https://github.com/taosdata/grafanaplugin/releases/latest) and [TDinsight dashboard](https://grafana.com/grafana/dashboards/15167) with configurable parameters for command-line options to the [Grafana Provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) configuration file to automate deployment and updates, etc. With the alert setting options provided by this script, you can also get built-in support for AliCloud SMS alert notifications. +This script will automatically download the latest [Grafana TDengine data source plugin](https://github.com/taosdata/grafanaplugin/releases/latest) and [TDinsight dashboard](https://github.com/taosdata/grafanaplugin/blob/master/dashboards/TDinsightV3.json) with configurable parameters for command-line options to the [Grafana Provisioning](https://grafana.com/docs/grafana/latest/administration/provisioning/) configuration file to automate deployment and updates, etc. With the alert setting options provided by this script, you can also get built-in support for AliCloud SMS alert notifications. Assume you use TDengine and Grafana's default services on the same host. Run `. /TDinsight.sh` and open the Grafana browser window to see the TDinsight dashboard. @@ -106,18 +114,6 @@ Install and configure TDinsight dashboard in Grafana on Ubuntu 18.04/20.04 syste -E, --external-notifier Apply external notifier uid to TDinsight dashboard. -Alibaba Cloud SMS as Notifier: --s, --sms-enabled To enable tdengine-datasource plugin builtin Alibaba Cloud SMS webhook. --N, --sms-notifier-name Provisioning notifier name.[default: TDinsight Builtin SMS] --U, --sms-notifier-uid Provisioning notifier uid, use lowercase notifier name by default. --D, --sms-notifier-is-default Set notifier as default. --I, --sms-access-key-id Alibaba Cloud SMS access key id --K, --sms-access-key-secret Alibaba Cloud SMS access key secret --S, --sms-sign-name Sign name --C, --sms-template-code Template code --T, --sms-template-param Template param, a escaped JSON string like '{"alarm_level":"%s","time":"%s","name":"%s","content":"%s"}' --B, --sms-phone-numbers Comma-separated numbers list, eg "189xxxxxxxx,132xxxxxxxx" --L, --sms-listen-addr [default: 127.0.0.1:9100] ``` Most command-line options can take effect the same as environment variables. @@ -136,17 +132,6 @@ Most command-line options can take effect the same as environment variables. | -t | --tdinsight-title | TDINSIGHT_DASHBOARD_TITLE | TDinsight dashboard title. [Default: TDinsight] | -e | -tdinsight-title | -e | --tdinsight-editable | TDINSIGHT_DASHBOARD_EDITABLE | If the dashboard is configured to be editable. [Default: false] | -e | --external | -E | --external-notifier | EXTERNAL_NOTIFIER | Apply the external notifier uid to the TDinsight dashboard. | -s -| -s | --sms-enabled | SMS_ENABLED | Enable the tdengine-datasource plugin built into Alibaba Cloud SMS webhook. | -s -| -N | --sms-notifier-name | SMS_NOTIFIER_NAME | The name of the provisioning notifier. [Default: `TDinsight Builtin SMS`] | -U -| -U | --sms-notifier-uid | SMS_NOTIFIER_UID | "Notification Channel" `uid`, lowercase of the program name is used by default, other characters are replaced by "-". |-sms -| -D | --sms-notifier-is-default | SMS_NOTIFIER_IS_DEFAULT | Set built-in SMS notification to default value. |-sms-notifier-is-default -| -I | --sms-access-key-id | SMS_ACCESS_KEY_ID | Alibaba Cloud SMS access key id | -| -K | --sms-access-key-secret | SMS_ACCESS_KEY_SECRET | AliCloud SMS-access-secret-key | -| -S | --sms-sign-name | SMS_SIGN_NAME | Signature | -| -C | --sms-template-code | SMS_TEMPLATE_CODE | Template code | -| -T | --sms-template-param | SMS_TEMPLATE_PARAM | JSON template for template parameters | -| -B | --sms-phone-numbers | SMS_PHONE_NUMBERS | A comma-separated list of phone numbers, e.g. `"189xxxxxxxx,132xxxxxxxx"` | -| -L | --sms-listen-addr | SMS_LISTEN_ADDR | Built-in SMS webhook listener address, default is `127.0.0.1:9100` | Suppose you start a TDengine database on host `tdengine` with HTTP API port `6041`, user `root1`, and password `pass5ord`. Execute the script. @@ -166,24 +151,10 @@ Use the `uid` value obtained above as `-E` input. sudo ./TDinsight.sh -a http://tdengine:6041 -u root1 -p pass5ord -E existing-notifier ``` -If you want to use the [Alibaba Cloud SMS](https://www.aliyun.com/product/sms) service as a notification channel, you should enable it with the `-s` flag add the following parameters. - -- `-N`: Notification Channel name, default is `TDinsight Builtin SMS`. -- `-U`: Channel uid, default is lowercase of `name`, any other character is replaced with -, for the default `-N`, its uid is `tdinsight-builtin-sms`. -- `-I`: Alibaba Cloud SMS access key id. -- `-K`: Alibaba Cloud SMS access secret key. -- `-S`: Alibaba Cloud SMS signature. -- `-C`: Alibaba Cloud SMS template id. -- `-T`: Alibaba Cloud SMS template parameters, for JSON format template, example is as follows `'{"alarm_level":"%s", "time":"%s", "name":"%s", "content":"%s"}'`. There are four parameters: alarm level, time, name and alarm content. -- `-B`: a list of phone numbers, separated by a comma `,`. - If you want to monitor multiple TDengine clusters, you need to set up numerous TDinsight dashboards. Setting up non-default TDinsight requires some changes: the `-n` `-i` `-t` options need to be changed to non-default names, and `-N` and `-L` should also be changed if using the built-in SMS alerting feature. ```bash sudo . /TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata -i tdinsight-env1 -t 'TDinsight Env1' -# If using built-in SMS notifications -sudo . /TDengine.sh -n TDengine-Env1 -a http://another:6041 -u root -p taosdata -i tdinsight-env1 -t 'TDinsight Env1' \ - -s -N 'Env1 SMS' -I xx -K xx -S xx -C SMS_XX -T '' -B 00000000000 -L 127.0.0.01:10611 ``` Please note that the configuration data source, notification channel, and dashboard are not changeable on the front end. You should update the configuration again via this script or manually change the configuration file in the `/etc/grafana/provisioning` directory (this is the default directory for Grafana, use the `-P` option to change it as needed). @@ -249,21 +220,23 @@ Save and test. It will report 'TDengine Data source is working' under normal cir ### Importing dashboards -Point to **+** / **Create** - **import** (or `/dashboard/import` url). +In the page of configuring data source, click **Dashboards** tab. ![TDengine Database TDinsight Import Dashboard and Configuration](./assets/import_dashboard.webp) -Type the dashboard ID `15167` in the **Import via grafana.com** location and **Load**. +Choose `TDengine for 3.x` and click `import`. + +After the importing is done, `TDinsight for 3.x` dashboard is available on the page of `search dashboards by name`. -![TDengine Database TDinsight Import via grafana.com](./assets/import-dashboard-15167.webp) +![TDengine Database TDinsight Import via grafana.com](./assets/import_dashboard_view.webp) -Once the import is complete, the full page view of TDinsight is shown below. +In the `TDinsight for 3.x` dashboard, choose the database used by taosKeeper to store monitoring data, you can see the monitoring result. -![TDengine Database TDinsight show](./assets/TDinsight-full.webp) +![TDengine Database TDinsight 选择数据库](./assets/select_dashboard_db.webp) ## TDinsight dashboard details -The TDinsight dashboard is designed to provide the usage and status of TDengine-related resources [dnodes, mnodes, vnodes](../../taos-sql/node/) or databases. +The TDinsight dashboard is designed to provide the usage and status of TDengine-related resources, e.g. dnodes, mnodes, vnodes and databases. Details of the metrics are as follows. @@ -285,7 +258,6 @@ This section contains the current information and status of the cluster, the ale - **Measuring Points Used**: The number of measuring points used to enable the alert rule (no data available in the community version, healthy by default). - **Grants Expire Time**: the expiration time of the enterprise version of the enabled alert rule (no data available for the community version, healthy by default). - **Error Rate**: Aggregate error rate (average number of errors per second) for alert-enabled clusters. -- **Variables**: `show variables` table display. ### DNodes Status @@ -294,7 +266,6 @@ This section contains the current information and status of the cluster, the ale - **DNodes Status**: simple table view of `show dnodes`. - **DNodes Lifetime**: the time elapsed since the dnode was created. - **DNodes Number**: the number of DNodes changes. -- **Offline Reason**: if any dnode status is offline, the reason for offline is shown as a pie chart. ### MNode Overview @@ -309,7 +280,6 @@ This section contains the current information and status of the cluster, the ale 1. **Requests Rate(Inserts per Second)**: average number of inserts per second. 2. **Requests (Selects)**: number of query requests and change rate (count of second). -3. **Requests (HTTP)**: number of HTTP requests and request rate (count of second). ### Database @@ -319,9 +289,8 @@ Database usage, repeated for each value of the variable `$database` i.e. multipl 1. **STables**: number of super tables. 2. **Total Tables**: number of all tables. -3. **Sub Tables**: the number of all super table subtables. -4. **Tables**: graph of all normal table numbers over time. -5. **Tables Number Foreach VGroups**: The number of tables contained in each VGroups. +3. **Tables**: number of normal tables. +4. **Table number for each vgroup**: number of tables per vgroup. ### DNode Resource Usage @@ -356,12 +325,11 @@ Currently, only the number of logins per minute is reported. Support monitoring taosAdapter request statistics and status details. Includes. -1. **http_request**: contains the total number of requests, the number of failed requests, and the number of requests being processed -2. **top 3 request endpoint**: data of the top 3 requests by endpoint group -3. **Memory Used**: taosAdapter memory usage -4. **latency_quantile(ms)**: quantile of (1, 2, 5, 9, 99) stages -5. **top 3 failed request endpoint**: data of the top 3 failed requests by endpoint grouping -6. **CPU Used**: taosAdapter CPU usage +1. **http_request_inflight**: number of real-time requests. +2. **http_request_total**: number of total requests. +3. **http_request_fail**: number of failed requets. +4. **CPU Used**: CPU usage of taosAdapter. +5. **Memory Used**: Memory usage of taosAdapter. ## Upgrade @@ -403,13 +371,6 @@ services: TDENGINE_API: ${TDENGINE_API} TDENGINE_USER: ${TDENGINE_USER} TDENGINE_PASS: ${TDENGINE_PASS} - SMS_ACCESS_KEY_ID: ${SMS_ACCESS_KEY_ID} - SMS_ACCESS_KEY_SECRET: ${SMS_ACCESS_KEY_SECRET} - SMS_SIGN_NAME: ${SMS_SIGN_NAME} - SMS_TEMPLATE_CODE: ${SMS_TEMPLATE_CODE} - SMS_TEMPLATE_PARAM: '${SMS_TEMPLATE_PARAM}' - SMS_PHONE_NUMBERS: $SMS_PHONE_NUMBERS - SMS_LISTEN_ADDR: ${SMS_LISTEN_ADDR} ports: - 3000:3000 volumes: diff --git a/docs/en/20-third-party/13-Jupyter.md b/docs/en/20-third-party/13-Jupyter.md new file mode 100644 index 0000000000000000000000000000000000000000..fbd7e530f0959740c53e48ce1d73d92ce0d6c5c5 --- /dev/null +++ b/docs/en/20-third-party/13-Jupyter.md @@ -0,0 +1,98 @@ +--- +sidebar_label: JupyterLab +title: Connect JupyterLab to TDengine +--- + +JupyterLab is the next generation of the ubiquitous Jupyter Notebook. In this note we show you how to install the TDengine Python connector to connect to TDengine in JupyterLab. You can then insert data and perform queries against the TDengine instance within JupyterLab. + +## Install JupyterLab +Installing JupyterLab is very easy. Installation instructions can be found at: + +https://jupyterlab.readthedocs.io/en/stable/getting_started/installation.html. + +If you don't feel like clicking on the link here are the instructions. +Jupyter's preferred Python package manager is pip, so we show the instructions for pip. +You can also use **conda** or **pipenv** if you are managing Python environments. +```` +pip install jupyterlab +```` + +For **conda** you can run: +```` +conda install -c conda-forge jupyterlab +```` + +For **pipenv** you can run: +```` +pipenv install jupyterlab +pipenv shell +```` + +## Run JupyterLab +You can start JupyterLab from the command line by running: +```` +jupyter lab +```` +This will automatically launch your default browser and connect to your JupyterLab instance, usually on port 8888. + +## Install the TDengine Python connector +You can now install the TDengine Python connector as follows. + +Start a new Python kernel in JupyterLab. + +If using **conda** run the following: +```` +# Install a conda package in the current Jupyter kernel +import sys +!conda install --yes --prefix {sys.prefix} taospy +```` +If using **pip** run the following: +```` +# Install a pip package in the current Jupyter kernel +import sys +!{sys.executable} -m pip install taospy +```` + +## Connect to TDengine +You can find detailed examples to use the Python connector, in the TDengine documentation here. +Once you have installed the TDengine Python connector in your JupyterLab kernel, the process of connecting to TDengine is the same as that you would use if you weren't using JupyterLab. +Each TDengine instance, has a database called "log" which has monitoring information about the TDengine instance. +In the "log" database there is a [supertable](https://docs.tdengine.com/taos-sql/stable/) called "disks_info". + +The structure of this table is as follows: +```` +taos> desc disks_info; + Field | Type | Length | Note | +================================================================================= + ts | TIMESTAMP | 8 | | + datadir_l0_used | FLOAT | 4 | | + datadir_l0_total | FLOAT | 4 | | + datadir_l1_used | FLOAT | 4 | | + datadir_l1_total | FLOAT | 4 | | + datadir_l2_used | FLOAT | 4 | | + datadir_l2_total | FLOAT | 4 | | + dnode_id | INT | 4 | TAG | + dnode_ep | BINARY | 134 | TAG | +Query OK, 9 row(s) in set (0.000238s) +```` + +The code below is used to fetch data from this table into a pandas DataFrame. + +```` +import sys +import taos +import pandas + +def sqlQuery(conn): + df: pandas.DataFrame = pandas.read_sql("select * from log.disks_info limit 500", conn) + print(df) + return df + +conn = taos.connect() + +result = sqlQuery(conn) + +print(result) +```` + +TDengine has connectors for various languages including Node.js, Go, PHP and there are kernels for these languages which can be found [here](https://github.com/jupyter/jupyter/wiki/Jupyter-kernels). diff --git a/docs/en/28-releases.md b/docs/en/28-releases.md deleted file mode 100644 index a0c9eb119999571fb973b5e2243f237b8833b167..0000000000000000000000000000000000000000 --- a/docs/en/28-releases.md +++ /dev/null @@ -1,9 +0,0 @@ ---- -sidebar_label: Releases -title: Released Versions ---- - -import Release from "/components/ReleaseV3"; - - - diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md new file mode 100644 index 0000000000000000000000000000000000000000..e3901114d3271d040934f0a211135beecbdc2f7f --- /dev/null +++ b/docs/en/28-releases/01-tdengine.md @@ -0,0 +1,17 @@ +--- +sidebar_label: TDengine +title: TDengine +description: TDengine release history, Release Notes and download links. +--- + +import Release from "/components/ReleaseV3"; + + +## 3.0.1.1 + + + +## 3.0.1.0 + + + diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md new file mode 100644 index 0000000000000000000000000000000000000000..228990df3b15bf7ee111433994813dbb171843ab --- /dev/null +++ b/docs/en/28-releases/02-tools.md @@ -0,0 +1,15 @@ +--- +sidebar_label: taosTools +title: taosTools +description: taosTools release history, Release Notes, download links. +--- + +import Release from "/components/ReleaseV3"; + +## 2.2.0 + + + +## 2.1.3 + + diff --git a/docs/en/28-releases/_category_.yml b/docs/en/28-releases/_category_.yml new file mode 100644 index 0000000000000000000000000000000000000000..c1638352061f083c3b283d7517466fef5abdc9ea --- /dev/null +++ b/docs/en/28-releases/_category_.yml @@ -0,0 +1 @@ +label: Releases \ No newline at end of file diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 66863d1bd9419e6d8fc78cf1536e1efac38885d6..2c857d37f3a8591316949f9a7440a115740a8e2d 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -73,6 +73,7 @@ install.sh 安装脚本在执行过程中，会通过命令行交互界面询问 + 可以使用 `apt-get` 工具从官方仓库安装。 **配置包仓库** diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 2f5c13d9b0bc0e3940fb99b45c693e2ae80c8f47..d9a35ab4eb545872a0f9a8e10e6e40dcf6ee4ec7 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -218,7 +218,7 @@ void Close() ```sql DROP DATABASE IF EXISTS tmqdb; CREATE DATABASE tmqdb; -CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16) TAGS(t1 INT, t3 VARCHAR(16)); +CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16)); CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0"); CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); INSERT INTO tmqdb.ctb0 VALUES(now, 0, 0, 'a0')(now+1s, 0, 0, 'a00'); diff --git a/docs/zh/10-deployment/01-deploy.md b/docs/zh/10-deployment/01-deploy.md index 03b4ce30f980cd77e9845076ce9bb35c4474f948..eecb86ce415b6ad0a24a7fcd0d6d9d9ce1357a33 100644 --- a/docs/zh/10-deployment/01-deploy.md +++ b/docs/zh/10-deployment/01-deploy.md @@ -168,7 +168,7 @@ Query OK, 8 row(s) in set (0.001154s) ## 删除数据节点 -先停止要删除的数据节点的 taosd 进程，然后启动 CLI 程序 taos，执行： +启动 CLI 程序 taos，执行： ```sql DROP DNODE "fqdn:port"; diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index b0a7d88efea4811b193117f220871af64341ff97..c49351b0c901f60b5a02f3a08a5d1c44135631c3 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -104,7 +104,7 @@ SELECT location, groupid, current FROM d1001 LIMIT 2; ### 结果去重 -`DISINTCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。 +`DISTINCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。 对标签列去重： @@ -356,7 +356,7 @@ SELECT ... FROM (SELECT ... FROM ...) ...; - 与非嵌套的查询语句相比，外层查询所能支持的功能特性存在如下限制： - 计算函数部分： - 如果内层查询的结果数据未提供时间戳，那么计算过程隐式依赖时间戳的函数在外层会无法正常工作。例如：INTERP, DERIVATIVE, IRATE, LAST_ROW, FIRST, LAST, TWA, STATEDURATION, TAIL, UNIQUE。 - - 如果内层查询的结果数据不是有效的时间序列，那么计算过程依赖数据为时间序列的函数在外层会无法正常工作。例如：LEASTSQUARES, ELAPSED, INTERP, DERIVATIVE, IRATE, TWA, DIFF, STATECOUNT, STATEDURATION, CSUM, MAVG, TAIL, UNIQUE。 + - 如果内层查询的结果数据不是按时间戳有序，那么计算过程依赖数据按时间有序的函数在外层会无法正常工作。例如：LEASTSQUARES, ELAPSED, INTERP, DERIVATIVE, IRATE, TWA, DIFF, STATECOUNT, STATEDURATION, CSUM, MAVG, TAIL, UNIQUE。 - 计算过程需要两遍扫描的函数，在外层查询中无法正常工作。例如：此类函数包括：PERCENTILE。 ::: diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index ae0a496f03e8e545525fce49ae2394a10696c09c..7780dc2fe9a5742ce9d6855599ca080804a493b5 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -79,7 +79,7 @@ password = "taosdata" # 需要被监控的 taosAdapter [taosAdapter] -address = ["127.0.0.1:6041","192.168.1.95:6041"] +address = ["127.0.0.1:6041"] [metrics] # 监控指标前缀 @@ -92,7 +92,7 @@ cluster = "production" database = "log" # 指定需要监控的普通表 -tables = ["normal_table"] +tables = [] ``` ### 获取监控指标 @@ -141,4 +141,4 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 # HELP taos_cluster_info_first_ep # TYPE taos_cluster_info_first_ep gauge taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 -``` \ No newline at end of file +``` diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index a6ec560d3c5951ea1500893640be5905a31e8d61..59a241b6c8458b42313b55f8e814ea574c6fd32c 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,6 +6,11 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; + +## 3.0.1.1 + + + ## 3.0.1.0 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 9e8757cc4e09f661cfd4c552ff27537411e2d420..9ff8f4222471cc4245885909e2255869ccf1cb99 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,10 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; +## 2.2.0 + + + ## 2.1.3 diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index df16f4f0ab9ad1a79c11ede9e54fdc086e9204df..5b495601756d246abd08a3ef07f5efa8a6b47e62 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -36,8 +36,13 @@ typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; typedef struct STagVal STagVal; typedef struct STag STag; +typedef struct SColData SColData; -// bitmap +#define HAS_NONE ((uint8_t)0x1) +#define HAS_NULL ((uint8_t)0x2) +#define HAS_VALUE ((uint8_t)0x4) + +// bitmap ================================ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00000100, 0b00001000, 2}, {0b00000000, 0b00010000, 0b00100000, 4}, @@ -51,21 +56,21 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, #define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) -// STSchema +// STSchema ================================ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); -// SValue +// SValue ================================ int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type); int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); -// SColVal +// SColVal ================================ #define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1}) #define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1}) #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)}) -// STSRow2 +// STSRow2 ================================ #define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) #define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) @@ -77,7 +82,7 @@ int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); -// STSRowBuilder +// STSRowBuilder ================================ #define tsRowBuilderInit() ((STSRowBuilder){0}) #define tsRowBuilderClear(B) \ do { \ @@ -86,7 +91,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); } \ } while (0) -// STag +// STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); bool tTagIsJson(const void *pTag); @@ -100,7 +105,16 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); -// STRUCT ================= +// SColData ================================ +void tColDataDestroy(void *ph); +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); +void tColDataClear(SColData *pColData); +int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); +void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); +uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal); +int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); + +// STRUCT ================================ struct STColumn { col_id_t colId; int8_t type; @@ -166,6 +180,18 @@ struct SColVal { SValue value; }; +struct SColData { + int16_t cid; + int8_t type; + int8_t smaOn; + int32_t nVal; + uint8_t flag; + uint8_t *pBitMap; + int32_t *aOffset; + int32_t nData; + uint8_t *pData; +}; + #pragma pack(push, 1) struct STagVal { // char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 8aeb86102a7b4237276f59f25fe50d36c6f99efa..1e86a04775cd0a56c6581d1eb4ad87cf4d7c018b 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -151,6 +151,8 @@ typedef struct SVnodeModifyLogicNode { SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; SNode* pAffectedRows; // SColumnNode + SNode* pStartTs; // SColumnNode + SNode* pEndTs; // SColumnNode uint64_t tableId; uint64_t stableId; int8_t tableType; // table type @@ -525,6 +527,8 @@ typedef struct SDataDeleterNode { char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; SNode* pAffectedRows; + SNode* pStartTs; + SNode* pEndTs; } SDataDeleterNode; typedef struct SSubplan { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 3a1eaf289e4ba245544b985e893f746845c37c88..e90c994e8f2770fa95da82c71f5cc9ba16b1425f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -315,6 +315,8 @@ typedef struct SDeleteStmt { SNode* pFromTable; // FROM clause SNode* pWhere; // WHERE clause SNode* pCountFunc; // count the number of rows affected + SNode* pFirstFunc; // the start timestamp when the data was actually deleted + SNode* pLastFunc; // the end timestamp when the data was actually deleted SNode* pTagCond; // pWhere divided into pTagCond and timeRange STimeWindow timeRange; uint8_t precision; diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index e484e3c59c0e48535f25ad0640e7a93092603f73..7ce80553a0b53a1e100478ae83953de68b9ad702 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -173,7 +173,8 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { pTscObj->pAppInfo->totalDnodes = pRsp->query->totalDnodes; pTscObj->pAppInfo->onlineDnodes = pRsp->query->onlineDnodes; pTscObj->connId = pRsp->query->connId; - tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, pTscObj->pAppInfo->totalDnodes); + tscTrace("conn %p hb rsp, dnodes %d/%d", pTscObj->connId, pTscObj->pAppInfo->onlineDnodes, + pTscObj->pAppInfo->totalDnodes); if (pRsp->query->killRid) { tscDebug("request rid %" PRIx64 " need to be killed now", pRsp->query->killRid); @@ -297,7 +298,8 @@ static int32_t hbAsyncCallBack(void *param, SDataBuf *pMsg, int32_t code) { if (code != 0) { (*pInst)->onlineDnodes = ((*pInst)->totalDnodes ? 0 : -1); - tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, (*pInst)->totalDnodes); + tscDebug("hb rsp error %s, update server status %d/%d", tstrerror(code), (*pInst)->onlineDnodes, + (*pInst)->totalDnodes); } if (rspNum) { @@ -657,6 +659,8 @@ int32_t hbGatherAppInfo(void) { for (int32_t i = 0; i < sz; ++i) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + if (pAppHbMgr == NULL) continue; + uint64_t clusterId = pAppHbMgr->pAppInstInfo->clusterId; SAppHbReq *pApp = taosHashGet(clientHbMgr.appSummary, &clusterId, sizeof(clusterId)); if (NULL == pApp) { @@ -694,15 +698,21 @@ static void *hbThreadFunc(void *param) { hbGatherAppInfo(); } + SArray *mgr = taosArrayInit(sz, sizeof(void *)); for (int i = 0; i < sz; i++) { SAppHbMgr *pAppHbMgr = taosArrayGetP(clientHbMgr.appHbMgrs, i); + if (pAppHbMgr == NULL) { + continue; + } int32_t connCnt = atomic_load_32(&pAppHbMgr->connKeyCnt); if (connCnt == 0) { + taosArrayPush(mgr, &pAppHbMgr); continue; } SClientHbBatchReq *pReq = hbGatherAllInfo(pAppHbMgr); - if (pReq == NULL) { + if (pReq == NULL || taosArrayGetP(clientHbMgr.appHbMgrs, i) == NULL) { + tFreeClientHbBatchReq(pReq); continue; } int tlen = tSerializeSClientHbBatchReq(NULL, 0, pReq); @@ -711,6 +721,7 @@ static void *hbThreadFunc(void *param) { terrno = TSDB_CODE_TSC_OUT_OF_MEMORY; tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); + taosArrayPush(mgr, &pAppHbMgr); break; } @@ -722,6 +733,7 @@ static void *hbThreadFunc(void *param) { tFreeClientHbBatchReq(pReq); // hbClearReqInfo(pAppHbMgr); taosMemoryFree(buf); + taosArrayPush(mgr, &pAppHbMgr); break; } pInfo->fp = hbAsyncCallBack; @@ -729,7 +741,7 @@ static void *hbThreadFunc(void *param) { pInfo->msgInfo.len = tlen; pInfo->msgType = TDMT_MND_HEARTBEAT; pInfo->param = strdup(pAppHbMgr->key); - pInfo->paramFreeFp = taosMemoryFree; + pInfo->paramFreeFp = taosMemoryFree; pInfo->requestId = generateRequestId(); pInfo->requestObjRefId = 0; @@ -741,8 +753,12 @@ static void *hbThreadFunc(void *param) { // hbClearReqInfo(pAppHbMgr); atomic_add_fetch_32(&pAppHbMgr->reportCnt, 1); + taosArrayPush(mgr, &pAppHbMgr); } + taosArrayDestroy(clientHbMgr.appHbMgrs); + clientHbMgr.appHbMgrs = mgr; + taosThreadMutexUnlock(&clientHbMgr.lock); taosMsleep(HEARTBEAT_INTERVAL); @@ -834,7 +850,7 @@ void hbRemoveAppHbMrg(SAppHbMgr **pAppHbMgr) { if (pItem == *pAppHbMgr) { hbFreeAppHbMgr(*pAppHbMgr); *pAppHbMgr = NULL; - taosArrayRemove(clientHbMgr.appHbMgrs, i); + taosArraySet(clientHbMgr.appHbMgrs, i, pAppHbMgr); break; } } @@ -845,6 +861,7 @@ void appHbMgrCleanup(void) { int sz = taosArrayGetSize(clientHbMgr.appHbMgrs); for (int i = 0; i < sz; i++) { SAppHbMgr *pTarget = taosArrayGetP(clientHbMgr.appHbMgrs, i); + if (pTarget == NULL) continue; hbFreeAppHbMgr(pTarget); } } @@ -859,7 +876,14 @@ int hbMgrInit() { clientHbMgr.appSummary = taosHashInit(10, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); clientHbMgr.appHbMgrs = taosArrayInit(0, sizeof(void *)); - taosThreadMutexInit(&clientHbMgr.lock, NULL); + + TdThreadMutexAttr attr = {0}; + taosThreadMutexAttrSetType(&attr, PTHREAD_MUTEX_RECURSIVE); + int ret = taosThreadMutexAttrInit(&attr); + assert(ret == 0); + + taosThreadMutexInit(&clientHbMgr.lock, &attr); + taosThreadMutexAttrDestroy(&attr); // init handle funcs hbMgrInitHandle(); diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b40f449a0550140784250b9c2250d191552e4652..15a369fe404da6e279afe14bdaf3e68ac6a07f8a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tdataformat.h" +#include "tRealloc.h" #include "tcoding.h" #include "tdatablock.h" #include "tlog.h" @@ -680,7 +681,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { return n; } -// STSchema +// STSchema ======================================== int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); if (*ppTSchema == NULL) { @@ -720,9 +721,7 @@ void tTSchemaDestroy(STSchema *pTSchema) { if (pTSchema) taosMemoryFree(pTSchema); } -// STSRowBuilder - -// STag +// STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { return -1; @@ -1172,4 +1171,495 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { return pSchema; } -#endif \ No newline at end of file +#endif + +// SColData ======================================== +void tColDataDestroy(void *ph) { + SColData *pColData = (SColData *)ph; + + tFree(pColData->pBitMap); + tFree((uint8_t *)pColData->aOffset); + tFree(pColData->pData); +} + +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { + pColData->cid = cid; + pColData->type = type; + pColData->smaOn = smaOn; + tColDataClear(pColData); +} + +void tColDataClear(SColData *pColData) { + pColData->nVal = 0; + pColData->flag = 0; + pColData->nData = 0; +} + +static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColData->type)) { + code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1)); + if (code) goto _exit; + pColData->aOffset[pColData->nVal] = pColData->nData; + + if (pColVal->value.nData) { + code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (code) goto _exit; + memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); + pColData->nData += pColVal->value.nData; + } + } else { + ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); + code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); + if (code) goto _exit; + pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type); + } + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 + int32_t code = 0; + + if (pColVal->isNone) { + pColData->flag = HAS_NONE; + } else if (pColVal->isNull) { + pColData->flag = HAS_NULL; + } else { + pColData->flag = HAS_VALUE; + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE + int32_t code = 0; + + if (!pColVal->isNone) { + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + + code = tRealloc(&pColData->pBitMap, nBit); + if (code) goto _exit; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + if (pColVal->isNull) { + pColData->flag |= HAS_NULL; + } else { + pColData->flag |= HAS_VALUE; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) goto _exit; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) goto _exit; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL + int32_t code = 0; + + if (!pColVal->isNull) { + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) goto _exit; + + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else { + pColData->flag |= HAS_VALUE; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) goto _exit; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) goto _exit; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE + int32_t code = 0; + + if (pColVal->isNone) { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else if (pColVal->isNull) { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + } else { + pColData->flag |= HAS_VALUE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pColData->nVal, 2); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) goto _exit; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) goto _exit; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE + int32_t code = 0; + + if (pColVal->isNone || pColVal->isNull) { + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + } else { + pColData->flag |= HAS_NULL; + } + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) goto _exit; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } else { + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE + int32_t code = 0; + + if (pColVal->isNull) { + pColData->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pColData->nVal, 1); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + } else { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + if (pColVal->isNone) { + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else { + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + } + } + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL + int32_t code = 0; + + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pColData->nVal, 0); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + } else { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + if (pColVal->isNull) { + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else { + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + } + } + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, + SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + if (pColVal->isNone) { + SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + } else if (pColVal->isNull) { + SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + } else { + SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + } + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + + pColData->nVal++; + +_exit: + return code; +} +static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { + tColDataAppendValue0, // 0 + tColDataAppendValue1, // HAS_NONE + tColDataAppendValue2, // HAS_NULL + tColDataAppendValue3, // HAS_NULL|HAS_NONE + tColDataAppendValue4, // HAS_VALUE + tColDataAppendValue5, // HAS_VALUE|HAS_NONE + tColDataAppendValue6, // HAS_VALUE|HAS_NULL + tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE +}; +int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { + ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); + return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); +} + +static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); +} +static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); +} +static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL|HAS_NONE + switch (GET_BIT1(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + break; + case 1: + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + break; + default: + ASSERT(0); + } +} +static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_VALUE + SValue value; + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iVal + 1 < pColData->nVal) { + value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal]; + } else { + value.nData = pColData->nData - pColData->aOffset[iVal]; + } + value.pData = pColData->pData + pColData->aOffset[iVal]; + } else { + tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); + } + *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); +} +static FORCE_INLINE void tColDataGetValue5(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_VALUE|HAS_NONE + switch (GET_BIT1(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + break; + case 1: + tColDataGetValue4(pColData, iVal, pColVal); + break; + default: + ASSERT(0); + } +} +static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_VALUE|HAS_NULL + switch (GET_BIT1(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + break; + case 1: + tColDataGetValue4(pColData, iVal, pColVal); + break; + default: + ASSERT(0); + } +} +static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE + switch (GET_BIT2(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + break; + case 1: + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + break; + case 2: + tColDataGetValue4(pColData, iVal, pColVal); + break; + default: + ASSERT(0); + } +} +static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal *pColVal) = { + NULL, // 0 + tColDataGetValue1, // HAS_NONE + tColDataGetValue2, // HAS_NULL + tColDataGetValue3, // HAS_NULL | HAS_NONE + tColDataGetValue4, // HAS_VALUE + tColDataGetValue5, // HAS_VALUE | HAS_NONE + tColDataGetValue6, // HAS_VALUE | HAS_NULL + tColDataGetValue7 // HAS_VALUE | HAS_NULL | HAS_NONE +}; +void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { + ASSERT(iVal >= 0 && iVal < pColData->nVal && pColData->flag); + tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal); +} + +uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) { + uint8_t v; + switch (pColData->flag) { + case HAS_NONE: + v = 0; + break; + case HAS_NULL: + v = 1; + break; + case (HAS_NULL | HAS_NONE): + v = GET_BIT1(pColData->pBitMap, iVal); + break; + case HAS_VALUE: + v = 2; + break; + case (HAS_VALUE | HAS_NONE): + v = GET_BIT1(pColData->pBitMap, iVal); + if (v) v = 2; + break; + case (HAS_VALUE | HAS_NULL): + v = GET_BIT1(pColData->pBitMap, iVal) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + v = GET_BIT2(pColData->pBitMap, iVal); + break; + default: + ASSERT(0); + break; + } + return v; +} + +int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { + int32_t code = 0; + int32_t size; + + ASSERT(pColDataSrc->nVal > 0); + ASSERT(pColDataDest->cid = pColDataSrc->cid); + ASSERT(pColDataDest->type = pColDataSrc->type); + + pColDataDest->smaOn = pColDataSrc->smaOn; + pColDataDest->nVal = pColDataSrc->nVal; + pColDataDest->flag = pColDataSrc->flag; + + // bitmap + if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { + size = BIT2_SIZE(pColDataSrc->nVal); + code = tRealloc(&pColDataDest->pBitMap, size); + if (code) goto _exit; + memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + } + + // offset + if (IS_VAR_DATA_TYPE(pColDataDest->type)) { + size = sizeof(int32_t) * pColDataSrc->nVal; + + code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); + if (code) goto _exit; + + memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + } + + // value + pColDataDest->nData = pColDataSrc->nData; + code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); + if (code) goto _exit; + memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); + +_exit: + return code; +} \ No newline at end of file diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index 9c4a5afb032e6677997b7a84e919451b238b2068..030f4542b97666492f18964ac732517956822077 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -790,10 +790,20 @@ static bool mndCheckTransConflict(SMnode *pMnode, STrans *pNew) { if (mndCheckDbConflict(pNew->dbname1, pTrans)) conflict = true; if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true; } + if (pTrans->conflict == TRN_CONFLICT_DB_INSIDE) { + if (mndCheckDbConflict(pNew->dbname2, pTrans)) conflict = true; // for stb + } } - mError("trans:%d, can't execute since conflict with trans:%d, db1:%s db2:%s", pNew->id, pTrans->id, pTrans->dbname1, - pTrans->dbname2); + if (conflict) { + mError("trans:%d, db1:%s db2:%s type:%d, can't execute since conflict with trans:%d db1:%s db2:%s type:%d", + pNew->id, pNew->dbname1, pNew->dbname2, pNew->conflict, pTrans->id, pTrans->dbname1, pTrans->dbname2, + pTrans->conflict); + } else { + mDebug("trans:%d, db1:%s db2:%s type:%d, not conflict with trans:%d db1:%s db2:%s type:%d", pNew->id, + pNew->dbname1, pNew->dbname2, pNew->conflict, pTrans->id, pTrans->dbname1, pTrans->dbname2, + pTrans->conflict); + } sdbRelease(pMnode->pSdb, pTrans); } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index a836fa2bc5c678e144dbcfed659eeb180d74eb69..817089f23796ba218988205e939adae8cdc97d37 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -44,7 +44,6 @@ typedef struct SMapData SMapData; typedef struct SBlockIdx SBlockIdx; typedef struct SDataBlk SDataBlk; typedef struct SSttBlk SSttBlk; -typedef struct SColData SColData; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; typedef struct SDelFile SDelFile; @@ -71,10 +70,6 @@ typedef struct SLDataIter SLDataIter; #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 -#define HAS_NONE ((int8_t)0x1) -#define HAS_NULL ((int8_t)0x2) -#define HAS_VALUE ((int8_t)0x4) - #define VERSION_MIN 0 #define VERSION_MAX INT64_MAX @@ -148,15 +143,6 @@ int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); -// SColdata -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); -void tColDataReset(SColData *pColData); -void tColDataClear(void *ph); -int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); -int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); -int32_t tPutColData(uint8_t *p, SColData *pColData); -int32_t tGetColData(uint8_t *p, SColData *pColData); // SBlockData #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) @@ -299,8 +285,8 @@ int32_t tsdbDelFReaderClose(SDelFReader **ppReader); int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData); int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== -int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap); -void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); +int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap, const char* id); +void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap, const char* id); // tsdbMerge.c ============================================================================================== int32_t tsdbMerge(STsdb *pTsdb); @@ -470,18 +456,6 @@ struct SSttBlk { SBlockInfo bInfo; }; -struct SColData { - int16_t cid; - int8_t type; - int8_t smaOn; - int32_t nVal; - uint8_t flag; - uint8_t *pBitMap; - int32_t *aOffset; - int32_t nData; - uint8_t *pData; -}; - // (SBlockData){.suid = 0, .uid = 0}: block data not initialized // (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index f9c2757c3778446dfc75cca2fa55556e54d28d44..c8841e5e169c7cdf249894348ab949a225f58acc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -832,7 +832,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { tDecoderClear(pCoder); int32_t sz = taosArrayGetSize(pRes->uidList); - if (sz == 0) { + if (sz == 0 || pRes->affectedRows == 0) { taosArrayDestroy(pRes->uidList); return 0; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 8da783a5bd82ccd23fe051e76396489cc6516058..8452d14c869f1a41c81b904b90e13a49169f8f1e 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -847,7 +847,7 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs tb_uid_t suid = getTableSuidByUid(uid, pTsdb); - tsdbTakeReadSnap(pTsdb, &pIter->pReadSnap); + tsdbTakeReadSnap(pTsdb, &pIter->pReadSnap, NULL); STbData *pMem = NULL; if (pIter->pReadSnap->pMem) { @@ -941,7 +941,7 @@ static int32_t nextRowIterClose(CacheNextRowIter *pIter) { taosArrayDestroy(pIter->pSkyline); } - tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap); + tsdbUntakeReadSnap(pIter->pTsdb, pIter->pReadSnap, NULL); _err: return code; diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 60d967681bc3851f0438f4fb9221f46db326acdc..14dbfbd63e9019ee06fd2b70e412fb4dad277f5b 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -903,7 +903,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn // null value exists, check one-by-one if (pData->flag != HAS_VALUE) { for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) { - uint8_t v = GET_BIT2(pData->pBitMap, j); + uint8_t v = tColDataGetBitValue(pData, j); if (v == 0 || v == 1) { colDataSetNull_f(pColData->nullbitmap, rowIndex); } @@ -1362,15 +1362,10 @@ static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pInfo->hasDupTs = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); - // todo here we need to each key in the last files to identify if it is really overlapped with last block - // todo - bool overlapWithlastBlock = false; -#if 0 - if (taosArrayGetSize(pLastBlockReader->pSstBlk) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { - SSttBlk* pSstBlk = taosArrayGet(pLastBlockReader->pSstBlk, pLastBlockReader->currentBlockIndex); - overlapWithlastBlock = !(pBlock->maxKey.ts < pSstBlk->minKey || pBlock->minKey.ts > pSstBlk->maxKey); + if (hasDataInLastBlock(pLastBlockReader)) { + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + pInfo->overlapWithLastBlock = !(pBlock->maxKey.ts < tsLast || pBlock->minKey.ts > tsLast); } -#endif pInfo->moreThanCapcity = pBlock->nRow > pReader->capacity; pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); @@ -1896,151 +1891,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } -#if 0 -static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { - SRowMerger merge = {0}; - STSRow* pTSRow = NULL; - - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SArray* pDelList = pBlockScanInfo->delSkyline; - - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); - ASSERT(pRow != NULL && piRow != NULL); - - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - bool freeTSRow = false; - - uint64_t uid = pBlockScanInfo->uid; - - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); - if (ASCENDING_TRAVERSE(pReader->order)) { - // [1&2] key <= [k.ts && ik.ts] - if (key <= k.ts && key <= ik.ts) { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - - if (ik.ts == key) { - tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - } - - if (k.ts == key) { - tRowMerge(&merge, pRow); - doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - } - - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - return TSDB_CODE_SUCCESS; - } else { // key > ik.ts || key > k.ts - ASSERT(key != ik.ts); - - // [3] ik.ts < key <= k.ts - // [4] ik.ts < k.ts <= key - if (ik.ts < k.ts) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - if (freeTSRow) { - taosMemoryFree(pTSRow); - } - return TSDB_CODE_SUCCESS; - } - - // [5] k.ts < key <= ik.ts - // [6] k.ts < ik.ts <= key - if (k.ts < ik.ts) { - doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - if (freeTSRow) { - taosMemoryFree(pTSRow); - } - return TSDB_CODE_SUCCESS; - } - - // [7] k.ts == ik.ts < key - if (k.ts == ik.ts) { - ASSERT(key > ik.ts && key > k.ts); - - doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); - return TSDB_CODE_SUCCESS; - } - } - } else { // descending order scan - // [1/2] k.ts >= ik.ts && k.ts >= key - if (k.ts >= ik.ts && k.ts >= key) { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - - if (ik.ts == k.ts) { - tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - } - - if (k.ts == key) { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMerge(&merge, &fRow); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } - - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - return TSDB_CODE_SUCCESS; - } else { - ASSERT(ik.ts != k.ts); // this case has been included in the previous if branch - - // [3] ik.ts > k.ts >= Key - // [4] ik.ts > key >= k.ts - if (ik.ts > key) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - if (freeTSRow) { - taosMemoryFree(pTSRow); - } - return TSDB_CODE_SUCCESS; - } - - // [5] key > ik.ts > k.ts - // [6] key > k.ts > ik.ts - if (key > ik.ts) { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); - return TSDB_CODE_SUCCESS; - } - - //[7] key = ik.ts > k.ts - if (key == ik.ts) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMerge(&merge, &fRow); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - - taosMemoryFree(pTSRow); - return TSDB_CODE_SUCCESS; - } - } - } - - ASSERT(0); - return -1; -} -#endif - static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { if (pBlockScanInfo->iterInit) { return TSDB_CODE_SUCCESS; @@ -2246,8 +2096,10 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // it is a clean block, load it directly if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { - copyBlockDataToSDataBlock(pReader, pBlockScanInfo); - goto _end; + if (pReader->order == TSDB_ORDER_ASC || (pReader->order == TSDB_ORDER_DESC && (!hasDataInLastBlock(pLastBlockReader)))) { + copyBlockDataToSDataBlock(pReader, pBlockScanInfo); + goto _end; + } } } else { // file blocks not exist pBlockScanInfo = pReader->status.pTableIter; @@ -2257,9 +2109,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - while (1) { - // todo check the validate of row in file block bool hasBlockData = false; { while (pBlockData->nRow > 0) { // find the first qualified row in data block @@ -2620,7 +2470,7 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { code = buildComposedDataBlock(pReader); } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { // data in memory that are earlier than current file block - // todo rows in buffer should be less than the file block in asc, greater than file block in desc + // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { @@ -3512,7 +3362,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl goto _err; } - code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap); + code = tsdbTakeReadSnap(pReader->pTsdb, &pReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3536,7 +3386,7 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl STsdbReader* pPrevReader = pReader->innerReader[0]; SDataBlockIter* pBlockIter = &pPrevReader->status.blockIter; - code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap); + code = tsdbTakeReadSnap(pPrevReader->pTsdb, &pPrevReader->pReadSnap, pReader->idStr); if (code != TSDB_CODE_SUCCESS) { goto _err; } @@ -3593,7 +3443,7 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); + tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap, pReader->idStr); taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); SIOCostSummary* pCost = &pReader->cost; @@ -3869,8 +3719,8 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { } } - tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, query range:%" PRId64 " - %" PRId64 " in query %s", - pReader, pReader->suid, numOfTables, pReader->window.skey, pReader->window.ekey, pReader->idStr); + tsdbDebug("%p reset reader, suid:%" PRIu64 ", numOfTables:%d, skey:%"PRId64", query range:%" PRId64 " - %" PRId64 " in query %s", + pReader, pReader->suid, numOfTables, pCond->twindows.skey, pReader->window.skey, pReader->window.ekey, pReader->idStr); return code; } @@ -4015,7 +3865,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 return TSDB_CODE_SUCCESS; } -int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap) { +int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap, const char* idStr) { int32_t code = 0; // alloc @@ -4058,12 +3908,12 @@ int32_t tsdbTakeReadSnap(STsdb* pTsdb, STsdbReadSnap** ppSnap) { goto _exit; } - tsdbTrace("vgId:%d, take read snapshot", TD_VID(pTsdb->pVnode)); + tsdbTrace("vgId:%d, take read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); _exit: return code; } -void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) { +void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap, const char* idStr) { if (pSnap) { if (pSnap->pMem) { tsdbUnrefMemTable(pSnap->pMem); @@ -4076,6 +3926,5 @@ void tsdbUntakeReadSnap(STsdb* pTsdb, STsdbReadSnap* pSnap) { tsdbFSUnref(pTsdb, &pSnap->fs); taosMemoryFree(pSnap); } - - tsdbTrace("vgId:%d, untake read snapshot", TD_VID(pTsdb->pVnode)); -} \ No newline at end of file + tsdbTrace("vgId:%d, untake read snapshot, %s", TD_VID(pTsdb->pVnode), idStr); +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index eb4151079b9ccf4bae1f1e03b4e240e9e6f18214..2c68c571765f063c79898f4b7bc722a675cb1a08 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -16,19 +16,9 @@ #include "tsdb.h" static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { - STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; - - if ((keepCfg->keep0 == keepCfg->keep1) && (keepCfg->keep1 == keepCfg->keep2)) { - return false; - } - - if (tfsGetLevel(pTsdb->pVnode->pTfs) <= 1) { - return false; - } - for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); + int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; if (expLevel == pSet->diskId.level) continue; diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index caeca45e01d0ec2def492d65f4ce9c6bd112e322..64c150484b4f873bd15fb69c2b238bd840e1ea1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -909,248 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr return code; } -// SColData ======================================== -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { - pColData->cid = cid; - pColData->type = type; - pColData->smaOn = smaOn; - tColDataReset(pColData); -} - -void tColDataReset(SColData *pColData) { - pColData->nVal = 0; - pColData->flag = 0; - pColData->nData = 0; -} - -void tColDataClear(void *ph) { - SColData *pColData = (SColData *)ph; - - tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); - tFree(pColData->pData); -} - -int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; - int64_t size; - SValue value = {0}; - SValue *pValue = &value; - - ASSERT(pColVal->cid == pColData->cid); - ASSERT(pColVal->type == pColData->type); - - // realloc bitmap - size = BIT2_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, size); - if (code) goto _exit; - if ((pColData->nVal & 3) == 0) { - pColData->pBitMap[pColData->nVal >> 2] = 0; - } - - // put value - if (pColVal->isNone) { - pColData->flag |= HAS_NONE; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (pColVal->isNull) { - pColData->flag |= HAS_NULL; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - } else { - pColData->flag |= HAS_VALUE; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - pValue = &pColVal->value; - } - - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1)); - if (code) goto _exit; - pColData->aOffset[pColData->nVal] = pColData->nData; - - // value - if ((!pColVal->isNone) && (!pColVal->isNull)) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); - if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; - } - } else { - code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type)); - if (code) goto _exit; - pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type); - } - - pColData->nVal++; - -_exit: - return code; -} - -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { - int32_t code = 0; - int32_t size; - - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid = pColDataSrc->cid); - ASSERT(pColDataDest->type = pColDataSrc->type); - - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; - - // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); - } - - // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; - - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; - - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); - } - - // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); - -_exit: - return code; -} - -int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { - int32_t code = 0; - - ASSERT(iVal < pColData->nVal); - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) { - *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); - goto _exit; - } else if (pColData->flag == HAS_NULL) { - *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); - goto _exit; - } else if (pColData->flag != HAS_VALUE) { - uint8_t v = GET_BIT2(pColData->pBitMap, iVal); - if (v == 0) { - *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); - goto _exit; - } else if (v == 1) { - *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); - goto _exit; - } - } - - // get value - SValue value; - if (IS_VAR_DATA_TYPE(pColData->type)) { - if (iVal + 1 < pColData->nVal) { - value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal]; - } else { - value.nData = pColData->nData - pColData->aOffset[iVal]; - } - - value.pData = pColData->pData + pColData->aOffset[iVal]; - } else { - tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); - } - *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); - -_exit: - return code; -} - -int32_t tPutColData(uint8_t *p, SColData *pColData) { - int32_t n = 0; - - n += tPutI16v(p ? p + n : p, pColData->cid); - n += tPutI8(p ? p + n : p, pColData->type); - n += tPutI8(p ? p + n : p, pColData->smaOn); - n += tPutI32v(p ? p + n : p, pColData->nVal); - n += tPutU8(p ? p + n : p, pColData->flag); - - if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; - if (pColData->flag != HAS_VALUE) { - // bitmap - - int32_t size = BIT2_SIZE(pColData->nVal); - if (p) { - memcpy(p + n, pColData->pBitMap, size); - } - n += size; - } - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - - int32_t size = sizeof(int32_t) * pColData->nVal; - if (p) { - memcpy(p + n, pColData->aOffset, size); - } - n += size; - } - n += tPutI32v(p ? p + n : p, pColData->nData); - if (p) { - memcpy(p + n, pColData->pData, pColData->nData); - } - n += pColData->nData; - -_exit: - return n; -} - -int32_t tGetColData(uint8_t *p, SColData *pColData) { - int32_t n = 0; - - n += tGetI16v(p + n, &pColData->cid); - n += tGetI8(p + n, &pColData->type); - n += tGetI8(p + n, &pColData->smaOn); - n += tGetI32v(p + n, &pColData->nVal); - n += tGetU8(p + n, &pColData->flag); - - if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; - if (pColData->flag != HAS_VALUE) { - // bitmap - - int32_t size = BIT2_SIZE(pColData->nVal); - pColData->pBitMap = p + n; - n += size; - } - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - - int32_t size = sizeof(int32_t) * pColData->nVal; - pColData->aOffset = (int32_t *)(p + n); - n += size; - } - n += tGetI32v(p + n, &pColData->nData); - pColData->pData = p + n; - n += pColData->nData; - -_exit: - return n; -} - -static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { - SColData *pColData1 = (SColData *)p1; - SColData *pColData2 = (SColData *)p2; - - if (pColData1->cid < pColData2->cid) { - return -1; - } else if (pColData1->cid > pColData2->cid) { - return 1; - } - - return 0; -} - // SBlockData ====================================================== int32_t tBlockDataCreate(SBlockData *pBlockData) { int32_t code = 0; @@ -1182,7 +940,7 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); - taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL); + taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); pBlockData->aUid = NULL; pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; @@ -1251,7 +1009,7 @@ void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tColDataReset(pColData); + tColDataClear(pColData); } } @@ -1501,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD while (lidx <= ridx) { int32_t midx = (lidx + ridx) / 2; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); - int32_t c = tColDataCmprFn(pColData, &(SColData){.cid = cid}); + int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); if (c == 0) { *ppColData = pColData; @@ -1986,47 +1744,16 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol int32_t size = 0; // bitmap if (pColData->flag != HAS_VALUE) { - uint8_t *pBitMap = pColData->pBitMap; - int32_t szBitMap = BIT2_SIZE(pColData->nVal); - - // BIT2 to BIT1 - if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + int32_t szBitMap; + if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + szBitMap = BIT2_SIZE(pColData->nVal); + } else { szBitMap = BIT1_SIZE(pColData->nVal); - pBitMap = taosMemoryCalloc(1, szBitMap); - if (pBitMap == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - uint8_t v = GET_BIT2(pColData->pBitMap, iVal); - switch (pColData->flag) { - case (HAS_NULL | HAS_NONE): - SET_BIT1(pBitMap, iVal, v); - break; - case (HAS_VALUE | HAS_NONE): - if (v) { - SET_BIT1(pBitMap, iVal, 1); - } else { - SET_BIT1(pBitMap, iVal, 0); - } - break; - case (HAS_VALUE | HAS_NULL): - SET_BIT1(pBitMap, iVal, v - 1); - break; - default: - ASSERT(0); - } - } } - code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap, - ppBuf); + code = tsdbCmprData(pColData->pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, + &pBlockCol->szBitmap, ppBuf); if (code) goto _exit; - - if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { - taosMemoryFree(pBitMap); - } } size += pBlockCol->szBitmap; @@ -2064,46 +1791,15 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t *p = pIn; // bitmap if (pBlockCol->szBitmap) { - if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { - uint8_t *pBitMap = NULL; - code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap, - BIT1_SIZE(pColData->nVal), ppBuf); - if (code) goto _exit; - - code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal)); - if (code) { - tFree(pBitMap); - goto _exit; - } - - // BIT1 to BIT2 - for (int32_t iVal = 0; iVal < nVal; iVal++) { - uint8_t v = GET_BIT1(pBitMap, iVal); - switch (pBlockCol->flag) { - case (HAS_NULL | HAS_NONE): - SET_BIT2(pColData->pBitMap, iVal, v); - break; - case (HAS_VALUE | HAS_NONE): - if (v) { - SET_BIT2(pColData->pBitMap, iVal, 2); - } else { - SET_BIT2(pColData->pBitMap, iVal, 0); - } - break; - case (HAS_VALUE | HAS_NULL): - SET_BIT2(pColData->pBitMap, iVal, v + 1); - break; - default: - ASSERT(0); - } - } - - tFree(pBitMap); + int32_t szBitMap; + if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + szBitMap = BIT2_SIZE(pColData->nVal); } else { - code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, - BIT2_SIZE(pColData->nVal), ppBuf); - if (code) goto _exit; + szBitMap = BIT1_SIZE(pColData->nVal); } + + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, szBitMap, ppBuf); + if (code) goto _exit; } p += pBlockCol->szBitmap; diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 51d83d8eedf6845e67a5ee81ff96debe52275bd1..6e9eba306a68ad91c22e5adb16e8ce5048b23903 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -533,6 +533,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR taosArrayPush(rsp.pArray, &cRsp); } + vDebug("vgId:%d, add %d new created tables into query table list", TD_VID(pVnode), (int32_t)taosArrayGetSize(tbUids)); tqUpdateTbUidList(pVnode->pTq, tbUids, true); if (tdUpdateTbUidList(pVnode->pSma, pStore, true) < 0) { goto _exit; @@ -885,8 +886,9 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq if (NULL != submitBlkRsp.pMeta) { vnodeUpdateMetaRsp(pVnode, submitBlkRsp.pMeta); } + + taosArrayPush(newTbUids, &createTbReq.uid); } - taosArrayPush(newTbUids, &createTbReq.uid); submitBlkRsp.uid = createTbReq.uid; submitBlkRsp.tblFName = taosMemoryMalloc(strlen(pVnode->config.dbname) + strlen(createTbReq.name) + 2); @@ -917,6 +919,11 @@ static int32_t vnodeProcessSubmitReq(SVnode *pVnode, int64_t version, void *pReq submitRsp.affectedRows += submitBlkRsp.affectedRows; taosArrayPush(submitRsp.pArray, &submitBlkRsp); } + + if (taosArrayGetSize(newTbUids) > 0) { + vDebug("vgId:%d, add %d table into query table list in handling submit", TD_VID(pVnode), (int32_t)taosArrayGetSize(newTbUids)); + } + tqUpdateTbUidList(pVnode->pTq, newTbUids, true); _exit: diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 40198615eab8a00a09479693193306026b4ba938..55a1a1fdb914ff80acdaf5fb9d23dcb2bd5921f2 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -79,25 +79,33 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pEntry->dataLen = sizeof(SDeleterRes); ASSERT(1 == pEntry->numOfRows); - ASSERT(1 == pEntry->numOfCols); + ASSERT(3 == pEntry->numOfCols); pBuf->useSize = sizeof(SDataCacheEntry); SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); + SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1); + SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2); SDeleterRes* pRes = (SDeleterRes*)pEntry->data; pRes->suid = pHandle->pParam->suid; pRes->uidList = pHandle->pParam->pUidList; - pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; - pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; strcpy(pRes->tableName, pHandle->pDeleter->tableFName); strcpy(pRes->tsColName, pHandle->pDeleter->tsColName); pRes->affectedRows = *(int64_t*)pColRes->pData; + if (pRes->affectedRows) { + pRes->skey = *(int64_t*)pColSKey->pData; + pRes->ekey = *(int64_t*)pColEKey->pData; + ASSERT(pRes->skey <= pRes->ekey); + } else { + pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; + pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; + } pBuf->useSize += pEntry->dataLen; - - atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); - atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + + atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); } static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) { @@ -172,7 +180,8 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData; *pLen = pEntry->dataLen; *pQueryEnd = pDeleter->queryEnd; - qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); + qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, + ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); } static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { @@ -186,14 +195,14 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { return TSDB_CODE_SUCCESS; } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData); - memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); + memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pDeleter->pParam->pUidList = NULL; pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDeleter); @@ -202,7 +211,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->useconds = pDeleter->useconds; pOutput->precision = pDeleter->pSchema->precision; taosThreadMutexUnlock(&pDeleter->mutex); - + return TSDB_CODE_SUCCESS; } @@ -211,7 +220,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize); taosMemoryFreeClear(pDeleter->nextOutput.pData); taosArrayDestroy(pDeleter->pParam->pUidList); - taosMemoryFree(pDeleter->pParam); + taosMemoryFree(pDeleter->pParam); while (!taosQueueEmpty(pDeleter->pDataBlocks)) { SDataDeleterBuf* pBuf = NULL; taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); @@ -230,14 +239,15 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { return TSDB_CODE_SUCCESS; } -int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam) { +int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, + void* pParam) { SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SDataDeleterNode* pDeleterNode = (SDataDeleterNode *)pDataSink; + SDataDeleterNode* pDeleterNode = (SDataDeleterNode*)pDataSink; deleter->sink.fPut = putDataBlock; deleter->sink.fEndPut = endPut; deleter->sink.fGetLen = getDataLength; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 95415e1113f0fe807157ac5a0e6509caef97def6..a2bcca9545a97c10554cb201655efea6c92c5d1f 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -265,6 +265,15 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + STableListInfo* pListInfo = &pTaskInfo->tableqinfoList; + + if (isAdd) { + qDebug("add %d tables id into query list, %s", (int32_t) taosArrayGetSize(tableIdList), pTaskInfo->id.str); + } + + if (pListInfo->map == NULL) { + pListInfo->map = taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + } // traverse to the stream scanner node to add this table id SOperatorInfo* pInfo = pTaskInfo->pRoot; @@ -311,13 +320,19 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } - taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); - if (pTaskInfo->tableqinfoList.map == NULL) { - pTaskInfo->tableqinfoList.map = - taosHashInit(32, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + bool exists = false; + for (int32_t k = 0; k < taosArrayGetSize(pListInfo->pTableList); ++k) { + STableKeyInfo* pKeyInfo = taosArrayGet(pListInfo->pTableList, k); + if (pKeyInfo->uid == keyInfo.uid) { + qWarn("ignore duplicated query table uid:%" PRIu64 " added, %s", pKeyInfo->uid, pTaskInfo->id.str); + exists = true; + } } - taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); + if (!exists) { + taosArrayPush(pTaskInfo->tableqinfoList.pTableList, &keyInfo); + taosHashPut(pTaskInfo->tableqinfoList.map, uid, sizeof(uid), &keyInfo.groupId, sizeof(keyInfo.groupId)); + } } if (keyBuf != NULL) { @@ -480,6 +495,7 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { if (ret != TSDB_CODE_SUCCESS) { pTaskInfo->code = ret; cleanUpUdfs(); + qDebug("%s task abort due to error/cancel occurs, code:%s", GET_TASKID(pTaskInfo), tstrerror(pTaskInfo->code)); atomic_store_64(&pTaskInfo->owner, 0); @@ -512,8 +528,8 @@ int32_t qExecTaskOpt(qTaskInfo_t tinfo, SArray* pResList, uint64_t* useconds) { } cleanUpUdfs(); - uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; + uint64_t total = pTaskInfo->pRoot->resultInfo.totalRows; qDebug("%s task suspended, %d rows in %d blocks returned, total:%" PRId64 " rows, in sinkNode:%d, elapsed:%.2f ms", GET_TASKID(pTaskInfo), current, (int32_t)taosArrayGetSize(pResList), total, 0, el / 1000.0); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index ad9cd1ffe7909c9a67e5af2e98193995757a05c2..a6d718bafac17df1c2314cdd13efb482c4c2eb01 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -617,19 +617,28 @@ static SSDataBlock* doTableScan(SOperatorInfo* pOperator) { // if scan table by table if (pInfo->scanMode == TABLE_SCAN__TABLE_ORDER) { - if (pInfo->noTable) return NULL; + if (pInfo->noTable) { + return NULL; + } + + int32_t numOfTables = taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList); + while (1) { SSDataBlock* result = doTableScanGroup(pOperator); if (result) { return result; } + // if no data, switch to next table and continue scan pInfo->currentTable++; - if (pInfo->currentTable >= taosArrayGetSize(pTaskInfo->tableqinfoList.pTableList)) { + if (pInfo->currentTable >= numOfTables) { return NULL; } + STableKeyInfo* pTableInfo = taosArrayGet(pTaskInfo->tableqinfoList.pTableList, pInfo->currentTable); tsdbSetTableId(pInfo->dataReader, pTableInfo->uid); + qDebug("set uid:%" PRIu64 " into scanner, total tables:%d, index:%d %s", pTableInfo->uid, numOfTables, pInfo->currentTable, pTaskInfo->id.str); + tsdbReaderReset(pInfo->dataReader, &pInfo->cond); pInfo->scanTimes = 0; } diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index eb0b604d37459342a403ab5120c22e2bf5dc4b13..48e21e137f9f7167bebfdfba7f3f281dbd6232b2 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -399,6 +399,8 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi COPY_SCALAR_FIELD(modifyType); COPY_SCALAR_FIELD(msgType); CLONE_NODE_FIELD(pAffectedRows); + CLONE_NODE_FIELD(pStartTs); + CLONE_NODE_FIELD(pEndTs); COPY_SCALAR_FIELD(tableId); COPY_SCALAR_FIELD(stableId); COPY_SCALAR_FIELD(tableType); diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index a41462e3fd879a73baeb37ee829724fda560719c..afd05eff1d1c6a2c895581b4be0f76e75e9ca631 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -2431,6 +2431,8 @@ static const char* jkDeletePhysiPlanTsColName = "TsColName"; static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows"; +static const char* jkDeletePhysiPlanStartTs = "StartTs"; +static const char* jkDeletePhysiPlanEndTs = "EndTs"; static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; @@ -2457,6 +2459,12 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeletePhysiPlanStartTs, nodeToJson, pNode->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeletePhysiPlanEndTs, nodeToJson, pNode->pEndTs); + } return code; } @@ -2486,6 +2494,12 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeletePhysiPlanStartTs, &pNode->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeletePhysiPlanEndTs, &pNode->pEndTs); + } return code; } diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c index efe820fee211af67ce1b168c5a5cddc821b0eeba..202ee6717a046a3149a45130189d25308f310648 100644 --- a/source/libs/nodes/src/nodesMsgFuncs.c +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -2665,7 +2665,9 @@ enum { PHY_DELETER_CODE_TABLE_FNAME, PHY_DELETER_CODE_TS_COL_NAME, PHY_DELETER_CODE_DELETE_TIME_RANGE, - PHY_DELETER_CODE_AFFECTED_ROWS + PHY_DELETER_CODE_AFFECTED_ROWS, + PHY_DELETER_CODE_START_TS, + PHY_DELETER_CODE_END_TS }; static int32_t physiDeleteNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { @@ -2690,6 +2692,12 @@ static int32_t physiDeleteNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { if (TSDB_CODE_SUCCESS == code) { code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_AFFECTED_ROWS, nodeToMsg, pNode->pAffectedRows); } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_START_TS, nodeToMsg, pNode->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_END_TS, nodeToMsg, pNode->pEndTs); + } return code; } @@ -2722,6 +2730,12 @@ static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { case PHY_DELETER_CODE_AFFECTED_ROWS: code = msgToNodeFromTlv(pTlv, (void**)&pNode->pAffectedRows); break; + case PHY_DELETER_CODE_START_TS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartTs); + break; + case PHY_DELETER_CODE_END_TS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndTs); + break; default: break; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 61b2ad954f3c91ed8d3bc8f7b9fb76c3c49cda9c..805ddb9e422db5a42c6aed1594985d5a233c4ccd 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -727,6 +727,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pWhere); nodesDestroyNode(pStmt->pCountFunc); + nodesDestroyNode(pStmt->pFirstFunc); + nodesDestroyNode(pStmt->pLastFunc); nodesDestroyNode(pStmt->pTagCond); break; } @@ -791,6 +793,8 @@ void nodesDestroyNode(SNode* pNode) { destroyVgDataBlockArray(pLogicNode->pDataBlocks); // pVgDataBlocks is weak reference nodesDestroyNode(pLogicNode->pAffectedRows); + nodesDestroyNode(pLogicNode->pStartTs); + nodesDestroyNode(pLogicNode->pEndTs); taosMemoryFreeClear(pLogicNode->pVgroupList); nodesDestroyList(pLogicNode->pInsertCols); break; @@ -997,6 +1001,8 @@ void nodesDestroyNode(SNode* pNode) { SDataDeleterNode* pSink = (SDataDeleterNode*)pNode; destroyDataSinkNode((SDataSinkNode*)pSink); nodesDestroyNode(pSink->pAffectedRows); + nodesDestroyNode(pSink->pStartTs); + nodesDestroyNode(pSink->pEndTs); break; } case QUERY_NODE_PHYSICAL_SUBPLAN: { diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 511ae25810f37d8d31d04c41fcf4bfec53610256..6f11c653a4a853744922c9ad17464aad71d77142 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1787,10 +1787,10 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb return (SNode*)pStmt; } -SNode* createCountFuncForDelete(SAstCreateContext* pCxt) { +SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); - strcpy(pFunc->functionName, "count"); + strcpy(pFunc->functionName, pFuncName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); @@ -1804,8 +1804,10 @@ SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) { CHECK_OUT_OF_MEM(pStmt); pStmt->pFromTable = pTable; pStmt->pWhere = pWhere; - pStmt->pCountFunc = createCountFuncForDelete(pCxt); - if (NULL == pStmt->pCountFunc) { + pStmt->pCountFunc = createFuncForDelete(pCxt, "count"); + pStmt->pFirstFunc = createFuncForDelete(pCxt, "first"); + pStmt->pLastFunc = createFuncForDelete(pCxt, "last"); + if (NULL == pStmt->pCountFunc || NULL == pStmt->pFirstFunc || NULL == pStmt->pLastFunc) { nodesDestroyNode((SNode*)pStmt); CHECK_OUT_OF_MEM(NULL); } diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index ea235ce45db0f9a5f07a349c59ec29cdbb636280..ea9eefbb2999da482202e99de1075cd3588f2b26 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -3347,10 +3347,16 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { if (TSDB_CODE_SUCCESS == code) { code = translateDeleteWhere(pCxt, pDelete); } + pCxt->currClause = SQL_CLAUSE_SELECT; if (TSDB_CODE_SUCCESS == code) { - pCxt->currClause = SQL_CLAUSE_SELECT; code = translateExpr(pCxt, &pDelete->pCountFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, &pDelete->pFirstFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, &pDelete->pLastFunc); + } return code; } diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index bf72f5210577d6f43f8ae97d098091b3020aeb16..fea2be55f962af0ff1d37b94ea4ddbe244eb3363 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1372,9 +1372,21 @@ static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pD } int32_t code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pDelete->pCountFunc)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pAgg->pAggFuncs, nodesCloneNode(pDelete->pFirstFunc)); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pAgg->pAggFuncs, nodesCloneNode(pDelete->pLastFunc)); + } if (TSDB_CODE_SUCCESS == code) { code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pCountFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pFirstFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pLastFunc); + } // set the output if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); @@ -1405,7 +1417,9 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet strcpy(pModify->tsColName, pRealTable->pMeta->schema->name); pModify->deleteTimeRange = pDelete->timeRange; pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc); - if (NULL == pModify->pAffectedRows) { + pModify->pStartTs = nodesCloneNode(pDelete->pFirstFunc); + pModify->pEndTs = nodesCloneNode(pDelete->pLastFunc); + if (NULL == pModify->pAffectedRows || NULL == pModify->pStartTs || NULL == pModify->pEndTs) { nodesDestroyNode((SNode*)pModify); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 0cbb833a4d4506b5123b45a0184bbc6023b53c2a..b0177e61ed290e8ef687e5de8c54c824545d8103 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1323,9 +1323,9 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) { - SPartitionPhysiNode* pPart = - (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, - pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION); + SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pPartLogicNode, + pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION); if (NULL == pPart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1670,6 +1670,12 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, &pDeleter->pAffectedRows); + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pStartTs, &pDeleter->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pEndTs, &pDeleter->pEndTs); + } if (TSDB_CODE_SUCCESS == code) { pDeleter->sink.pInputDataBlockDesc = (SDataBlockDescNode*)nodesCloneNode((SNode*)pRoot->pOutputDataBlockDesc); if (NULL == pDeleter->sink.pInputDataBlockDesc) { diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 61c38f59db2d08e960100e1b615bdf8f0a104127..4dcc06b4706b6c85b626bb03877c578e3fe2f8fe 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -9,6 +9,7 @@ #include "tcommon.h" #include "tmsg.h" #include "tname.h" +#include "tdatablock.h" SQWorkerMgmt gQwMgmt = { .lock = 0, @@ -16,6 +17,11 @@ SQWorkerMgmt gQwMgmt = { .qwNum = 0, }; +static void freeBlock(void* param) { + SSDataBlock* pBlock = *(SSDataBlock**)param; + blockDataDestroy(pBlock); +} + int32_t qwProcessHbLinkBroken(SQWorker *mgmt, SQWMsg *qwMsg, SSchedulerHbReq *req) { int32_t code = 0; SSchedulerHbRsp rsp = {0}; @@ -88,6 +94,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { // if *taskHandle is NULL, it's killed right now if (taskHandle) { qwDbgSimulateSleep(); + code = qExecTaskOpt(taskHandle, pResList, &useconds); if (code) { if (code != TSDB_CODE_OPS_NOT_SUPPORT) { @@ -150,8 +157,7 @@ int32_t qwExecTask(QW_FPARAMS_DEF, SQWTaskCtx *ctx, bool *queryStop) { } _return: - - taosArrayDestroy(pResList); + taosArrayDestroyEx(pResList, freeBlock); QW_RET(code); } @@ -915,13 +921,13 @@ void qwProcessHbTimerEvent(void *param, void *tmrId) { void *pIter = taosHashIterate(mgmt->schHash, NULL); while (pIter) { - SQWSchStatus *sch = (SQWSchStatus *)pIter; - if (NULL == sch->hbConnInfo.handle) { + SQWSchStatus *sch1 = (SQWSchStatus *)pIter; + if (NULL == sch1->hbConnInfo.handle) { uint64_t *sId = taosHashGetKey(pIter, NULL); QW_TLOG("cancel send hb to sch %" PRIx64 " cause of no connection handle", *sId); - if (sch->hbBrokenTs > 0 && ((currentMs - sch->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) && - taosHashGetSize(sch->tasksHash) <= 0) { + if (sch1->hbBrokenTs > 0 && ((currentMs - sch1->hbBrokenTs) > QW_SCH_TIMEOUT_MSEC) && + taosHashGetSize(sch1->tasksHash) <= 0) { taosArrayPush(pExpiredSch, sId); } diff --git a/source/util/test/trefTest.c b/source/util/test/trefTest.c index 89561e61fe02e4665828bc8336ec79f6df46fc68..5a6790ef1da3b5a6b015c836d09150b06e0a003f 100644 --- a/source/util/test/trefTest.c +++ b/source/util/test/trefTest.c @@ -94,7 +94,7 @@ void *openRefSpace(void *param) { pSpace->rsetId = taosOpenRef(50, myfree); if (pSpace->rsetId < 0) { - printf("failed to open ref, reson:%s\n", tstrerror(pSpace->rsetId)); + printf("failed to open ref, reason:%s\n", tstrerror(pSpace->rsetId)); return NULL; }