diff --git a/CMakeLists.txt b/CMakeLists.txt index 0c7e31bfbb891cc23c2cd5e788772d6f33bc329e..90e841d5e04fd72338f38ca11f1dd5a522b61918 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,7 +34,7 @@ endif(${BUILD_TEST}) add_subdirectory(source) add_subdirectory(tools) -add_subdirectory(tests) +add_subdirectory(utils) add_subdirectory(examples/c) # docs diff --git a/cmake/cmake.platform b/cmake/cmake.platform index 3aa1ffc07e73acdf480a21b478d55e05153694f8..e4d440d76edbfcf0d1d6f932cfa598fe0a0f43d2 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -87,7 +87,7 @@ IF ("${CPUTYPE}" STREQUAL "") SET(TD_ARM_32 TRUE) ADD_DEFINITIONS("-D_TD_ARM_") ADD_DEFINITIONS("-D_TD_ARM_32") - ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(aarch64)|(arm64)") MESSAGE(STATUS "The current platform is aarch64") SET(PLATFORM_ARCH_STR "arm64") SET(TD_ARM_64 TRUE) diff --git a/cmake/cmake.version b/cmake/cmake.version index db29644b387306ce8f3ee473921dab4c7d05b10a..7c895b12ff3397a221be5d2cadd95f4e0e64897d 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.0.1") + SET(TD_VER_NUMBER "3.0.1.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index f182beed33c76200649f93d96b68c153ec452b9a..eb0faf6d5d8072fdfeffae9e1732337fb7440aaf 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG abed566 + GIT_TAG 71e7ccf SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index a5bda0d7261161e79827860b97286e9009c012ff..6fb2a69847316f97ba016ac2d71481c0f18d7b4c 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 2460442 + GIT_TAG e7270c9 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taosws_CMakeLists.txt.in b/cmake/taosws_CMakeLists.txt.in index 506559a245ee7a3e506e8481a12a3fe7f01dd5ac..04b1262cafd6f1dd984f568b847454c409d301ed 100644 --- a/cmake/taosws_CMakeLists.txt.in +++ b/cmake/taosws_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosws-rs ExternalProject_Add(taosws-rs GIT_REPOSITORY https://github.com/taosdata/taos-connector-rust.git - GIT_TAG 7a54d21 + GIT_TAG e771403 SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosws-rs" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/docs/en/04-concept/index.md b/docs/en/04-concept/index.md index b0a0c25d85e99244858a461317ff54359d1ceff8..0b1b226c17100d56313b5480e26f437841afe8c7 100644 --- a/docs/en/04-concept/index.md +++ b/docs/en/04-concept/index.md @@ -6,101 +6,100 @@ In order to explain the basic concepts and provide some sample code, the TDengin
- - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Device IDTime StampCollected MetricsTags
Device IDTimestampCollected MetricsTags
Device IDTime StampcurrentvoltagephaselocationgroupId
d1001153854868500010.32190.31California.SanFrancisco2
d1002153854868400010.22200.23California.SanFrancisco3
d1003153854868650011.52210.35California.LosAngeles3
d1004153854868550013.42230.29California.LosAngeles2
d1001153854869500012.62180.33California.SanFrancisco2
d1004153854869660011.82210.28California.LosAngeles2
d1002153854869665010.32180.25California.SanFrancisco3
d1001153854869680012.32210.31California.SanFrancisco2
currentvoltagephaselocationgroupid
d1001153854868500010.32190.31California.SanFrancisco2
d1002153854868400010.22200.23California.SanFrancisco3
d1003153854868650011.52210.35California.LosAngeles3
d1004153854868550013.42230.29California.LosAngeles2
d1001153854869500012.62180.33California.SanFrancisco2
d1004153854869660011.82210.28California.LosAngeles2
d1002153854869665010.32180.25California.SanFrancisco3
d1001153854869680012.32210.31California.SanFrancisco2
Table 1: Smart meter example data
-Each row contains the device ID, time stamp, collected metrics (current, voltage, phase as above), and static tags (location and groupId in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated time stamps. +Each row contains the device ID, timestamp, collected metrics (`current`, `voltage`, `phase` as above), and static tags (`location` and `groupid` in Table 1) associated with the devices. Each smart meter generates a row (measurement) in a pre-defined time interval or triggered by an external event. The device produces a sequence of measurements with associated timestamps. ## Metric @@ -112,22 +111,22 @@ Label/Tag refers to the static properties of sensors, equipment or other types o ## Data Collection Point -Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same time stamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points. +Data Collection Point (DCP) refers to hardware or software that collects metrics based on preset time periods or triggered by events. A data collection point can collect one or multiple metrics, but these metrics are collected at the same time and have the same timestamp. For some complex equipment, there are often multiple data collection points, and the sampling rate of each collection point may be different, and fully independent. For example, for a car, there could be a data collection point to collect GPS position metrics, a data collection point to collect engine status metrics, and a data collection point to collect the environment metrics inside the car. So in this example the car would have three data collection points. In the smart meters example, d1001, d1002, d1003, and d1004 are the data collection points. ## Table Since time-series data is most likely to be structured data, TDengine adopts the traditional relational database model to process them with a short learning curve. You need to create a database, create tables, then insert data points and execute queries to explore the data. -To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices D1001, D1002, D1003, and D1004 to store the data collected. This design has several benefits: +To make full use of time-series data characteristics, TDengine adopts a strategy of "**One Table for One Data Collection Point**". TDengine requires the user to create a table for each data collection point (DCP) to store collected time-series data. For example, if there are over 10 million smart meters, it means 10 million tables should be created. For the table above, 4 tables should be created for devices d1001, d1002, d1003, and d1004 to store the data collected. This design has several benefits: 1. Since the metric data from different DCP are fully independent, the data source of each DCP is unique, and a table has only one writer. In this way, data points can be written in a lock-free manner, and the writing speed can be greatly improved. 2. For a DCP, the metric data generated by DCP is ordered by timestamp, so the write operation can be implemented by simple appending, which further greatly improves the data writing speed. 3. The metric data from a DCP is continuously stored, block by block. If you read data for a period of time, it can greatly reduce random read operations and improve read and query performance by orders of magnitude. 4. Inside a data block for a DCP, columnar storage is used, and different compression algorithms are used for different data types. Metrics generally don't vary as significantly between themselves over a time range as compared to other metrics, which allows for a higher compression rate. -If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. ** One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.** +If the metric data of multiple DCPs are traditionally written into a single table, due to uncontrollable network delays, the timing of the data from different DCPs arriving at the server cannot be guaranteed, write operations must be protected by locks, and metric data from one DCP cannot be guaranteed to be continuously stored together. **One table for one data collection point can ensure the best performance of insert and query of a single data collection point to the greatest possible extent.** -TDengine suggests using DCP ID as the table name (like D1001 in the above table). Each DCP may collect one or multiple metrics (like the current, voltage, phase as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the time stamp as the index, and won’t build the index on any metrics stored. Column wise storage is used. +TDengine suggests using DCP ID as the table name (like d1001 in the above table). Each DCP may collect one or multiple metrics (like the `current`, `voltage`, `phase` as above). Each metric has a corresponding column in the table. The data type for a column can be int, float, string and others. In addition, the first column in the table must be a timestamp. TDengine uses the timestamp as the index, and won’t build the index on any metrics stored. Column wise storage is used. Complex devices, such as connected cars, may have multiple DCPs. In this case, multiple tables are created for a single device, one table per DCP. @@ -156,9 +155,16 @@ The relationship between a STable and the subtables created based on this STable Queries can be executed on both a table (subtable) and a STable. For a query on a STable, TDengine will treat the data in all its subtables as a whole data set for processing. TDengine will first find the subtables that meet the tag filter conditions, then scan the time-series data of these subtables to perform aggregation operation, which reduces the number of data sets to be scanned which in turn greatly improves the performance of data aggregation across multiple DCPs. In essence, querying a supertable is a very efficient aggregate query on multiple DCPs of the same type. -In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table meters. +In TDengine, it is recommended to use a subtable instead of a regular table for a DCP. In the smart meters example, we can create subtables like d1001, d1002, d1003, and d1004 under super table `meters`. + +To better understand the data model using metrics, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example. + +
+ +![Meters Data Model Diagram](./supertable.webp) -To better understand the data model using metri, tags, super table and subtable, please refer to the diagram below which demonstrates the data model of the smart meters example. ![Meters Data Model Diagram](./supertable.webp) +
Figure 1. Meters Data Model Diagram
+
## Database @@ -172,4 +178,4 @@ FQDN (Fully Qualified Domain Name) is the full domain name of a specific compute Each node of a TDengine cluster is uniquely identified by an End Point, which consists of an FQDN and a Port, such as h1.tdengine.com:6030. In this way, when the IP changes, we can still use the FQDN to dynamically find the node without changing any configuration of the cluster. In addition, FQDN is used to facilitate unified access to the same cluster from the Intranet and the Internet. -TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management. +TDengine does not recommend using an IP address to access the cluster. FQDN is recommended for cluster management. diff --git a/docs/en/05-get-started/01-docker.md b/docs/en/05-get-started/01-docker.md index de5b620a779557a8a3b8422a14caf67b354d1e7a..6191492b37e09096ad1d7c8da4567130579ca20b 100644 --- a/docs/en/05-get-started/01-docker.md +++ b/docs/en/05-get-started/01-docker.md @@ -13,7 +13,7 @@ If Docker is already installed on your computer, run the following command: docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine ``` -Note that TDengine Server uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed. +Note that TDengine Server 3.0 uses TCP port 6030. Port 6041 is used by taosAdapter for the REST API service. Ports 6043 through 6049 are used by taosAdapter for other connectors. You can open these ports as needed. Run the following command to ensure that your container is running: @@ -21,7 +21,7 @@ Run the following command to ensure that your container is running: docker ps ``` -Enter the container and open the bash shell: +Enter the container and open the `bash` shell: ```shell docker exec -it bash @@ -31,68 +31,68 @@ You can now access TDengine or run other Linux commands. Note: For information about installing docker, see the [official documentation](https://docs.docker.com/get-docker/). -## Insert Data into TDengine - -You can use the `taosBenchmark` tool included with TDengine to write test data into your deployment. +## Open the TDengine CLI -To do so, run the following command: +On the container, run the following command to open the TDengine CLI: - ```bash - $ taosBenchmark - - ``` +``` +$ taos -This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`. +taos> - The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required depends on the hardware specifications of the local system. +``` - You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](/reference/taosbenchmark). +## Test data insert performance -## Open the TDengine CLI +After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance: -On the container, run the following command to open the TDengine CLI: +Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal. +```bash +taosBenchmark ``` -$ taos -taos> +This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `California.Campbell`, `California.Cupertino`, `California.LosAngeles`, `California.MountainView`, `California.PaloAlto`, `California.SanDiego`, `California.SanFrancisco`, `California.SanJose`, `California.SantaClara` or `California.Sunnyvale`. -``` +The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds. -## Query Data in TDengine +You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark). -After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance. For example: +## Test data query performance -From the TDengine CLI query the number of rows in the `meters` supertable: +After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance: + +From the TDengine CLI (taos) query the number of rows in the `meters` supertable: ```sql -select count(*) from test.meters; +SELECT COUNT(*) FROM test.meters; ``` Query the average, maximum, and minimum values of all 100 million rows of data: ```sql -select avg(current), max(voltage), min(phase) from test.meters; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters; ``` -Query the number of rows whose `location` tag is `San Francisco`: +Query the number of rows whose `location` tag is `California.SanFrancisco`: ```sql -select count(*) from test.meters where location="San Francisco"; +SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco"; ``` Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`: ```sql -select avg(current), max(voltage), min(phase) from test.meters where groupId=10; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10; ``` -Query the average, maximum, and minimum values for table `d10` in 1 second intervals: +Query the average, maximum, and minimum values for table `d10` in 10 second intervals: ```sql -select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s); +SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s); ``` -In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/). + +In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/). ## Additional Information diff --git a/docs/en/05-get-started/03-package.md b/docs/en/05-get-started/03-package.md index 88096a759c58529d4150c0a750a4354a88da988f..b0400de67388a433355358d827aa41f35598655c 100644 --- a/docs/en/05-get-started/03-package.md +++ b/docs/en/05-get-started/03-package.md @@ -9,23 +9,24 @@ import PkgListV3 from "/components/PkgListV3"; For information about installing TDengine on Docker, see [Quick Install on Docker](../../get-started/docker). If you want to view the source code, build TDengine yourself, or contribute to the project, see the [TDengine GitHub repository](https://github.com/taosdata/TDengine). -The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface, and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter). +The full package of TDengine includes the TDengine Server (`taosd`), TDengine Client (`taosc`), taosAdapter for connecting with third-party systems and providing a RESTful interface, a command-line interface (CLI, taos), and some tools. Note that taosAdapter supports Linux only. In addition to connectors for multiple languages, TDengine also provides a [REST API](../../reference/rest-api) through [taosAdapter](../../reference/taosadapter). -The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download a lite package that includes only `taosd` and the C/C++ connector. +The standard server installation package includes `taos`, `taosd`, `taosAdapter`, `taosBenchmark`, and sample code. You can also download the Lite package that includes only `taosd` and the C/C++ connector. -The TDengine Community Edition is released as .deb and .rpm packages. The .deb package can be installed on Debian, Ubuntu, and derivative systems. The .rpm package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the .deb or .rpm package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows servers. +The TDengine Community Edition is released as Deb and RPM packages. The Deb package can be installed on Debian, Ubuntu, and derivative systems. The RPM package can be installed on CentOS, RHEL, SUSE, and derivative systems. A .tar.gz package is also provided for enterprise customers, and you can install TDengine over `apt-get` as well. The .tar.tz package includes `taosdump` and the TDinsight installation script. If you want to use these utilities with the Deb or RPM package, download and install taosTools separately. TDengine can also be installed on 64-bit Windows. ## Installation -1. Download the .deb installation package. - +1. Download the Deb installation package. + 2. In the directory where the package is located, use `dpkg` to install the package: +> Please replace `` with the corresponding version of the package downloaded + ```bash -# Enter the name of the package that you downloaded. sudo dpkg -i TDengine-server--Linux-x64.deb ``` @@ -34,11 +35,12 @@ sudo dpkg -i TDengine-server--Linux-x64.deb 1. Download the .rpm installation package. - + 2. In the directory where the package is located, use rpm to install the package: +> Please replace `` with the corresponding version of the package downloaded + ```bash -# Enter the name of the package that you downloaded. sudo rpm -ivh TDengine-server--Linux-x64.rpm ``` @@ -47,11 +49,12 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm 1. Download the .tar.gz installation package. - + 2. In the directory where the package is located, use `tar` to decompress the package: +> Please replace `` with the corresponding version of the package downloaded + ```bash -# Enter the name of the package that you downloaded. tar -zxvf TDengine-server--Linux-x64.tar.gz ``` @@ -96,23 +99,23 @@ sudo apt-get install tdengine This installation method is supported only for Debian and Ubuntu. :::: - + -Note: TDengine only supports Windows Server 2016/2019 and windows 10/11 system versions on the windows platform. +Note: TDengine only supports Windows Server 2016/2019 and Windows 10/11 on the Windows platform. 1. Download the Windows installation package. - + 2. Run the downloaded package to install TDengine. :::info -For information about TDengine releases, see [Release History](../../releases). +For information about TDengine releases, see [Release History](../../releases). ::: :::note -On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the end point of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine. +On the first node in your TDengine cluster, leave the `Enter FQDN:` prompt blank and press **Enter**. On subsequent nodes, you can enter the endpoint of the first dnode in the cluster. You can also configure this setting after you have finished installing TDengine. ::: @@ -147,7 +150,7 @@ Active: inactive (dead) After confirming that TDengine is running, run the `taos` command to access the TDengine CLI. -The following `systemctl` commands can help you manage TDengine: +The following `systemctl` commands can help you manage TDengine service: - Start TDengine Server: `systemctl start taosd` @@ -159,7 +162,7 @@ The following `systemctl` commands can help you manage TDengine: :::info -- The `systemctl` command requires _root_ privileges. If you are not logged in as the `root` user, use the `sudo` command. +- The `systemctl` command requires _root_ privileges. If you are not logged in as the _root_ user, use the `sudo` command. - The `systemctl stop taosd` command does not instantly stop TDengine Server. The server is stopped only after all data in memory is flushed to disk. The time required depends on the cache size. - If your system does not include `systemd`, you can run `/usr/local/taos/bin/taosd` to start TDengine manually. @@ -174,23 +177,9 @@ After the installation is complete, run `C:\TDengine\taosd.exe` to start TDengin -## Test data insert performance +## Command Line Interface (CLI) -After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance: - -```bash -taosBenchmark -``` - -This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `Campbell`, `Cupertino`, `Los Angeles`, `Mountain View`, `Palo Alto`, `San Diego`, `San Francisco`, `San Jose`, `Santa Clara` or `Sunnyvale`. - -The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in less than a minute. - -You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark). - -## Command Line Interface - -You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, run the following command: +You can use the TDengine CLI to monitor your TDengine deployment and execute ad hoc queries. To open the CLI, you can execute `taos` in the Linux terminal where TDengine is installed, or you can run `taos.exe` in the `C:\TDengine` directory of the Windows terminal where TDengine is installed to start the TDengine command line. ```bash taos @@ -205,52 +194,71 @@ taos> For example, you can create and delete databases and tables and run all types of queries. Each SQL command must be end with a semicolon (;). For example: ```sql -create database demo; -use demo; -create table t (ts timestamp, speed int); -insert into t values ('2019-07-15 00:00:00', 10); -insert into t values ('2019-07-15 01:00:00', 20); -select * from t; +CREATE DATABASE demo; +USE demo; +CREATE TABLE t (ts TIMESTAMP, speed INT); +INSERT INTO t VALUES ('2019-07-15 00:00:00', 10); +INSERT INTO t VALUES ('2019-07-15 01:00:00', 20); +SELECT * FROM t; + ts | speed | ======================================== 2019-07-15 00:00:00.000 | 10 | 2019-07-15 01:00:00.000 | 20 | + Query OK, 2 row(s) in set (0.003128s) ``` You can also can monitor the deployment status, add and remove user accounts, and manage running instances. You can run the TDengine CLI on either Linux or Windows machines. For more information, see [TDengine CLI](../../reference/taos-shell/). - + +## Test data insert performance + +After your TDengine Server is running normally, you can run the taosBenchmark utility to test its performance: + +Start TDengine service and execute `taosBenchmark` (formerly named `taosdemo`) in a Linux or Windows terminal. + +```bash +taosBenchmark +``` + +This command creates the `meters` supertable in the `test` database. In the `meters` supertable, it then creates 10,000 subtables named `d0` to `d9999`. Each table has 10,000 rows and each row has four columns: `ts`, `current`, `voltage`, and `phase`. The timestamps of the data in these columns range from 2017-07-14 10:40:00 000 to 2017-07-14 10:40:09 999. Each table is randomly assigned a `groupId` tag from 1 to 10 and a `location` tag of either `California.Campbell`, `California.Cupertino`, `California.LosAngeles`, `California.MountainView`, `California.PaloAlto`, `California.SanDiego`, `California.SanFrancisco`, `California.SanJose`, `California.SantaClara` or `California.Sunnyvale`. + +The `taosBenchmark` command creates a deployment with 100 million data points that you can use for testing purposes. The time required to create the deployment depends on your hardware. On most modern servers, the deployment is created in ten to twenty seconds. + +You can customize the test deployment that taosBenchmark creates by specifying command-line parameters. For information about command-line parameters, run the `taosBenchmark --help` command. For more information about taosBenchmark, see [taosBenchmark](../../reference/taosbenchmark). + ## Test data query performance -After using taosBenchmark to create your test deployment, you can run queries in the TDengine CLI to test its performance: +After using `taosBenchmark` to create your test deployment, you can run queries in the TDengine CLI to test its performance: -From the TDengine CLI query the number of rows in the `meters` supertable: +From the TDengine CLI (taos) query the number of rows in the `meters` supertable: ```sql -select count(*) from test.meters; +SELECT COUNT(*) FROM test.meters; ``` Query the average, maximum, and minimum values of all 100 million rows of data: ```sql -select avg(current), max(voltage), min(phase) from test.meters; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters; ``` -Query the number of rows whose `location` tag is `San Francisco`: +Query the number of rows whose `location` tag is `California.SanFrancisco`: ```sql -select count(*) from test.meters where location="San Francisco"; +SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco"; ``` Query the average, maximum, and minimum values of all rows whose `groupId` tag is `10`: ```sql -select avg(current), max(voltage), min(phase) from test.meters where groupId=10; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10; ``` -Query the average, maximum, and minimum values for table `d10` in 1 second intervals: +Query the average, maximum, and minimum values for table `d10` in 10 second intervals: ```sql -select first(ts), avg(current), max(voltage), min(phase) from test.d10 interval(1s); +SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s); ``` -In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be _wstart which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/). + +In the query above you are selecting the first timestamp (ts) in the interval, another way of selecting this would be `\_wstart` which will give the start of the time window. For more information about windowed queries, see [Time-Series Extensions](../../taos-sql/distinguished/). diff --git a/docs/en/12-taos-sql/06-select.md b/docs/en/12-taos-sql/06-select.md index 1dd0caed38235d3d10813b2cd74fec6446c5ec24..1f861119250e9e32fcac3344a5bdfb760e6191a7 100644 --- a/docs/en/12-taos-sql/06-select.md +++ b/docs/en/12-taos-sql/06-select.md @@ -52,11 +52,6 @@ window_clause: { | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] -changes_option: { - DURATION duration_val - | ROWS rows_val -} - group_by_clause: GROUP BY expr [, expr] ... HAVING condition @@ -126,7 +121,6 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; 1. Configuration parameter `maxNumOfDistinctRes` in `taos.cfg` is used to control the number of rows to output. The minimum configurable value is 100,000, the maximum configurable value is 100,000,000, the default value is 1,000,000. If the actual number of rows exceeds the value of this parameter, only the number of rows specified by this parameter will be output. 2. It can't be guaranteed that the results selected by using `DISTINCT` on columns of `FLOAT` or `DOUBLE` are exactly unique because of the precision errors in floating point numbers. -3. `DISTINCT` can't be used in the sub-query of a nested query statement, and can't be used together with aggregate functions, `GROUP BY` or `JOIN` in the same SQL statement. ::: diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index d6905c84a11ecf1f827ad239076ad5d4c93f8e3e..ab1d2f900b993cee80109fec22f01face20a3dbb 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -126,7 +126,7 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -**Description**: The rounded down value of a specific field +**Description**: The rounded down value of a specific field **More explanations**: The restrictions are same as those of the `CEIL` function. #### LOG @@ -173,7 +173,7 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -**Description**: The rounded value of a specific field. +**Description**: The rounded value of a specific field. **More explanations**: The restrictions are same as those of the `CEIL` function. @@ -434,7 +434,7 @@ SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause]; **More explanations**: - You can specify a time zone in the following format: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。 For example, TO_ISO8601(1, "+00:00"). -- If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp +- If the input is a UNIX timestamp, the precision of the returned value is determined by the digits of the input timestamp - If the input is a column of TIMESTAMP type, the precision of the returned value is same as the precision set for the current data base in use @@ -613,6 +613,7 @@ SELECT APERCENTILE(field_name, P[, algo_type]) FROM { tb_name | stb_name } [WHER **Explanations**： - _P_ is in range [0,100], when _P_ is 0, the result is same as using function MIN; when _P_ is 100, the result is same as function MAX. - `algo_type` can only be input as `default` or `t-digest` Enter `default` to use a histogram-based algorithm. Enter `t-digest` to use the t-digest algorithm to calculate the approximation of the quantile. `default` is used by default. +- The approximation result of `t-digest` algorithm is sensitive to input data order. For example, when querying STable with different input data order there might be minor differences in calculated results. ### AVG @@ -768,14 +769,14 @@ SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_nam **Explanations**： - bin_type: parameter to indicate the bucket type, valid inputs are: "user_input", "linear_bin", "log_bin"。 -- bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively: - - "user_input": "[1, 3, 5, 7]": +- bin_description: parameter to describe how to generate buckets，can be in the following JSON formats for each bin_type respectively: + - "user_input": "[1, 3, 5, 7]": User specified bin values. - + - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" "start" - bin starting point. "width" - bin offset. "count" - number of bins generated. "infinity" - whether to add（-inf, inf）as start/end point in generated set of bins. The above "linear_bin" descriptor generates a set of bins: [-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]. - + - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" "start" - bin starting point. "factor" - exponential factor of bin offset. "count" - number of bins generated. "infinity" - whether to add（-inf, inf）as start/end point in generated range of bins. The above "linear_bin" descriptor generates a set of bins: [-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]. @@ -861,9 +862,9 @@ SELECT INTERP(field_name) FROM { tb_name | stb_name } [WHERE where_condition] RA - `INTERP` is used to get the value that matches the specified time slice from a column. If no such value exists an interpolation value will be returned based on `FILL` parameter. - The input data of `INTERP` is the value of the specified column and a `where` clause can be used to filter the original data. If no `where` condition is specified then all original data is the input. -- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. -- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. -- Interpolation is performed based on `FILL` parameter. +- The output time range of `INTERP` is specified by `RANGE(timestamp1,timestamp2)` parameter, with timestamp1<=timestamp2. timestamp1 is the starting point of the output time range and must be specified. timestamp2 is the ending point of the output time range and must be specified. +- The number of rows in the result set of `INTERP` is determined by the parameter `EVERY`. Starting from timestamp1, one interpolation is performed for every time interval specified `EVERY` parameter. +- Interpolation is performed based on `FILL` parameter. - `INTERP` can only be used to interpolate in single timeline. So it must be used with `partition by tbname` when it's used on a STable. ### LAST @@ -967,7 +968,7 @@ SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] **Applicable table types**: standard tables and supertables -**More explanations**: +**More explanations**: This function cannot be used in expression calculation. - Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline @@ -1045,10 +1046,10 @@ SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] **Applicable table types**: standard tables and supertables -**More explanations**: - +**More explanations**: + - Arithmetic operation can't be performed on the result of `csum` function -- Can only be used with aggregate functions This function can be used with supertables and standard tables. +- Can only be used with aggregate functions This function can be used with supertables and standard tables. - Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline @@ -1066,8 +1067,8 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER **Applicable table types**: standard tables and supertables -**More explanation**: - +**More explanation**: + - It can be used together with `PARTITION BY tbname` against a STable. - It can be used together with a selected column. For example: select \_rowts, DERIVATIVE() from。 @@ -1085,7 +1086,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER **Applicable table types**: standard tables and supertables -**More explanation**: +**More explanation**: - The number of result rows is the number of rows subtracted by one, no output for the first row - It can be used together with a selected column. For example: select \_rowts, DIFF() from。 @@ -1122,9 +1123,9 @@ SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] **Applicable table types**: standard tables and supertables -**More explanations**: - -- Arithmetic operation can't be performed on the result of `MAVG`. +**More explanations**: + +- Arithmetic operation can't be performed on the result of `MAVG`. - Can only be used with data columns, can't be used with tags. - Can't be used with aggregate functions. - Must be used with `PARTITION BY tbname` when it's used on a STable to force the result on each single timeline diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index deb2522309d123cae453b1ab2454db2062b01746..17e4e4d1b0da6d0461c9ab478a9430855379fb12 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -44,7 +44,7 @@ For example, the following SQL statement creates a stream and automatically crea ```sql CREATE STREAM avg_vol_s INTO avg_vol AS -SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); +SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); ``` ## Delete a Stream diff --git a/docs/en/12-taos-sql/20-keywords.md b/docs/en/12-taos-sql/20-keywords.md index 6f166c8034382b0613845d18470556622106e673..f7eb067c960cd9599153a208cd2781b49b090511 100644 --- a/docs/en/12-taos-sql/20-keywords.md +++ b/docs/en/12-taos-sql/20-keywords.md @@ -5,7 +5,9 @@ title: Reserved Keywords ## Keyword List -There are about 200 keywords reserved by TDengine, they can't be used as the name of database, STable or table with either upper case, lower case or mixed case. The following list shows all reserved keywords: +There are more than 200 keywords reserved by TDengine, they can't be used as the name of database, table, STable, subtable, column or tag with either upper case, lower case or mixed case. If you need to use these keywords, use the symbol `` ` `` to enclose the keywords, e.g. \`ADD\`. + +The following list shows all reserved keywords: ### A @@ -14,15 +16,20 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - ACCOUNTS - ADD - AFTER +- AGGREGATE - ALL - ALTER +- ANALYZE - AND +- APPS - AS - ASC +- AT_ONCE - ATTACH ### B +- BALANCE - BEFORE - BEGIN - BETWEEN @@ -32,19 +39,27 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - BITNOT - BITOR - BLOCKS +- BNODE +- BNODES - BOOL +- BUFFER +- BUFSIZE - BY ### C - CACHE -- CACHELAST +- CACHEMODEL +- CACHESIZE - CASCADE +- CAST - CHANGE +- CLIENT_VERSION - CLUSTER - COLON - COLUMN - COMMA +- COMMENT - COMP - COMPACT - CONCAT @@ -52,15 +67,18 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - CONNECTION - CONNECTIONS - CONNS +- CONSUMER +- CONSUMERS +- CONTAINS - COPY +- COUNT - CREATE -- CTIME +- CURRENT_USER ### D - DATABASE - DATABASES -- DAYS - DBS - DEFERRED - DELETE @@ -69,18 +87,23 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - DESCRIBE - DETACH - DISTINCT +- DISTRIBUTED - DIVIDE - DNODE - DNODES - DOT - DOUBLE - DROP +- DURATION ### E +- EACH +- ENABLE - END -- EQ +- EVERY - EXISTS +- EXPIRED - EXPLAIN ### F @@ -88,18 +111,20 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - FAIL - FILE - FILL +- FIRST - FLOAT +- FLUSH - FOR - FROM -- FSYNC +- FUNCTION +- FUNCTIONS ### G -- GE - GLOB +- GRANT - GRANTS - GROUP -- GT ### H @@ -110,15 +135,18 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - ID - IF - IGNORE -- IMMEDIA +- IMMEDIATE - IMPORT - IN -- INITIAL +- INDEX +- INDEXES +- INITIALLY +- INNER - INSERT - INSTEAD - INT - INTEGER -- INTERVA +- INTERVAL - INTO - IS - ISNULL @@ -126,6 +154,7 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam ### J - JOIN +- JSON ### K @@ -135,46 +164,57 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam ### L -- LE +- LAST +- LAST_ROW +- LICENCES - LIKE - LIMIT - LINEAR - LOCAL -- LP -- LSHIFT -- LT ### M - MATCH +- MAX_DELAY - MAXROWS +- MERGE +- META - MINROWS - MINUS +- MNODE - MNODES - MODIFY - MODULES ### N -- NE +- NCHAR +- NEXT +- NMATCH - NONE - NOT - NOTNULL - NOW - NULL +- NULLS ### O - OF - OFFSET +- ON - OR - ORDER +- OUTPUTTYPE ### P -- PARTITION +- PAGES +- PAGESIZE +- PARTITIONS - PASS - PLUS +- PORT - PPS - PRECISION - PREV @@ -182,47 +222,63 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam ### Q +- QNODE +- QNODES - QTIME -- QUERIE +- QUERIES - QUERY -- QUORUM ### R - RAISE -- REM +- RANGE +- RATIO +- READ +- REDISTRIBUTE +- RENAME - REPLACE - REPLICA - RESET -- RESTRIC +- RESTRICT +- RETENTIONS +- REVOKE +- ROLLUP - ROW -- RP -- RSHIFT ### S +- SCHEMALESS - SCORES - SELECT - SEMI +- SERVER_STATUS +- SERVER_VERSION - SESSION - SET - SHOW -- SLASH +- SINGLE_STABLE - SLIDING - SLIMIT -- SMALLIN +- SMA +- SMALLINT +- SNODE +- SNODES - SOFFSET -- STable -- STableS +- SPLIT +- STABLE +- STABLES - STAR - STATE -- STATEMEN -- STATE_WI +- STATE_WINDOW +- STATEMENT - STORAGE - STREAM - STREAMS +- STRICT - STRING +- SUBSCRIPTIONS - SYNCDB +- SYSINFO ### T @@ -233,19 +289,24 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam - TBNAME - TIMES - TIMESTAMP +- TIMEZONE - TINYINT +- TO +- TODAY - TOPIC - TOPICS +- TRANSACTION +- TRANSACTIONS - TRIGGER +- TRIM - TSERIES +- TTL ### U -- UMINUS - UNION - UNSIGNED - UPDATE -- UPLUS - USE - USER - USERS @@ -253,9 +314,13 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam ### V +- VALUE - VALUES +- VARCHAR - VARIABLE - VARIABLES +- VERBOSE +- VGROUP - VGROUPS - VIEW - VNODES @@ -263,14 +328,25 @@ There are about 200 keywords reserved by TDengine, they can't be used as the nam ### W - WAL +- WAL_FSYNC_PERIOD +- WAL_LEVEL +- WAL_RETENTION_PERIOD +- WAL_RETENTION_SIZE +- WAL_ROLL_PERIOD +- WAL_SEGMENT_SIZE +- WATERMARK - WHERE +- WINDOW_CLOSE +- WITH +- WRITE ### \_ - \_C0 -- \_QSTART -- \_QSTOP - \_QDURATION -- \_WSTART -- \_WSTOP +- \_QEND +- \_QSTART +- \_ROWTS - \_WDURATION +- \_WEND +- \_WSTART diff --git a/docs/en/12-taos-sql/24-show.md b/docs/en/12-taos-sql/24-show.md index c9adb0cf782d1da63a8f9654f6c89b02a60a7cb7..5f3bef3546ea05745070268e1f6add25add4773b 100644 --- a/docs/en/12-taos-sql/24-show.md +++ b/docs/en/12-taos-sql/24-show.md @@ -5,16 +5,6 @@ title: SHOW Statement for Metadata `SHOW` command can be used to get brief system information. To get details about metatadata, information, and status in the system, please use `select` to query the tables in database `INFORMATION_SCHEMA`. -## SHOW ACCOUNTS - -```sql -SHOW ACCOUNTS; -``` - -Shows information about tenants on the system. - -Note: TDengine Enterprise Edition only. - ## SHOW APPS ```sql diff --git a/docs/en/14-reference/03-connector/06-rust.mdx b/docs/en/14-reference/03-connector/06-rust.mdx index 0d391c6ac308c5e9e998e2e7e3423cc5a809905e..530287e2a424592045755eeecf37894f59639ef3 100644 --- a/docs/en/14-reference/03-connector/06-rust.mdx +++ b/docs/en/14-reference/03-connector/06-rust.mdx @@ -155,15 +155,15 @@ async fn demo(taos: &Taos, db: &str) -> Result<(), Error> { let inserted = taos.exec_many([ // create super table "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \ - TAGS (`groupid` INT, `location` BINARY(16))", + TAGS (`groupid` INT, `location` BINARY(24))", // create child table - "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + "CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')", // insert into child table "INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)", // insert with NULL values "INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)", // insert and automatically create table with tags if not exists - "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)", + "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119, 0.33)", // insert many records in a single sql "INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)", ]).await?; diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index 49e2dceec531cf8449749ea9dbb111079771a788..5dc54f09343853b8c93cab6478a7323c8e617538 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -39,14 +39,14 @@ Comparing the connector support for TDengine functional features as follows. ### Using the native interface (taosc) -| **Functional Features** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | -| **Connection Management** | Support | Support | Support | Support | Support | Support | -| **Regular Query** | Support | Support | Support | Support | Support | Support | -| **Parameter Binding** | Support | Support | Support | Support | Support | Support | -| ** TMQ ** | Support | Support | Support | Support | Support | Support | -| **Schemaless** | Support | Support | Support | Support | Support | Support | -| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | +| **Functional Features** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | +| ----------------------------- | ------------- | ---------- | ------------- | ------------- | ------------- | ------------- | +| **Connection Management** | Support | Support | Support | Support | Support | Support | +| **Regular Query** | Support | Support | Support | Support | Support | Support | +| **Parameter Binding** | Support | Support | Support | Support | Support | Support | +| **Subscription (TMQ)** | Support | Support | Support | Support | Support | Support | +| **Schemaless** | Support | Support | Support | Support | Support | Support | +| **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | :::info The different database framework specifications for various programming languages do not mean that all C/C++ interfaces need a wrapper. @@ -54,16 +54,15 @@ The different database framework specifications for various programming language ### Use HTTP Interfaces (REST or WebSocket) -| **Functional Features** | **Java** | **Python** | **Go** | **C# (not supported yet)** | **Node.js** | **Rust** | -| ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- | -| **Connection Management** | Support | Support | Support | N/A | Support | Support | -| **Regular Query** | Support | Support | Support | N/A | Support | Support | -| **Continous Query ** | Support | Support | Support | N/A | Support | Support | -| **Parameter Binding** | Not supported | Not supported | Not supported | N/A | Not supported | Support | -| ** TMQ ** | Not supported | Not supported | Not supported | N/A | Not supported | Support | -| **Schemaless** | Not supported | Not supported | Not supported | N/A | Not supported | Not supported | -| **Bulk Pulling (based on WebSocket) **| Support | Support | Not Supported | N/A | Not Supported | Supported | -| **DataFrame** | Not supported | Support | Not supported | N/A | Not supported | Not supported | +| **Functional Features** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | +| -------------------------------------- | ------------- | --------------- | ------------- | ------------- | ------------- | ------------- | +| **Connection Management** | Support | Support | Support | Support | Support | Support | +| **Regular Query** | Support | Support | Support | Support | Support | Support | +| **Parameter Binding** | Not supported | Not supported | Not supported | Support | Not supported | Support | +| **Subscription (TMQ) ** | Not supported | Not supported | Not supported | Not supported | Not supported | Support | +| **Schemaless** | Not supported | Not supported | Not supported | Not supported | Not supported | Not supported | +| **Bulk Pulling (based on WebSocket) ** | Support | Support | Not Supported | support | Not Supported | Supported | +| **DataFrame** | Not supported | Support | Not supported | Not supported | Not supported | Not supported | :::warning diff --git a/docs/examples/csharp/SQLInsertExample.cs b/docs/examples/csharp/SQLInsertExample.cs index 192ea96d5713bbf7f37f2208687c41e3e66d473b..3ce70fe9144d997b320610500be29b329b69a08f 100644 --- a/docs/examples/csharp/SQLInsertExample.cs +++ b/docs/examples/csharp/SQLInsertExample.cs @@ -23,6 +23,7 @@ namespace TDengineExample CheckRes(conn, res, "failed to insert data"); int affectedRows = TDengine.AffectRows(res); Console.WriteLine("affectedRows " + affectedRows); + TDengine.FreeResult(res); ExitProgram(conn, 0); } diff --git a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java index 179e6e6911185631901b79e34a343967e73c4936..e9af5e9ce0c0473f4513cbb949dcbd9f433c0c92 100644 --- a/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java +++ b/docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java @@ -38,12 +38,12 @@ public class SubscribeDemo { statement.executeUpdate("create database " + DB_NAME); statement.executeUpdate("use " + DB_NAME); statement.executeUpdate( - "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(16))"); - statement.executeUpdate("CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')"); + "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT) TAGS (`groupid` INT, `location` BINARY(24))"); + statement.executeUpdate("CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')"); statement.executeUpdate("INSERT INTO `d0` values(now - 10s, 0.32, 116)"); statement.executeUpdate("INSERT INTO `d0` values(now - 8s, NULL, NULL)"); statement.executeUpdate( - "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119)"); + "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119)"); statement.executeUpdate( "INSERT INTO `d1` values (now-8s, 10, 120) (now - 6s, 10, 119) (now - 4s, 11.2, 118)"); // create topic @@ -75,4 +75,4 @@ public class SubscribeDemo { } timer.cancel(); } -} \ No newline at end of file +} diff --git a/docs/examples/java/src/main/java/com/taos/example/highvolume/MockDataSource.java b/docs/examples/java/src/main/java/com/taos/example/highvolume/MockDataSource.java index 6fe83f002ebcb9d82e026e9a32886fd22bfefbe9..f0ebc53b4b9a588ac4a23461553dd5c9f1a9f00b 100644 --- a/docs/examples/java/src/main/java/com/taos/example/highvolume/MockDataSource.java +++ b/docs/examples/java/src/main/java/com/taos/example/highvolume/MockDataSource.java @@ -16,7 +16,7 @@ class MockDataSource implements Iterator { private int currentTbId = -1; // mock values - String[] location = {"LosAngeles", "SanDiego", "Hollywood", "Compton", "San Francisco"}; + String[] location = {"California.LosAngeles", "California.SanDiego", "California.SanJose", "California.Campbell", "California.SanFrancisco"}; float[] current = {8.8f, 10.7f, 9.9f, 8.9f, 9.4f}; int[] voltage = {119, 116, 111, 113, 118}; float[] phase = {0.32f, 0.34f, 0.33f, 0.329f, 0.141f}; @@ -50,4 +50,4 @@ class MockDataSource implements Iterator { return sb.toString(); } -} \ No newline at end of file +} diff --git a/docs/examples/python/mockdatasource.py b/docs/examples/python/mockdatasource.py index 852860aec0adc8f9b043c9dcd5deb0bf00239201..1c516a800e007934f8e6815f82024a53fea70073 100644 --- a/docs/examples/python/mockdatasource.py +++ b/docs/examples/python/mockdatasource.py @@ -3,11 +3,11 @@ import time class MockDataSource: samples = [ - "8.8,119,0.32,LosAngeles,0", - "10.7,116,0.34,SanDiego,1", - "9.9,111,0.33,Hollywood,2", - "8.9,113,0.329,Compton,3", - "9.4,118,0.141,San Francisco,4" + "8.8,119,0.32,California.LosAngeles,0", + "10.7,116,0.34,California.SanDiego,1", + "9.9,111,0.33,California.SanJose,2", + "8.9,113,0.329,California.Campbell,3", + "9.4,118,0.141,California.SanFrancisco,4" ] def __init__(self, tb_name_prefix, table_count): diff --git a/docs/examples/rust/nativeexample/examples/stmt_example.rs b/docs/examples/rust/nativeexample/examples/stmt_example.rs index 26084746f20a3662383b417eb98016f09ad0913e..9cf8e8e1fc2526206486fa9a61c01f6320564131 100644 --- a/docs/examples/rust/nativeexample/examples/stmt_example.rs +++ b/docs/examples/rust/nativeexample/examples/stmt_example.rs @@ -12,7 +12,7 @@ async fn main() -> anyhow::Result<()> { // bind table name and tags stmt.set_tbname_tags( "d1001", - &[Value::VarChar("San Fransico".into()), Value::Int(2)], + &[Value::VarChar("California.SanFransico".into()), Value::Int(2)], )?; // bind values. let values = vec![ diff --git a/docs/examples/rust/nativeexample/examples/subscribe_demo.rs b/docs/examples/rust/nativeexample/examples/subscribe_demo.rs index 7e0a347948fc8450dead0babbbdd1eace2f06d1e..11d6d4e0043fddeff73c09d86c0fce0abc903a08 100644 --- a/docs/examples/rust/nativeexample/examples/subscribe_demo.rs +++ b/docs/examples/rust/nativeexample/examples/subscribe_demo.rs @@ -19,13 +19,13 @@ struct Record { async fn prepare(taos: Taos) -> anyhow::Result<()> { let inserted = taos.exec_many([ // create child table - "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + "CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')", // insert into child table "INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)", // insert with NULL values "INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)", // insert and automatically create table with tags if not exists - "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)", + "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119, 0.33)", // insert many records in a single sql "INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)", ]).await?; @@ -48,7 +48,7 @@ async fn main() -> anyhow::Result<()> { format!("CREATE DATABASE `{db}`"), format!("USE `{db}`"), // create super table - format!("CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(16))"), + format!("CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) TAGS (`groupid` INT, `location` BINARY(24))"), // create topic for subscription format!("CREATE TOPIC tmq_meters with META AS DATABASE {db}") ]) diff --git a/docs/examples/rust/restexample/examples/insert_example.rs b/docs/examples/rust/restexample/examples/insert_example.rs index 27b2bb4788615810d097b88f0dd616b96885538c..11a84f166103eba03b43549d4db77100a92a58e6 100644 --- a/docs/examples/rust/restexample/examples/insert_example.rs +++ b/docs/examples/rust/restexample/examples/insert_example.rs @@ -14,14 +14,14 @@ async fn main() -> anyhow::Result<()> { ]).await?; let inserted = taos.exec("INSERT INTO - power.d1001 USING power.meters TAGS('San Francisco', 2) + power.d1001 USING power.meters TAGS('California.SanFrancisco', 2) VALUES ('2018-10-03 14:38:05.000', 10.30000, 219, 0.31000) ('2018-10-03 14:38:15.000', 12.60000, 218, 0.33000) ('2018-10-03 14:38:16.800', 12.30000, 221, 0.31000) - power.d1002 USING power.meters TAGS('San Francisco', 3) + power.d1002 USING power.meters TAGS('California.SanFrancisco', 3) VALUES ('2018-10-03 14:38:16.650', 10.30000, 218, 0.25000) - power.d1003 USING power.meters TAGS('Los Angeles', 2) + power.d1003 USING power.meters TAGS('California.LosAngeles', 2) VALUES ('2018-10-03 14:38:05.500', 11.80000, 221, 0.28000) ('2018-10-03 14:38:16.600', 13.40000, 223, 0.29000) - power.d1004 USING power.meters TAGS('Los Angeles', 3) + power.d1004 USING power.meters TAGS('California.LosAngeles', 3) VALUES ('2018-10-03 14:38:05.000', 10.80000, 223, 0.29000) ('2018-10-03 14:38:06.500', 11.50000, 221, 0.35000)").await?; assert_eq!(inserted, 8); diff --git a/docs/zh/02-intro.md b/docs/zh/02-intro.md index 9a0a6fb547ceef373bf7f57e333fda93d40eb5b6..47bfd3f96b6fdbb27d3f3e326e14a6b22108d508 100644 --- a/docs/zh/02-intro.md +++ b/docs/zh/02-intro.md @@ -48,7 +48,7 @@ TDengine 的主要功能如下： - 多种[数据导出](../operation/export)方式 9. 工具 - 提供[交互式命令行程序（CLI）](../reference/taos-shell)，便于管理集群，检查系统状态，做即席查询 - - 提供压力测试工具[taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能 + - 提供压力测试工具 [taosBenchmark](../reference/taosbenchmark)，用于测试 TDengine 的性能 10. 编程 - 提供各种语言的[连接器（Connector）](../connector): 如 [C/C++](../connector/cpp)、[Java](../connector/java)、[Go](../connector/go)、[Node.js](../connector/node)、[Rust](../connector/rust)、[Python](../connector/python)、[C#](../connector/csharp) 等 - 支持 [REST 接口](../connector/rest-api/) diff --git a/docs/zh/04-concept/index.md b/docs/zh/04-concept/index.md index 89d3df9c973d9a319397285599e6b2e6be3785de..2cba68edcd152f5059845b9e25342b3f335f3b8b 100644 --- a/docs/zh/04-concept/index.md +++ b/docs/zh/04-concept/index.md @@ -4,119 +4,118 @@ title: 数据模型和基本概念 description: TDengine 的数据模型和基本概念 --- -为了便于解释基本概念，便于撰写示例程序，整个 TDengine 文档以智能电表作为典型时序数据场景。假设每个智能电表采集电流、电压、相位三个量，有多个智能电表，每个电表有位置 location 和分组 group ID 的静态属性. 其采集的数据类似如下的表格： +为了便于解释基本概念，便于撰写示例程序，整个 TDengine 文档以智能电表作为典型时序数据场景。假设每个智能电表采集电流、电压、相位三个量，有多个智能电表，每个电表有位置 Location 和分组 Group ID 的静态属性. 其采集的数据类似如下的表格：
- - - - - + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
Device IDTime StampCollected MetricsTags
Device IDTimestampCollected MetricsTags
Device IDTime StampcurrentvoltagephaselocationgroupId
d1001153854868500010.32190.31California.SanFrancisco2
d1002153854868400010.22200.23California.SanFrancisco3
d1003153854868650011.52210.35California.LosAngeles3
d1004153854868550013.42230.29California.LosAngeles2
d1001153854869500012.62180.33California.SanFrancisco2
d1004153854869660011.82210.28California.LosAngeles2
d1002153854869665010.32180.25California.SanFrancisco3
d1001153854869680012.32210.31California.SanFrancisco2
currentvoltagephaselocationgroupid
d1001153854868500010.32190.31California.SanFrancisco2
d1002153854868400010.22200.23California.SanFrancisco3
d1003153854868650011.52210.35California.LosAngeles3
d1004153854868550013.42230.29California.LosAngeles2
d1001153854869500012.62180.33California.SanFrancisco2
d1004153854869660011.82210.28California.LosAngeles2
d1002153854869665010.32180.25California.SanFrancisco3
d1001153854869680012.32210.31California.SanFrancisco2
-表 1：智能电表数据示例 +表 1. 智能电表数据示例
-每一条记录都有设备 ID，时间戳，采集的物理量以及每个设备相关的静态标签。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。 +每一条记录都有设备 ID、时间戳、采集的物理量（如上表中的 `current`、`voltage` 和 `phase`）以及每个设备相关的静态标签（`location` 和 `groupid`）。每个设备是受外界的触发，或按照设定的周期采集数据。采集的数据点是时序的，是一个数据流。 -## 采集量 (Metric) +## 采集量（Metric） 采集量是指传感器、设备或其他类型采集点采集的物理量，比如电流、电压、温度、压力、GPS 位置等，是随时间变化的，数据类型可以是整型、浮点型、布尔型，也可是字符串。随着时间的推移，存储的采集量的数据量越来越大。智能电表示例中的电流、电压、相位就是采集量。 -## 标签 (Label/Tag) +## 标签（Label/Tag） -标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的location与groupId就是标签。 +标签是指传感器、设备或其他类型采集点的静态属性，不是随时间变化的，比如设备型号、颜色、设备的所在地等，数据类型可以是任何类型。虽然是静态的，但 TDengine 容许用户修改、删除或增加标签值。与采集量不一样的是，随时间的推移，存储的标签的数据量不会有什么变化。智能电表示例中的 `location` 与 `groupid` 就是标签。 -## 数据采集点 (Data Collection Point) +## 数据采集点（Data Collection Point） -数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的d1001, d1002, d1003, d1004等就是数据采集点。 +数据采集点是指按照预设时间周期或受事件触发采集物理量的硬件或软件。一个数据采集点可以采集一个或多个采集量，**但这些采集量都是同一时刻采集的，具有相同的时间戳**。对于复杂的设备，往往有多个数据采集点，每个数据采集点采集的周期都可能不一样，而且完全独立，不同步。比如对于一台汽车，有数据采集点专门采集 GPS 位置，有数据采集点专门采集发动机状态，有数据采集点专门采集车内的环境，这样一台汽车就有三个数据采集点。智能电表示例中的 d1001、d1002、d1003、d1004 等就是数据采集点。 -## 表 (Table) +## 表（Table） 因为采集量一般是结构化数据，同时为降低学习门槛，TDengine 采用传统的关系型数据库模型管理数据。用户需要先创建库，然后创建表，之后才能插入或查询数据。 @@ -129,50 +128,56 @@ description: TDengine 的数据模型和基本概念 如果采用传统的方式，将多个数据采集点的数据写入一张表，由于网络延时不可控，不同数据采集点的数据到达服务器的时序是无法保证的，写入操作是要有锁保护的，而且一个数据采集点的数据是难以保证连续存储在一起的。**采用一个数据采集点一张表的方式，能最大程度的保证单个数据采集点的插入和查询的性能是最优的。** -TDengine 建议用数据采集点的名字（如上表中的 D1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 current，voltage，phase），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。 - -对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一台汽车建立多张表。 +TDengine 建议用数据采集点的名字（如上表中的 d1001）来做表名。每个数据采集点可能同时采集多个采集量（如上表中的 `current`、`voltage` 和 `phase`），每个采集量对应一张表中的一列，数据类型可以是整型、浮点型、字符串等。除此之外，表的第一列必须是时间戳，即数据类型为 Timestamp。对采集量，TDengine 将自动按照时间戳建立索引，但对采集量本身不建任何索引。数据用列式存储方式保存。 +对于复杂的设备，比如汽车，它有多个数据采集点，那么就需要为一辆汽车建立多张表。 -## 超级表 (STable) +## 超级表（STable） 由于一个数据采集点一张表，导致表的数量巨增，难以管理，而且应用经常需要做采集点之间的聚合操作，聚合的操作也变得复杂起来。为解决这个问题，TDengine 引入超级表（Super Table，简称为 STable）的概念。 -超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 schema，标签的数据类型可以是整数、浮点数、字符串，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。 +超级表是指某一特定类型的数据采集点的集合。同一类型的数据采集点，其表的结构是完全一样的，但每个表（数据采集点）的静态属性（标签）是不一样的。描述一个超级表（某一特定类型的数据采集点的集合），除需要定义采集量的表结构之外，还需要定义其标签的 Schema，标签的数据类型可以是整数、浮点数、字符串、JSON，标签可以有多个，可以事后增加、删除或修改。如果整个系统有 N 个不同类型的数据采集点，就需要建立 N 个超级表。 -在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表meters. +在 TDengine 的设计里，**表用来代表一个具体的数据采集点，超级表用来代表一组相同类型的数据采集点集合**。智能电表示例中，我们可以创建一个超级表 `meters`. -## 子表 (Subtable) +## 子表（Subtable） 当为某个具体数据采集点创建表时，用户可以使用超级表的定义做模板，同时指定该具体采集点（表）的具体标签值来创建该表。**通过超级表创建的表称之为子表**。正常的表与子表的差异在于： -1. 子表就是表，因此所有正常表的SQL操作都可以在子表上执行。 +1. 子表就是表，因此所有正常表的 SQL 操作都可以在子表上执行。 2. 子表在正常表的基础上有扩展，它是带有静态标签的，而且这些标签可以事后增加、删除、修改，而正常的表没有。 3. 子表一定属于一张超级表，但普通表不属于任何超级表 4. 普通表无法转为子表，子表也无法转为普通表。 超级表与与基于超级表建立的子表之间的关系表现在： -1. 一张超级表包含有多张子表，这些子表具有相同的采集量 schema，但带有不同的标签值。 +1. 一张超级表包含有多张子表，这些子表具有相同的采集量 Schema，但带有不同的标签值。 2. 不能通过子表调整数据或标签的模式，对于超级表的数据模式修改立即对所有的子表生效。 3. 超级表只定义一个模板，自身不存储任何数据或标签信息。因此，不能向一个超级表写入数据，只能将数据写入子表中。 查询既可以在表上进行，也可以在超级表上进行。针对超级表的查询，TDengine 将把所有子表中的数据视为一个整体数据集进行处理，会先把满足标签过滤条件的表从超级表中找出来，然后再扫描这些表的时序数据，进行聚合操作，这样需要扫描的数据集会大幅减少，从而显著提高查询的性能。本质上，TDengine 通过对超级表查询的支持，实现了多个同类数据采集点的高效聚合。 -TDengine系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表meters创建子表d1001, d1002, d1003, d1004等。 +TDengine 系统建议给一个数据采集点建表，需要通过超级表建表，而不是建普通表。在智能电表的示例中，我们可以通过超级表 meters 创建子表 d1001、d1002、d1003、d1004 等。 + +为了更好地理解采集量、标签、超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。 + +
+ +![智能电表数据模型示意图](./supertable.webp) -为了更好地理解超级与子表的关系，可以参考下面关于智能电表数据模型的示意图。 ![智能电表数据模型示意图](./supertable.webp) +
图 1. 智能电表数据模型示意图
+
-## 库 (database) +## 库（Database） 库是指一组表的集合。TDengine 容许一个运行实例有多个库，而且每个库可以配置不同的存储策略。不同类型的数据采集点往往具有不同的数据特征，包括数据采集频率的高低，数据保留时间的长短，副本的数目，数据块的大小，是否允许更新数据等等。为了在各种场景下 TDengine 都能最大效率的工作，TDengine 建议将不同数据特征的超级表创建在不同的库里。 一个库里，可以有一到多个超级表，但一个超级表只属于一个库。一个超级表所拥有的子表全部存在一个库里。 -## FQDN & End Point +## FQDN & Endpoint -FQDN (fully qualified domain name, 完全限定域名)是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS(Domain Name System)，负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。 +FQDN（Fully Qualified Domain Name，完全限定域名）是 Internet 上特定计算机或主机的完整域名。FQDN 由两部分组成:主机名和域名。例如，假设邮件服务器的 FQDN 可能是 mail.tdengine.com。主机名是 mail，主机位于域名 tdengine.com 中。DNS（Domain Name System），负责将 FQDN 翻译成 IP，是互联网应用的寻址方式。对于没有 DNS 的系统，可以通过配置 hosts 文件来解决。 -TDengine 集群的每个节点是由 End Point 来唯一标识的，End Point 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。 +TDengine 集群的每个节点是由 Endpoint 来唯一标识的，Endpoint 是由 FQDN 外加 Port 组成，比如 h1.tdengine.com:6030。这样当 IP 发生变化的时候，我们依然可以使用 FQDN 来动态找到节点，不需要更改集群的任何配置。而且采用 FQDN，便于内网和外网对同一个集群的统一访问。 TDengine 不建议采用直接的 IP 地址访问集群，不利于管理。不了解 FQDN 概念，请看博文[《一篇文章说清楚 TDengine 的 FQDN》](https://www.taosdata.com/blog/2020/09/11/1824.html)。 diff --git a/docs/zh/05-get-started/01-docker.md b/docs/zh/05-get-started/01-docker.md index e2be4195176a3f1ac7712a036d04b60b2fb77718..0f004581b5be470f3c99f48e1aaecb3578c442ab 100644 --- a/docs/zh/05-get-started/01-docker.md +++ b/docs/zh/05-get-started/01-docker.md @@ -4,11 +4,11 @@ title: 通过 Docker 快速体验 TDengine description: 使用 Docker 快速体验 TDengine 的高效写入和查询 --- -本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. +本节首先介绍如何通过 Docker 快速体验 TDengine，然后介绍如何在 Docker 环境下体验 TDengine 的写入和查询功能。如果你不熟悉 Docker，请使用[安装包的方式快速体验](../../get-started/package/)。如果您希望为 TDengine 贡献代码或对内部技术实现感兴趣，请参考 [TDengine GitHub 主页](https://github.com/taosdata/TDengine)下载源码构建和安装。 ## 启动 TDengine -如果已经安装了 docker， 只需执行下面的命令。 +如果已经安装了 Docker，只需执行下面的命令： ```shell docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043-6049/udp tdengine/tdengine @@ -16,84 +16,84 @@ docker run -d -p 6030:6030 -p 6041:6041 -p 6043-6049:6043-6049 -p 6043-6049:6043 注意：TDengine 3.0 服务端仅使用 6030 TCP 端口。6041 为 taosAdapter 所使用提供 REST 服务端口。6043-6049 为 taosAdapter 提供第三方应用接入所使用端口，可根据需要选择是否打开。 -确定该容器已经启动并且在正常运行 +确定该容器已经启动并且在正常运行。 ```shell docker ps ``` -进入该容器并执行 bash +进入该容器并执行 `bash` ```shell docker exec -it bash ``` -然后就可以执行相关的 Linux 命令操作和访问 TDengine +然后就可以执行相关的 Linux 命令操作和访问 TDengine。 -注: Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。 +注：Docker 工具自身的下载和使用请参考 [Docker 官网文档](https://docs.docker.com/get-docker/)。 ## 运行 TDengine CLI -进入容器，执行 taos +进入容器，执行 `taos`： ``` $ taos -taos> - +taos> ``` -## 写入数据 +## 使用 taosBenchmark 体验写入速度 -可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入。 +可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。 -进入容器，启动 taosBenchmark： +启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）： - ```bash - $ taosBenchmark - - ``` +```bash +$ taosBenchmark +``` - 该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "San Francisco" 或者 "Los Angeles"等城市名称。 +该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `California.Campbell`、`California.Cupertino`、`California.LosAngeles`、`California.MountainView`、`California.PaloAlto`、`California.SanDiego`、`California.SanFrancisco`、`California.SanJose`、`California.SantaClara` 或者 `California.Sunnyvale`。 - 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能。 +这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。 - taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [taosBenchmark 参考手册](../../reference/taosbenchmark)。 +taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。 -## 体验查询 +## 使用 TDengine CLI 体验查询速度 -使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。。 +使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。 -查询超级表下记录总条数： +查询超级表 `meters` 下的记录总条数： ```sql -taos> select count(*) from test.meters; +SELECT COUNT(*) FROM test.meters; ``` 查询 1 亿条记录的平均值、最大值、最小值等： ```sql -taos> select avg(current), max(voltage), min(phase) from test.meters; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters; ``` -查询 location="San Francisco" 的记录总条数： +查询 location = "California.SanFrancisco" 的记录总条数： ```sql -taos> select count(*) from test.meters where location="San Francisco"; +SELECT COUNT(*) FROM test.meters WHERE location = "California.SanFrancisco"; ``` -查询 groupId=10 的所有记录的平均值、最大值、最小值等： +查询 groupId = 10 的所有记录的平均值、最大值、最小值等： ```sql -taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10; ``` -对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： +对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计： ```sql -taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); +SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s); ``` +在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。 + ## 其它 -更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker) +更多关于在 Docker 环境下使用 TDengine 的细节，请参考 [在 Docker 下使用 TDengine](../../reference/docker)。 diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index 3e0fb056a5913d3a82a473bf879a79e398176075..66863d1bd9419e6d8fc78cf1536e1efac38885d6 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -10,23 +10,24 @@ import PkgListV3 from "/components/PkgListV3"; 您可以[用 Docker 立即体验](../../get-started/docker/) TDengine。如果您希望对 TDengine 贡献代码或对内部实现感兴趣，请参考我们的 [TDengine GitHub 主页](https://github.com/taosdata/TDengine) 下载源码构建和安装. -TDengine 完整的软件包包括服务端（taosd）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、应用驱动（taosc）、命令行程序 (CLI，taos) 和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。 +TDengine 完整的软件包包括服务端（taosd）、应用驱动（taosc）、用于与第三方系统对接并提供 RESTful 接口的 taosAdapter、命令行程序（CLI，taos）和一些工具软件。目前 taosAdapter 仅在 Linux 系统上安装和运行，后续将支持 Windows、macOS 等系统。TDengine 除了提供多种语言的连接器之外，还通过 [taosAdapter](../../reference/taosadapter/) 提供 [RESTful 接口](../../connector/rest-api/)。 -为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 lite 版本的安装包。 +为方便使用，标准的服务端安装包包含了 taosd、taosAdapter、taosc、taos、taosdump、taosBenchmark、TDinsight 安装脚本和示例代码；如果您只需要用到服务端程序和客户端连接的 C/C++ 语言支持，也可以仅下载 Lite 版本的安装包。 -在 Linux 系统上，TDengine 开源版本提供 deb 和 rpm 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 deb 支持 Debian/Ubuntu 及衍生系统，rpm 支持 CentOS/RHEL/SUSE 及衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，rpm 和 deb 包不含 taosdump 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。 +在 Linux 系统上，TDengine 社区版提供 Deb 和 RPM 格式安装包，用户可以根据自己的运行环境选择合适的安装包。其中 Deb 支持 Debian/Ubuntu 及其衍生系统，RPM 支持 CentOS/RHEL/SUSE 及其衍生系统。同时我们也为企业用户提供 tar.gz 格式安装包，也支持通过 `apt-get` 工具从线上进行安装。需要注意的是，RPM 和 Deb 包不含 `taosdump` 和 TDinsight 安装脚本，这些工具需要通过安装 taosTool 包获得。TDengine 也提供 Windows x64 平台的安装包。 ## 安装 -1. 从列表中下载获得 deb 安装包； - +1. 从列表中下载获得 Deb 安装包； + 2. 进入到安装包所在目录，执行如下的安装命令： +> 请将 `` 替换为下载的安装包版本 + ```bash -# 替换为下载的安装包版本 sudo dpkg -i TDengine-server--Linux-x64.deb ``` @@ -34,12 +35,13 @@ sudo dpkg -i TDengine-server--Linux-x64.deb -1. 从列表中下载获得 rpm 安装包； - +1. 从列表中下载获得 RPM 安装包； + 2. 进入到安装包所在目录，执行如下的安装命令： +> 请将 `` 替换为下载的安装包版本 + ```bash -# 替换为下载的安装包版本 sudo rpm -ivh TDengine-server--Linux-x64.rpm ``` @@ -48,44 +50,46 @@ sudo rpm -ivh TDengine-server--Linux-x64.rpm 1. 从列表中下载获得 tar.gz 安装包； - -2. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本： + +2. 进入到安装包所在目录，使用 `tar` 解压安装包； +3. 进入到安装包所在目录，先解压文件后，进入子目录，执行其中的 install.sh 安装脚本。 + +> 请将 `` 替换为下载的安装包版本 ```bash -# 替换为下载的安装包版本 tar -zxvf TDengine-server--Linux-x64.tar.gz ``` -解压后进入相应路径，执行 +解压文件后，进入相应子目录，执行其中的 `install.sh` 安装脚本： ```bash sudo ./install.sh ``` :::info -install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以用 -e no 参数来执行 install.sh 脚本。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。 +install.sh 安装脚本在执行过程中，会通过命令行交互界面询问一些配置信息。如果希望采取无交互安装方式，那么可以运行 `./install.sh -e no`。运行 `./install.sh -h` 指令可以查看所有参数的详细说明信息。 ::: -可以使用 apt-get 工具从官方仓库安装。 +可以使用 `apt-get` 工具从官方仓库安装。 -**安装包仓库** +**配置包仓库** ```bash wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-stable stable main" | sudo tee /etc/apt/sources.list.d/tdengine-stable.list ``` -如果安装 Beta 版需要安装包仓库 +如果安装 Beta 版需要安装包仓库： ```bash wget -qO - http://repos.taosdata.com/tdengine.key | sudo apt-key add - echo "deb [arch=amd64] http://repos.taosdata.com/tdengine-beta beta main" | sudo tee /etc/apt/sources.list.d/tdengine-beta.list ``` -**使用 apt-get 命令安装** +**使用 `apt-get` 命令安装** ```bash sudo apt-get update @@ -94,26 +98,26 @@ sudo apt-get install tdengine ``` :::tip -apt-get 方式只适用于 Debian 或 Ubuntu 系统 +apt-get 方式只适用于 Debian 或 Ubuntu 系统。 :::: - + -注意：目前 TDengine 在 Windows 平台上只支持 Windows server 2016/2019 和 Windows 10/11 系统版本。 +注意：目前 TDengine 在 Windows 平台上只支持 Windows Server 2016/2019 和 Windows 10/11。 1. 从列表中下载获得 exe 安装程序； - + 2. 运行可执行程序来安装 TDengine。 :::info -下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine) +下载其他组件、最新 Beta 版及之前版本的安装包，请点击[发布历史页面](../../releases/tdengine)。 ::: :::note -当安装第一个节点时，出现 Enter FQDN：提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。 +当安装第一个节点时，出现 `Enter FQDN:` 提示的时候，不需要输入任何内容。只有当安装第二个或以后更多的节点时，才需要输入已有集群中任何一个可用节点的 FQDN，支持该新节点加入集群。当然也可以不输入，而是在新节点启动前，配置到新节点的配置文件中。 ::: @@ -148,7 +152,7 @@ Active: inactive (dead) 如果 TDengine 服务正常工作，那么您可以通过 TDengine 的命令行程序 `taos` 来访问并体验 TDengine。 -systemctl 命令汇总： +如下 `systemctl` 命令可以帮助你管理 TDengine 服务： - 启动服务进程：`systemctl start taosd` @@ -160,7 +164,7 @@ systemctl 命令汇总： :::info -- systemctl 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 sudo 。 +- `systemctl` 命令需要 _root_ 权限来运行，如果您非 _root_ 用户，请在命令前添加 `sudo`。 - `systemctl stop taosd` 指令在执行后并不会马上停止 TDengine 服务，而是会等待系统中必要的落盘工作正常完成。在数据量很大的情况下，这可能会消耗较长时间。 - 如果系统中不支持 `systemd`，也可以用手动运行 `/usr/local/taos/bin/taosd` 方式启动 TDengine 服务。 @@ -170,87 +174,93 @@ systemctl 命令汇总： -安装后，在 C:\TDengine 目录下，运行 taosd.exe 来启动 TDengine 服务进程。 +安装后，在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 -## TDengine 命令行 (CLI) +## TDengine 命令行（CLI） -为便于检查 TDengine 的状态，执行数据库 (Database) 的各种即席(Ad Hoc)查询，TDengine 提供一命令行应用程序(以下简称为 TDengine CLI) taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 +为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在安装有 TDengine 的 Linux 终端执行 `taos` 即可，也可以在安装有 TDengine 的 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 ```bash taos ``` -如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。 TDengine CLI 的提示符号如下： +如果连接服务成功，将会打印出欢迎消息和版本信息。如果失败，则会打印错误消息出来（请参考 [FAQ](/train-faq/faq) 来解决终端连接服务端失败的问题）。TDengine CLI 的提示符号如下： ```cmd taos> ``` -在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库(database)插入查询操作。在终端中运行的 SQL 语句需要以分号结束来运行。示例： +在 TDengine CLI 中，用户可以通过 SQL 命令来创建/删除数据库、表等，并进行数据库（Database）插入查询操作。在终端中运行的 SQL 语句需要以分号（;）结束来运行。示例： ```sql -create database demo; -use demo; -create table t (ts timestamp, speed int); -insert into t values ('2019-07-15 00:00:00', 10); -insert into t values ('2019-07-15 01:00:00', 20); -select * from t; +CREATE DATABASE demo; +USE demo; +CREATE TABLE t (ts TIMESTAMP, speed INT); +INSERT INTO t VALUES ('2019-07-15 00:00:00', 10); +INSERT INTO t VALUES ('2019-07-15 01:00:00', 20); +SELECT * FROM t; + ts | speed | ======================================== 2019-07-15 00:00:00.000 | 10 | 2019-07-15 01:00:00.000 | 20 | + Query OK, 2 row(s) in set (0.003128s) ``` -除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [这里](../../reference/taos-shell/) +除执行 SQL 语句外，系统管理员还可以从 TDengine CLI 进行检查系统运行状态、添加删除用户账号等操作。TDengine CLI 连同应用驱动也可以独立安装在 Linux 或 Windows 机器上运行，更多细节请参考 [TDengine 命令行](../../reference/taos-shell/)。 ## 使用 taosBenchmark 体验写入速度 -启动 TDengine 的服务，在 Linux 或 windows 终端执行 `taosBenchmark` （曾命名为 `taosdemo`）： +可以使用 TDengine 的自带工具 taosBenchmark 快速体验 TDengine 的写入速度。 + +启动 TDengine 的服务，在 Linux 或 Windows 终端执行 `taosBenchmark`（曾命名为 `taosdemo`）： ```bash -taosBenchmark +$ taosBenchmark ``` -该命令将在数据库 test 下面自动创建一张超级表 meters，该超级表下有 1 万张表，表名为 "d0" 到 "d9999"，每张表有 1 万条记录，每条记录有 (ts, current, voltage, phase) 四个字段，时间戳从 "2017-07-14 10:40:00 000" 到 "2017-07-14 10:40:09 999"，每张表带有标签 location 和 groupId，groupId 被设置为 1 到 10， location 被设置为 "California.SanFrancisco" 或者 "California.LosAngeles"。 +该命令将在数据库 `test` 下面自动创建一张超级表 `meters`，该超级表下有 1 万张表，表名为 `d0` 到 `d9999`，每张表有 1 万条记录，每条记录有 `ts`、`current`、`voltage`、`phase` 四个字段，时间戳从 2017-07-14 10:40:00 000 到 2017-07-14 10:40:09 999，每张表带有标签 `location` 和 `groupId`，groupId 被设置为 1 到 10，location 被设置为 `California.Campbell`、`California.Cupertino`、`California.LosAngeles`、`California.MountainView`、`California.PaloAlto`、`California.SanDiego`、`California.SanFrancisco`、`California.SanJose`、`California.SantaClara` 或者 `California.Sunnyvale`。 这条命令很快完成 1 亿条记录的插入。具体时间取决于硬件性能，即使在一台普通的 PC 服务器往往也仅需十几秒。 -taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照 [如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)。 +taosBenchmark 命令本身带有很多选项，配置表的数目、记录条数等等，您可以设置不同参数进行体验，请执行 `taosBenchmark --help` 详细列出。taosBenchmark 详细使用方法请参照[如何使用 taosBenchmark 对 TDengine 进行性能测试](https://www.taosdata.com/2021/10/09/3111.html)和 [taosBenchmark 参考手册](../../reference/taosbenchmark)。 ## 使用 TDengine CLI 体验查询速度 -使用上述 taosBenchmark 插入数据后，可以在 TDengine CLI 输入查询命令，体验查询速度。 +使用上述 `taosBenchmark` 插入数据后，可以在 TDengine CLI（taos）输入查询命令，体验查询速度。 -查询超级表下记录总条数： +查询超级表 `meters` 下的记录总条数： ```sql -taos> select count(*) from test.meters; +SELECT COUNT(*) FROM test.meters; ``` 查询 1 亿条记录的平均值、最大值、最小值等： ```sql -taos> select avg(current), max(voltage), min(phase) from test.meters; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters; ``` -查询 location="California.SanFrancisco" 的记录总条数： +查询 location = "California.SanFrancisco" 的记录总条数： ```sql -taos> select count(*) from test.meters where location="California.SanFrancisco"; +SELECT COUNT(*) FROM test.meters WHERE location = "Calaifornia.SanFrancisco"; ``` -查询 groupId=10 的所有记录的平均值、最大值、最小值等： +查询 groupId = 10 的所有记录的平均值、最大值、最小值等： ```sql -taos> select avg(current), max(voltage), min(phase) from test.meters where groupId=10; +SELECT AVG(current), MAX(voltage), MIN(phase) FROM test.meters WHERE groupId = 10; ``` -对表 d10 按 10s 进行平均值、最大值和最小值聚合统计： +对表 `d10` 按 10 每秒进行平均值、最大值和最小值聚合统计： ```sql -taos> select avg(current), max(voltage), min(phase) from test.d10 interval(10s); +SELECT FIRST(ts), AVG(current), MAX(voltage), MIN(phase) FROM test.d10 INTERVAL(10s); ``` + +在上面的查询中，你选择的是区间内的第一个时间戳（ts），另一种选择方式是 `\_wstart`，它将给出时间窗口的开始。关于窗口查询的更多信息，参见[特色查询](../../taos-sql/distinguished/)。 diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index ef1cd4797a217b601b1b8e3eaa0e74b8c2907c88..3239eae49b05180c4a0dba5850de9f1c5e08a4f3 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -116,7 +116,7 @@ aggfn为函数名的占位符，需要修改为自己的函数名，如l2norm。 参数的具体含义是： - inputDataBlock: 输入的数据块 - - resultColumn: 输出列。输出列 + - resultColumn: 输出列 ### 聚合接口函数 diff --git a/docs/zh/07-develop/_sub_java.mdx b/docs/zh/07-develop/_sub_java.mdx index d14b5fd6095dd90f89dd2c2e828858585cfddff9..e7de158cc8d2b0b686b25bbe96e7a092c2a68e51 100644 --- a/docs/zh/07-develop/_sub_java.mdx +++ b/docs/zh/07-develop/_sub_java.mdx @@ -1,7 +1,5 @@ ```java {{#include docs/examples/java/src/main/java/com/taos/example/SubscribeDemo.java}} -{{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} -{{#include docs/examples/java/src/main/java/com/taos/example/Meters.java}} ``` ```java {{#include docs/examples/java/src/main/java/com/taos/example/MetersDeserializer.java}} diff --git a/docs/zh/08-connector/06-rust.mdx b/docs/zh/08-connector/06-rust.mdx index 26f53c82d630fda168dd98b4c8ec993afc5e3a1d..b838e5c5a2f5c33f09dce6cf1a8d5f61ddc5f264 100644 --- a/docs/zh/08-connector/06-rust.mdx +++ b/docs/zh/08-connector/06-rust.mdx @@ -155,15 +155,15 @@ async fn demo(taos: &Taos, db: &str) -> Result<(), Error> { let inserted = taos.exec_many([ // create super table "CREATE TABLE `meters` (`ts` TIMESTAMP, `current` FLOAT, `voltage` INT, `phase` FLOAT) \ - TAGS (`groupid` INT, `location` BINARY(16))", + TAGS (`groupid` INT, `location` BINARY(24))", // create child table - "CREATE TABLE `d0` USING `meters` TAGS(0, 'Los Angles')", + "CREATE TABLE `d0` USING `meters` TAGS(0, 'California.LosAngles')", // insert into child table "INSERT INTO `d0` values(now - 10s, 10, 116, 0.32)", // insert with NULL values "INSERT INTO `d0` values(now - 8s, NULL, NULL, NULL)", // insert and automatically create table with tags if not exists - "INSERT INTO `d1` USING `meters` TAGS(1, 'San Francisco') values(now - 9s, 10.1, 119, 0.33)", + "INSERT INTO `d1` USING `meters` TAGS(1, 'California.SanFrancisco') values(now - 9s, 10.1, 119, 0.33)", // insert many records in a single sql "INSERT INTO `d1` values (now-8s, 10, 120, 0.33) (now - 6s, 10, 119, 0.34) (now - 4s, 11.2, 118, 0.322)", ]).await?; diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index 17de8e926cd9a3633dc8746b0fb49c38ff8ca61f..f54470f7420ada71c2cd283eff52c5fc6e9ada1a 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -41,14 +41,14 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 ### 使用原生接口（taosc） -| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | -| -------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | -| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| ** TMQ ** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | -| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | +| **功能特性** | **Java** | **Python** | **Go** | **C#** | **Node.js** | **Rust** | +| ------------------- | -------- | ---------- | ------ | ------ | ----------- | -------- | +| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **参数绑定** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **Schemaless** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | :::info 由于不同编程语言数据库框架规范不同，并不意味着所有 C/C++ 接口都需要对应封装支持。 @@ -56,16 +56,15 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 ### 使用 http (REST 或 WebSocket) 接口 -| **功能特性** | **Java** | **Python** | **Go** | **C#（暂不支持）** | **Node.js** | **Rust** | -| ------------------------------ | -------- | ---------- | -------- | ------------------ | ----------- | -------- | -| **连接管理** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | -| **普通查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | -| **连续查询** | 支持 | 支持 | 支持 | N/A | 支持 | 支持 | -| **参数绑定** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | -| ** TMQ ** | 不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 支持 | -| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | N/A | 不支持 | 暂不支持 | -| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | N/A | 不支持 | 支持 | -| **DataFrame** | 不支持 | 支持 | 不支持 | N/A | 不支持 | 不支持 | +| **功能特性** | **Java** | **Python** | **Go** | **C# ** | **Node.js** | **Rust** | +| ------------------------------ | -------- | ---------- | -------- | -------- | ----------- | -------- | +| **连接管理** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | +| **参数绑定** | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | 暂不支持 | 支持 | +| **数据订阅（TMQ）** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 支持 | +| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | +| **批量拉取（基于 WebSocket）** | 支持 | 支持 | 暂不支持 | 支持 | 暂不支持 | 支持 | +| **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | :::warning diff --git a/docs/zh/12-taos-sql/01-data-type.md b/docs/zh/12-taos-sql/01-data-type.md index b8ef050fb79fce5e5d2d65753480a6b156cfbc40..ee7b3a4715a11346b9a06da20dbc93ef309c0a3d 100644 --- a/docs/zh/12-taos-sql/01-data-type.md +++ b/docs/zh/12-taos-sql/01-data-type.md @@ -1,6 +1,6 @@ --- -sidebar_label: 支持的数据类型 -title: 支持的数据类型 +sidebar_label: 数据类型 +title: 数据类型 description: "TDengine 支持的数据类型: 时间戳、浮点型、JSON 类型等" --- diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index 2e5a42d7bdbe30e8fcb34be3bcaa4248174c846c..c76311f008433f36259b08acaf56cafa729550b7 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -1,6 +1,6 @@ --- -sidebar_label: 数据库管理 -title: 数据库管理 +sidebar_label: 数据库 +title: 数据库 description: "创建、删除数据库，查看、修改数据库参数" --- diff --git a/docs/zh/12-taos-sql/03-table.md b/docs/zh/12-taos-sql/03-table.md index 9c33c45efcf006344ba5d84a0cbce7bc683f8559..f6790e3c692b815c1031413933c47eb7ad203204 100644 --- a/docs/zh/12-taos-sql/03-table.md +++ b/docs/zh/12-taos-sql/03-table.md @@ -1,5 +1,5 @@ --- -title: 表管理 +title: 表 sidebar_label: 表 description: 对表的各种管理操作 --- @@ -23,10 +23,7 @@ create_subtable_clause: { } create_definition: - col_name column_definition - -column_definition: - type_name [comment 'string_value'] + col_name column_type table_options: table_option ... diff --git a/docs/zh/12-taos-sql/04-stable.md b/docs/zh/12-taos-sql/04-stable.md index 450ff07fd8eb636b3ee185e5594d77d645195c56..95ef405fa780e831628e21766e1b3c3b18265059 100644 --- a/docs/zh/12-taos-sql/04-stable.md +++ b/docs/zh/12-taos-sql/04-stable.md @@ -1,6 +1,6 @@ --- -sidebar_label: 超级表管理 -title: 超级表 STable 管理 +sidebar_label: 超级表 +title: 超级表 description: 对超级表的各种管理操作 --- diff --git a/docs/zh/12-taos-sql/06-select.md b/docs/zh/12-taos-sql/06-select.md index d8ff3f04ed261ade9f8253d57a33e9c56adfefd6..0d38cb06359ee5c51a225b8769fae40a33f0ffd5 100644 --- a/docs/zh/12-taos-sql/06-select.md +++ b/docs/zh/12-taos-sql/06-select.md @@ -53,11 +53,6 @@ window_clause: { | STATE_WINDOW(col) | INTERVAL(interval_val [, interval_offset]) [SLIDING (sliding_val)] [WATERMARK(watermark_val)] [FILL(fill_mod_and_val)] -changes_option: { - DURATION duration_val - | ROWS rows_val -} - group_by_clause: GROUP BY expr [, expr] ... HAVING condition @@ -109,7 +104,7 @@ SELECT location, groupid, current FROM d1001 LIMIT 2; ### 结果去重 -`DISINTCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。 +`DISTINCT` 关键字可以对结果集中的一列或多列进行去重，去除的列既可以是标签列也可以是数据列。 对标签列去重： @@ -127,7 +122,6 @@ SELECT DISTINCT col_name [, col_name ...] FROM tb_name; 1. cfg 文件中的配置参数 maxNumOfDistinctRes 将对 DISTINCT 能够输出的数据行数进行限制。其最小值是 100000，最大值是 100000000，默认值是 10000000。如果实际计算结果超出了这个限制，那么会仅输出这个数量范围内的部分。 2. 由于浮点数天然的精度机制原因，在特定情况下，对 FLOAT 和 DOUBLE 列使用 DISTINCT 并不能保证输出值的完全唯一性。 -3. 在当前版本下，DISTINCT 不能在嵌套查询的子查询中使用，也不能与聚合函数、GROUP BY、或 JOIN 在同一条语句中混用。 ::: diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 9c5b7f771ecaf52da55a693ed5c789197ab57b05..86e9aaa80fdebecbd390ae231437491530d6c2b5 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -127,7 +127,7 @@ SELECT COS(field_name) FROM { tb_name | stb_name } [WHERE clause] SELECT FLOOR(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -**功能说明**：获得指定字段的向下取整数的结果。 +**功能说明**：获得指定字段的向下取整数的结果。 其他使用说明参见 CEIL 函数描述。 #### LOG @@ -174,7 +174,7 @@ SELECT POW(field_name, power) FROM { tb_name | stb_name } [WHERE clause] SELECT ROUND(field_name) FROM { tb_name | stb_name } [WHERE clause]; ``` -**功能说明**：获得指定字段的四舍五入的结果。 +**功能说明**：获得指定字段的四舍五入的结果。 其他使用说明参见 CEIL 函数描述。 @@ -435,7 +435,7 @@ SELECT TO_ISO8601(ts[, timezone]) FROM { tb_name | stb_name } [WHERE clause]; **使用说明**： - timezone 参数允许输入的时区格式为: [z/Z, +/-hhmm, +/-hh, +/-hh:mm]。例如，TO_ISO8601(1, "+00:00")。 -- 如果输入是表示 UNIX 时间戳的整形，返回格式精度由时间戳的位数决定; +- 如果输入是表示 UNIX 时间戳的整形，返回格式精度由时间戳的位数决定; - 如果输入是 TIMESTAMP 类型的列，返回格式的时间戳精度与当前 DATABASE 设置的时间精度一致。 @@ -770,14 +770,14 @@ SELECT HISTOGRAM(field_name，bin_type, bin_description, normalized) FROM tb_nam **详细说明**： - bin_type 用户指定的分桶类型, 有效输入类型为"user_input“, ”linear_bin", "log_bin"。 -- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： - - "user_input": "[1, 3, 5, 7]" +- bin_description 描述如何生成分桶区间，针对三种桶类型，分别为以下描述格式(均为 JSON 格式字符串)： + - "user_input": "[1, 3, 5, 7]" 用户指定 bin 的具体数值。 - + - "linear_bin": "{"start": 0.0, "width": 5.0, "count": 5, "infinity": true}" "start" 表示数据起始点，"width" 表示每次 bin 偏移量, "count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， 生成区间为[-inf, 0.0, 5.0, 10.0, 15.0, 20.0, +inf]。 - + - "log_bin": "{"start":1.0, "factor": 2.0, "count": 5, "infinity": true}" "start" 表示数据起始点，"factor" 表示按指数递增的因子，"count" 为 bin 的总数，"infinity" 表示是否添加（-inf, inf）作为区间起点和终点， 生成区间为[-inf, 1.0, 2.0, 4.0, 8.0, 16.0, +inf]。 @@ -918,7 +918,7 @@ SELECT MAX(field_name) FROM { tb_name | stb_name } [WHERE clause]; **返回数据类型**：同应用的字段。 -**适用数据类型**：数值类型，时间戳类型。 +**适用数据类型**：数值类型。 **适用于**：表和超级表。 @@ -933,7 +933,7 @@ SELECT MIN(field_name) FROM {tb_name | stb_name} [WHERE clause]; **返回数据类型**：同应用的字段。 -**适用数据类型**：数值类型，时间戳类型。 +**适用数据类型**：数值类型。 **适用于**：表和超级表。 @@ -969,7 +969,7 @@ SELECT SAMPLE(field_name, K) FROM { tb_name | stb_name } [WHERE clause] **适用于**：表和超级表。 -**使用说明**： +**使用说明**： - 不能参与表达式计算；该函数可以应用在普通表和超级表上； - 使用在超级表上的时候，需要搭配 PARTITION by tbname 使用，将结果强制规约到单个时间线。 @@ -1047,10 +1047,10 @@ SELECT CSUM(field_name) FROM { tb_name | stb_name } [WHERE clause] **适用于**：表和超级表。 -**使用说明**： - +**使用说明**： + - 不支持 +、-、*、/ 运算，如 csum(col1) + csum(col2)。 -- 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。 +- 只能与聚合（Aggregation）函数一起使用。 该函数可以应用在普通表和超级表上。 - 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。 @@ -1068,8 +1068,8 @@ SELECT DERIVATIVE(field_name, time_interval, ignore_negative) FROM tb_name [WHER **适用于**：表和超级表。 -**使用说明**: - +**使用说明**: + - DERIVATIVE 函数可以在由 PARTITION BY 划分出单独时间线的情况下用于超级表（也即 PARTITION BY tbname）。 - 可以与选择相关联的列一起使用。 例如: select \_rowts, DERIVATIVE() from。 @@ -1087,7 +1087,7 @@ SELECT {DIFF(field_name, ignore_negative) | DIFF(field_name)} FROM tb_name [WHER **适用于**：表和超级表。 -**使用说明**: +**使用说明**: - 输出结果行数是范围内总行数减一，第一行没有结果输出。 - 可以与选择相关联的列一起使用。 例如: select \_rowts, DIFF() from。 @@ -1124,9 +1124,9 @@ SELECT MAVG(field_name, K) FROM { tb_name | stb_name } [WHERE clause] **适用于**：表和超级表。 -**使用说明**： - -- 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); +**使用说明**： + +- 不支持 +、-、*、/ 运算，如 mavg(col1, k1) + mavg(col2, k1); - 只能与普通列，选择（Selection）、投影（Projection）函数一起使用，不能与聚合（Aggregation）函数一起使用； - 使用在超级表上的时候，需要搭配 PARTITION BY tbname使用，将结果强制规约到单个时间线。 diff --git a/docs/zh/12-taos-sql/12-distinguished.md b/docs/zh/12-taos-sql/12-distinguished.md index 016c1929fe5bd016f8327dc9fae587fe015786b8..268712e757304fe22848318befd16d1a93de5dac 100644 --- a/docs/zh/12-taos-sql/12-distinguished.md +++ b/docs/zh/12-taos-sql/12-distinguished.md @@ -1,6 +1,6 @@ --- -sidebar_label: 时序数据特色查询 -title: 时序数据特色查询 +sidebar_label: 特色查询 +title: 特色查询 description: TDengine 提供的时序数据特有的查询功能 --- diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index 86437c762e340811ce72c3c98b951b00ba6813e5..cd726e0a0ea644f575e16c656eeb4bb2cabf425d 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -44,7 +44,7 @@ window_clause: { ```sql CREATE STREAM avg_vol_s INTO avg_vol AS -SELECT _wstartts, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); +SELECT _wstart, count(*), avg(voltage) FROM meters PARTITION BY tbname INTERVAL(1m) SLIDING(30s); ``` ## 流式计算的 partition diff --git a/docs/zh/12-taos-sql/17-json.md b/docs/zh/12-taos-sql/17-json.md index 4cbd8eef364b1ea4e4285a34bb419a8ab3c7fc1d..18c25cfe230f81bf0b0e421634c1a768ae8e4628 100644 --- a/docs/zh/12-taos-sql/17-json.md +++ b/docs/zh/12-taos-sql/17-json.md @@ -1,6 +1,6 @@ --- -sidebar_label: JSON 类型使用说明 -title: JSON 类型使用说明 +sidebar_label: JSON 类型 +title: JSON 类型 description: 对 JSON 类型如何使用的详细说明 --- diff --git a/docs/zh/12-taos-sql/18-escape.md b/docs/zh/12-taos-sql/18-escape.md index 7e543743a30aeaa125375b14ad8baf49b634d248..5e0d292d396fdb54bd3df553544353a900415283 100644 --- a/docs/zh/12-taos-sql/18-escape.md +++ b/docs/zh/12-taos-sql/18-escape.md @@ -1,5 +1,5 @@ --- -title: 转义字符说明 +title: 转义字符 sidebar_label: 转义字符 description: TDengine 中使用转义字符的详细规则 --- diff --git a/docs/zh/12-taos-sql/19-limit.md b/docs/zh/12-taos-sql/19-limit.md index 0dbe00f80063bbc62cae38c540e3e7b6627d53d3..a9743adddabe96440ffca8c8585787081d29398f 100644 --- a/docs/zh/12-taos-sql/19-limit.md +++ b/docs/zh/12-taos-sql/19-limit.md @@ -1,6 +1,6 @@ --- -sidebar_label: 命名与边界限制 -title: 命名与边界限制 +sidebar_label: 命名与边界 +title: 命名与边界 description: 合法字符集和命名中的限制规则 --- diff --git a/docs/zh/12-taos-sql/20-keywords.md b/docs/zh/12-taos-sql/20-keywords.md index 047c6b08c9646927fc8ec16a2fd390569e4404fb..7530e803db44f49048f02e2ad6980a288c14fa12 100644 --- a/docs/zh/12-taos-sql/20-keywords.md +++ b/docs/zh/12-taos-sql/20-keywords.md @@ -1,12 +1,14 @@ --- sidebar_label: 保留关键字 -title: TDengine 保留关键字 +title: 保留关键字 description: TDengine 保留关键字的详细列表 --- ## 保留关键字 -目前 TDengine 有将近 200 个内部保留关键字，这些关键字无论大小写均不可以用作库名、表名、STable 名、数据列名及标签列名等。这些关键字列表如下： +目前 TDengine 有 200 多个内部保留关键字，这些关键字如果需要用作库名、表名、超级表名、子表名、数据列名及标签列名等，无论大小写，需要使用符号 `` ` `` 将关键字括起来使用，例如 \`ADD\`。 + +关键字列表如下： ### A @@ -15,15 +17,20 @@ description: TDengine 保留关键字的详细列表 - ACCOUNTS - ADD - AFTER +- AGGREGATE - ALL - ALTER +- ANALYZE - AND +- APPS - AS - ASC +- AT_ONCE - ATTACH ### B +- BALANCE - BEFORE - BEGIN - BETWEEN @@ -33,19 +40,27 @@ description: TDengine 保留关键字的详细列表 - BITNOT - BITOR - BLOCKS +- BNODE +- BNODES - BOOL +- BUFFER +- BUFSIZE - BY ### C - CACHE -- CACHELAST +- CACHEMODEL +- CACHESIZE - CASCADE +- CAST - CHANGE +- CLIENT_VERSION - CLUSTER - COLON - COLUMN - COMMA +- COMMENT - COMP - COMPACT - CONCAT @@ -53,15 +68,18 @@ description: TDengine 保留关键字的详细列表 - CONNECTION - CONNECTIONS - CONNS +- CONSUMER +- CONSUMERS +- CONTAINS - COPY +- COUNT - CREATE -- CTIME +- CURRENT_USER ### D - DATABASE - DATABASES -- DAYS - DBS - DEFERRED - DELETE @@ -70,18 +88,23 @@ description: TDengine 保留关键字的详细列表 - DESCRIBE - DETACH - DISTINCT +- DISTRIBUTED - DIVIDE - DNODE - DNODES - DOT - DOUBLE - DROP +- DURATION ### E +- EACH +- ENABLE - END -- EQ +- EVERY - EXISTS +- EXPIRED - EXPLAIN ### F @@ -89,18 +112,20 @@ description: TDengine 保留关键字的详细列表 - FAIL - FILE - FILL +- FIRST - FLOAT +- FLUSH - FOR - FROM -- FSYNC +- FUNCTION +- FUNCTIONS ### G -- GE - GLOB +- GRANT - GRANTS - GROUP -- GT ### H @@ -111,15 +136,18 @@ description: TDengine 保留关键字的详细列表 - ID - IF - IGNORE -- IMMEDIA +- IMMEDIATE - IMPORT - IN -- INITIAL +- INDEX +- INDEXES +- INITIALLY +- INNER - INSERT - INSTEAD - INT - INTEGER -- INTERVA +- INTERVAL - INTO - IS - ISNULL @@ -127,6 +155,7 @@ description: TDengine 保留关键字的详细列表 ### J - JOIN +- JSON ### K @@ -136,46 +165,57 @@ description: TDengine 保留关键字的详细列表 ### L -- LE +- LAST +- LAST_ROW +- LICENCES - LIKE - LIMIT - LINEAR - LOCAL -- LP -- LSHIFT -- LT ### M - MATCH +- MAX_DELAY - MAXROWS +- MERGE +- META - MINROWS - MINUS +- MNODE - MNODES - MODIFY - MODULES ### N -- NE +- NCHAR +- NEXT +- NMATCH - NONE - NOT - NOTNULL - NOW - NULL +- NULLS ### O - OF - OFFSET +- ON - OR - ORDER +- OUTPUTTYPE ### P -- PARTITION +- PAGES +- PAGESIZE +- PARTITIONS - PASS - PLUS +- PORT - PPS - PRECISION - PREV @@ -183,47 +223,63 @@ description: TDengine 保留关键字的详细列表 ### Q +- QNODE +- QNODES - QTIME -- QUERIE +- QUERIES - QUERY -- QUORUM ### R - RAISE -- REM +- RANGE +- RATIO +- READ +- REDISTRIBUTE +- RENAME - REPLACE - REPLICA - RESET -- RESTRIC +- RESTRICT +- RETENTIONS +- REVOKE +- ROLLUP - ROW -- RP -- RSHIFT ### S +- SCHEMALESS - SCORES - SELECT - SEMI +- SERVER_STATUS +- SERVER_VERSION - SESSION - SET - SHOW -- SLASH +- SINGLE_STABLE - SLIDING - SLIMIT -- SMALLIN +- SMA +- SMALLINT +- SNODE +- SNODES - SOFFSET -- STable -- STableS +- SPLIT +- STABLE +- STABLES - STAR - STATE -- STATEMEN -- STATE_WI +- STATE_WINDOW +- STATEMENT - STORAGE - STREAM - STREAMS +- STRICT - STRING +- SUBSCRIPTIONS - SYNCDB +- SYSINFO ### T @@ -234,19 +290,24 @@ description: TDengine 保留关键字的详细列表 - TBNAME - TIMES - TIMESTAMP +- TIMEZONE - TINYINT +- TO +- TODAY - TOPIC - TOPICS +- TRANSACTION +- TRANSACTIONS - TRIGGER +- TRIM - TSERIES +- TTL ### U -- UMINUS - UNION - UNSIGNED - UPDATE -- UPLUS - USE - USER - USERS @@ -254,9 +315,13 @@ description: TDengine 保留关键字的详细列表 ### V +- VALUE - VALUES +- VARCHAR - VARIABLE - VARIABLES +- VERBOSE +- VGROUP - VGROUPS - VIEW - VNODES @@ -264,14 +329,25 @@ description: TDengine 保留关键字的详细列表 ### W - WAL +- WAL_FSYNC_PERIOD +- WAL_LEVEL +- WAL_RETENTION_PERIOD +- WAL_RETENTION_SIZE +- WAL_ROLL_PERIOD +- WAL_SEGMENT_SIZE +- WATERMARK - WHERE +- WINDOW_CLOSE +- WITH +- WRITE ### \_ - \_C0 -- \_QSTART -- \_QSTOP - \_QDURATION -- \_WSTART -- \_WSTOP +- \_QEND +- \_QSTART +- \_ROWTS - \_WDURATION +- \_WEND +- \_WSTART diff --git a/docs/zh/12-taos-sql/22-meta.md b/docs/zh/12-taos-sql/22-meta.md index 3ae444e8fe692fee4f2331f1a01fb67899fe7930..c1ffc4a757500276f348d08cd577f63072dfece2 100644 --- a/docs/zh/12-taos-sql/22-meta.md +++ b/docs/zh/12-taos-sql/22-meta.md @@ -1,6 +1,6 @@ --- sidebar_label: 元数据 -title: 存储元数据的 Information_Schema 数据库 +title: 元数据 description: Information_Schema 数据库中存储了系统中所有的元数据信息 --- diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index 808d9ae31a35a215a07b57e29073e998beb1e25d..d4ee0e178c02e65eb3f1ceaa73e170893f65cc88 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -1,6 +1,6 @@ --- sidebar_label: 统计数据 -title: 存储统计数据的 Performance_Schema 数据库 +title: 统计数据 description: Performance_Schema 数据库中存储了系统中的各种统计信息 --- diff --git a/docs/zh/12-taos-sql/24-show.md b/docs/zh/12-taos-sql/24-show.md index b4aafdaa0af644e05e47106b76e0c7ab074a61b8..31b7c085a1ba97630223c16e06022ec9dfd9ea50 100644 --- a/docs/zh/12-taos-sql/24-show.md +++ b/docs/zh/12-taos-sql/24-show.md @@ -1,21 +1,11 @@ --- sidebar_label: SHOW 命令 -title: 使用 SHOW 命令查看系统元数据 +title: SHOW 命令 description: SHOW 命令的完整列表 --- SHOW 命令可以用来获取简要的系统信息。若想获取系统中详细的各种元数据、系统信息和状态，请使用 select 语句查询 INFORMATION_SCHEMA 数据库中的表。 -## SHOW ACCOUNTS - -```sql -SHOW ACCOUNTS; -``` - -显示当前系统中所有租户的信息。 - -注：企业版独有 - ## SHOW APPS ```sql diff --git a/docs/zh/12-taos-sql/26-udf.md b/docs/zh/12-taos-sql/26-udf.md index 764fde6e1f2e8aa38b90b4b8bc0131c9eaf44da6..6dc1b6eb5fbe346ae65993e4e290566179b0e6ee 100644 --- a/docs/zh/12-taos-sql/26-udf.md +++ b/docs/zh/12-taos-sql/26-udf.md @@ -1,6 +1,6 @@ --- sidebar_label: 自定义函数 -title: 用户自定义函数 +title: 自定义函数 description: 使用 UDF 的详细指南 --- diff --git a/docs/zh/12-taos-sql/27-index.md b/docs/zh/12-taos-sql/27-index.md index f88c6cf4ffe53ae19926e09c760bedd2997a952d..aa84140296832f79a6498d0da2b5a8f500cd1e90 100644 --- a/docs/zh/12-taos-sql/27-index.md +++ b/docs/zh/12-taos-sql/27-index.md @@ -1,6 +1,6 @@ --- sidebar_label: 索引 -title: 使用索引 +title: 索引 description: 索引功能的使用细节 --- diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index d653c59a5cd1309fbdcd6ef7e3706e33c4a43dee..e63825045d5ddc26d289af4bbd7fa808719bb99c 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -1,6 +1,6 @@ --- -sidebar_label: 3.0 版本语法变更 -title: 3.0 版本语法变更 +sidebar_label: 语法变更 +title: 语法变更 description: "TDengine 3.0 版本的语法变更说明" --- diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index ae0a496f03e8e545525fce49ae2394a10696c09c..7780dc2fe9a5742ce9d6855599ca080804a493b5 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -79,7 +79,7 @@ password = "taosdata" # 需要被监控的 taosAdapter [taosAdapter] -address = ["127.0.0.1:6041","192.168.1.95:6041"] +address = ["127.0.0.1:6041"] [metrics] # 监控指标前缀 @@ -92,7 +92,7 @@ cluster = "production" database = "log" # 指定需要监控的普通表 -tables = ["normal_table"] +tables = [] ``` ### 获取监控指标 @@ -141,4 +141,4 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 # HELP taos_cluster_info_first_ep # TYPE taos_cluster_info_first_ep gauge taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 -``` \ No newline at end of file +``` diff --git a/docs/zh/21-tdinternal/01-arch.md b/docs/zh/21-tdinternal/01-arch.md index 704524fd210152af34e15d248d3d4dbe050e4fef..b128b9d438371f22763b0ccea605e19d2dc9aa7b 100644 --- a/docs/zh/21-tdinternal/01-arch.md +++ b/docs/zh/21-tdinternal/01-arch.md @@ -26,7 +26,7 @@ TDengine 分布式架构的逻辑结构图如下： **管理节点（mnode）：** 一个虚拟的逻辑单元，负责所有数据节点运行状态的监控和维护，以及节点之间的负载均衡（图中 M）。同时，管理节点也负责元数据（包括用户、数据库、超级表等）的存储和管理，因此也称为 Meta Node。TDengine 集群中可配置多个（最多不超过 3 个）mnode，它们自动构建成为一个虚拟管理节点组（图中 M1，M2，M3）。mnode 支持多副本，采用 RAFT 一致性协议，保证系统的高可用与高可靠，任何数据更新操作只能在 Leader 上进行。mnode 集群的第一个节点在集群部署时自动完成，其他节点的创建与删除由用户通过 SQL 命令完成。每个 dnode 上至多有一个 mnode，由所属的数据节点的 EP 来唯一标识。每个 dnode 通过内部消息交互自动获取整个集群中所有 mnode 所在的 dnode 的 EP。 -**弹性计算节点（qnode）：** 一个虚拟的逻辑单元，运行查询计算任务，也包括基于系统表来实现的 show 命令（图中 Q）。集群中可配置多个 qnode，在整个集群内部共享使用（图中 Q1，Q2，Q3）。qnode 不与具体的 DB 绑定，即一个 qnode 可以同时执行多个 DB 的查询任务。每个 dnode 上至多有一个 qnode，由所属的数据节点的 EP 来唯一标识。客户端通过与 mnode 交互，获取可用的 qnode 列表，当没有可用的 qnode 时，计算任务在 vnode 中执行。 +**计算节点（qnode）：** 一个虚拟的逻辑单元，运行查询计算任务，也包括基于系统表来实现的 show 命令（图中 Q）。集群中可配置多个 qnode，在整个集群内部共享使用（图中 Q1，Q2，Q3）。qnode 不与具体的 DB 绑定，即一个 qnode 可以同时执行多个 DB 的查询任务。每个 dnode 上至多有一个 qnode，由所属的数据节点的 EP 来唯一标识。客户端通过与 mnode 交互，获取可用的 qnode 列表，当没有可用的 qnode 时，计算任务在 vnode 中执行。当一个查询执行时，依赖执行计划，调度器会安排一个或多个 qnode 来一起执行。qnode 能从 vnode 获取数据，也可以将自己的计算结果发给其他 qnode 做进一步的处理。通过引入独立的计算节点，TDengine 实现了存储和计算分离。 **流计算节点（snode）：** 一个虚拟的逻辑单元，只运行流计算任务（图中 S）。集群中可配置多个 snode，在整个集群内部共享使用（图中 S1，S2，S3）。snode 不与具体的 stream 绑定，即一个 snode 可以同时执行多个 stream 的计算任务。每个 dnode 上至多有一个 snode，由所属的数据节点的 EP 来唯一标识。由 mnode 调度可用的 snode 完成流计算任务，当没有可用的 snode 时，流计算任务在 vnode 中执行。 diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index e3e146313115fee12e539a161792234c2df671a5..a6ec560d3c5951ea1500893640be5905a31e8d61 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -6,11 +6,7 @@ description: TDengine 发布历史、Release Notes 及下载链接 import Release from "/components/ReleaseV3"; -## 3.0.0.1 +## 3.0.1.0 - - - + diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index 61129d74e57504286660a178f757cb816b75dbb5..9e8757cc4e09f661cfd4c552ff27537411e2d420 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -6,6 +6,6 @@ description: taosTools 的发布历史、Release Notes 和下载链接 import Release from "/components/ReleaseV3"; -## 2.1.2 +## 2.1.3 - \ No newline at end of file + diff --git a/include/common/taosdef.h b/include/common/taosdef.h index 9bfee56e2974832593578c9c2b1c984373763088..bf4de9d4ded1d0955bef05b1e3000be0bf34d8aa 100644 --- a/include/common/taosdef.h +++ b/include/common/taosdef.h @@ -65,13 +65,6 @@ typedef enum { TSDB_STATIS_NONE = 1, // statis part not exist } ETsdbStatisStatus; -typedef enum { - TSDB_SMA_STAT_UNKNOWN = -1, // unknown - TSDB_SMA_STAT_OK = 0, // ready to provide service - TSDB_SMA_STAT_EXPIRED = 1, // not ready or expired - TSDB_SMA_STAT_DROPPED = 2, // sma dropped -} ETsdbSmaStat; // bit operation - typedef enum { TSDB_SMA_TYPE_BLOCK = 0, // Block-wise SMA TSDB_SMA_TYPE_TIME_RANGE = 1, // Time-range-wise SMA diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 891c9ab040cfa6acdff55be1889a2bebe01ec2d3..ba4baa0130602da80da5eb14a45a48dea8a31a03 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -45,8 +45,8 @@ enum { // clang-format on typedef struct { - TSKEY ts; uint64_t groupId; + TSKEY ts; } SWinKey; static inline int SWinKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { @@ -68,6 +68,37 @@ static inline int SWinKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, i return 0; } +typedef struct { + uint64_t groupId; + TSKEY ts; + int32_t exprIdx; +} STupleKey; + +static inline int STupleKeyCmpr(const void* pKey1, int kLen1, const void* pKey2, int kLen2) { + STupleKey* pTuple1 = (STupleKey*)pKey1; + STupleKey* pTuple2 = (STupleKey*)pKey2; + + if (pTuple1->groupId > pTuple2->groupId) { + return 1; + } else if (pTuple1->groupId < pTuple2->groupId) { + return -1; + } + + if (pTuple1->ts > pTuple2->ts) { + return 1; + } else if (pTuple1->ts < pTuple2->ts) { + return -1; + } + + if (pTuple1->exprIdx > pTuple2->exprIdx) { + return 1; + } else if (pTuple1->exprIdx < pTuple2->exprIdx) { + return -1; + } + + return 0; +} + enum { TMQ_MSG_TYPE__DUMMY = 0, TMQ_MSG_TYPE__POLL_RSP, @@ -184,7 +215,6 @@ typedef struct SQueryTableDataCond { STimeWindow twindows; int64_t startVersion; int64_t endVersion; - int64_t schemaVersion; } SQueryTableDataCond; int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 410fa02ded3c16bd0e1fd2c669b5c8c46a7e1801..73d043b2d0ac680d69b517d042b02dfa71167435 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -184,7 +184,8 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows); +int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, + uint32_t numOfRows); int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int32_t* capacity, const SColumnInfoData* pSource, int32_t numOfRow2); int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, @@ -225,15 +226,16 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); int32_t blockDataKeepFirstNRows(SSDataBlock* pBlock, size_t n); -int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); -int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t assignOneDataBlock(SSDataBlock* dst, const SSDataBlock* src); +int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src); SSDataBlock* createDataBlock(); void* blockDataDestroy(SSDataBlock* pBlock); void blockDataFreeRes(SSDataBlock* pBlock); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createSpecialDataBlock(EStreamType type); -int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); SColumnInfoData* bdGetColumnInfoData(const SSDataBlock* pBlock, int32_t index); @@ -249,7 +251,6 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** dumpBuf); int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlocks, STSchema* pTSchema, int32_t vgId, tb_uid_t suid); - char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index df16f4f0ab9ad1a79c11ede9e54fdc086e9204df..5b495601756d246abd08a3ef07f5efa8a6b47e62 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -36,8 +36,13 @@ typedef struct STSRow2 STSRow2; typedef struct STSRowBuilder STSRowBuilder; typedef struct STagVal STagVal; typedef struct STag STag; +typedef struct SColData SColData; -// bitmap +#define HAS_NONE ((uint8_t)0x1) +#define HAS_NULL ((uint8_t)0x2) +#define HAS_VALUE ((uint8_t)0x4) + +// bitmap ================================ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, {0b00000000, 0b00000100, 0b00001000, 2}, {0b00000000, 0b00010000, 0b00100000, 4}, @@ -51,21 +56,21 @@ const static uint8_t BIT2_MAP[4][4] = {{0b00000000, 0b00000001, 0b00000010, 0}, #define SET_BIT2(p, i, v) ((p)[(i) >> 2] = (p)[(i) >> 2] & N1(BIT2_MAP[(i)&3][3]) | BIT2_MAP[(i)&3][(v)]) #define GET_BIT2(p, i) (((p)[(i) >> 2] >> BIT2_MAP[(i)&3][3]) & ((uint8_t)3)) -// STSchema +// STSchema ================================ int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t nCols, STSchema **ppTSchema); void tTSchemaDestroy(STSchema *pTSchema); -// SValue +// SValue ================================ int32_t tPutValue(uint8_t *p, SValue *pValue, int8_t type); int32_t tGetValue(uint8_t *p, SValue *pValue, int8_t type); int tValueCmprFn(const SValue *pValue1, const SValue *pValue2, int8_t type); -// SColVal +// SColVal ================================ #define COL_VAL_NONE(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNone = 1}) #define COL_VAL_NULL(CID, TYPE) ((SColVal){.cid = (CID), .type = (TYPE), .isNull = 1}) #define COL_VAL_VALUE(CID, TYPE, V) ((SColVal){.cid = (CID), .type = (TYPE), .value = (V)}) -// STSRow2 +// STSRow2 ================================ #define TSROW_LEN(PROW, V) tGetI32v((uint8_t *)(PROW)->data, (V) ? &(V) : NULL) #define TSROW_SVER(PROW, V) tGetI32v((PROW)->data + TSROW_LEN(PROW, NULL), (V) ? &(V) : NULL) @@ -77,7 +82,7 @@ int32_t tTSRowToArray(STSRow2 *pRow, STSchema *pTSchema, SArray **ppArray); int32_t tPutTSRow(uint8_t *p, STSRow2 *pRow); int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); -// STSRowBuilder +// STSRowBuilder ================================ #define tsRowBuilderInit() ((STSRowBuilder){0}) #define tsRowBuilderClear(B) \ do { \ @@ -86,7 +91,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow); } \ } while (0) -// STag +// STag ================================ int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); bool tTagIsJson(const void *pTag); @@ -100,7 +105,16 @@ void tTagSetCid(const STag *pTag, int16_t iTag, int16_t cid); void debugPrintSTag(STag *pTag, const char *tag, int32_t ln); // TODO: remove int32_t parseJsontoTagData(const char *json, SArray *pTagVals, STag **ppTag, void *pMsgBuf); -// STRUCT ================= +// SColData ================================ +void tColDataDestroy(void *ph); +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); +void tColDataClear(SColData *pColData); +int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); +void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal); +uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal); +int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); + +// STRUCT ================================ struct STColumn { col_id_t colId; int8_t type; @@ -166,6 +180,18 @@ struct SColVal { SValue value; }; +struct SColData { + int16_t cid; + int8_t type; + int8_t smaOn; + int32_t nVal; + uint8_t flag; + uint8_t *pBitMap; + int32_t *aOffset; + int32_t nData; + uint8_t *pData; +}; + #pragma pack(push, 1) struct STagVal { // char colName[TSDB_COL_NAME_LEN]; // only used for tmq_get_meta diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 4f041518d5a8ec23d9928e26e45acf9d3e3c9e28..85a81e1767ea9a489c6825ae58aa4c3ba05eee01 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -784,6 +784,10 @@ typedef struct { int64_t walRetentionSize; int32_t walRollPeriod; int64_t walSegmentSize; + int32_t sstTrigger; + int16_t hashPrefix; + int16_t hashSuffix; + int32_t tsdbPageSize; } SCreateDbReq; int32_t tSerializeSCreateDbReq(void* buf, int32_t bufLen, SCreateDbReq* pReq); @@ -805,6 +809,7 @@ typedef struct { int8_t strict; int8_t cacheLast; int8_t replications; + int32_t sstTrigger; } SAlterDbReq; int32_t tSerializeSAlterDbReq(void* buf, int32_t bufLen, SAlterDbReq* pReq); @@ -841,6 +846,8 @@ typedef struct { int64_t uid; int32_t vgVersion; int32_t vgNum; + int16_t hashPrefix; + int16_t hashSuffix; int8_t hashMethod; SArray* pVgroupInfos; // Array of SVgroupInfo } SUseDbRsp; @@ -1066,6 +1073,7 @@ typedef struct { typedef struct { int32_t vgId; int32_t syncState; + int64_t cacheUsage; int64_t numOfTables; int64_t numOfTimeSeries; int64_t totalStorage; @@ -1190,6 +1198,10 @@ typedef struct { int64_t walRetentionSize; int32_t walRollPeriod; int64_t walSegmentSize; + int16_t sstTrigger; + int16_t hashPrefix; + int16_t hashSuffix; + int32_t tsdbPageSize; } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); @@ -2078,9 +2090,9 @@ int32_t tDeserializeSVCreateTbBatchRsp(void* buf, int32_t bufLen, SVCreateTbBatc // TDMT_VND_DROP_TABLE ================= typedef struct { - char* name; - uint64_t suid; // for tmq in wal format - int8_t igNotExists; + char* name; + uint64_t suid; // for tmq in wal format + int8_t igNotExists; } SVDropTbReq; typedef struct { diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 006ba7f21bf0177c2b0104a51ef7908785cced2d..3f917ff0d1665d90de079dfc3eae884412ed0e7f 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -272,6 +272,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_SYNC_LEADER_TRANSFER, "sync-leader-transfer", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SET_MNODE_STANDBY, "set-mnode-standby", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_SET_VNODE_STANDBY, "set-vnode-standby", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT, "sync-heartbeat", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_SYNC_HEARTBEAT_REPLY, "sync-heartbeat-reply", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_SYNC_MAX_MSG, "sync-max", NULL, NULL) #if defined(TD_MSG_NUMBER_) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index b38ec664358d08622c06d8f941873b59e43b9455..3e170d5098ab34b1dc41c6d87d74484eff503fbc 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -89,237 +89,241 @@ #define TK_KEEP 71 #define TK_PAGES 72 #define TK_PAGESIZE 73 -#define TK_PRECISION 74 -#define TK_REPLICA 75 -#define TK_STRICT 76 -#define TK_VGROUPS 77 -#define TK_SINGLE_STABLE 78 -#define TK_RETENTIONS 79 -#define TK_SCHEMALESS 80 -#define TK_WAL_LEVEL 81 -#define TK_WAL_FSYNC_PERIOD 82 -#define TK_WAL_RETENTION_PERIOD 83 -#define TK_WAL_RETENTION_SIZE 84 -#define TK_WAL_ROLL_PERIOD 85 -#define TK_WAL_SEGMENT_SIZE 86 -#define TK_NK_COLON 87 -#define TK_TABLE 88 -#define TK_NK_LP 89 -#define TK_NK_RP 90 -#define TK_STABLE 91 -#define TK_ADD 92 -#define TK_COLUMN 93 -#define TK_MODIFY 94 -#define TK_RENAME 95 -#define TK_TAG 96 -#define TK_SET 97 -#define TK_NK_EQ 98 -#define TK_USING 99 -#define TK_TAGS 100 -#define TK_COMMENT 101 -#define TK_BOOL 102 -#define TK_TINYINT 103 -#define TK_SMALLINT 104 -#define TK_INT 105 -#define TK_INTEGER 106 -#define TK_BIGINT 107 -#define TK_FLOAT 108 -#define TK_DOUBLE 109 -#define TK_BINARY 110 -#define TK_TIMESTAMP 111 -#define TK_NCHAR 112 -#define TK_UNSIGNED 113 -#define TK_JSON 114 -#define TK_VARCHAR 115 -#define TK_MEDIUMBLOB 116 -#define TK_BLOB 117 -#define TK_VARBINARY 118 -#define TK_DECIMAL 119 -#define TK_MAX_DELAY 120 -#define TK_WATERMARK 121 -#define TK_ROLLUP 122 -#define TK_TTL 123 -#define TK_SMA 124 -#define TK_FIRST 125 -#define TK_LAST 126 -#define TK_SHOW 127 -#define TK_DATABASES 128 -#define TK_TABLES 129 -#define TK_STABLES 130 -#define TK_MNODES 131 -#define TK_MODULES 132 -#define TK_QNODES 133 -#define TK_FUNCTIONS 134 -#define TK_INDEXES 135 -#define TK_ACCOUNTS 136 -#define TK_APPS 137 -#define TK_CONNECTIONS 138 -#define TK_LICENCES 139 -#define TK_GRANTS 140 -#define TK_QUERIES 141 -#define TK_SCORES 142 -#define TK_TOPICS 143 -#define TK_VARIABLES 144 -#define TK_BNODES 145 -#define TK_SNODES 146 -#define TK_CLUSTER 147 -#define TK_TRANSACTIONS 148 -#define TK_DISTRIBUTED 149 -#define TK_CONSUMERS 150 -#define TK_SUBSCRIPTIONS 151 -#define TK_LIKE 152 -#define TK_INDEX 153 -#define TK_FUNCTION 154 -#define TK_INTERVAL 155 -#define TK_TOPIC 156 -#define TK_AS 157 -#define TK_WITH 158 -#define TK_META 159 -#define TK_CONSUMER 160 -#define TK_GROUP 161 -#define TK_DESC 162 -#define TK_DESCRIBE 163 -#define TK_RESET 164 -#define TK_QUERY 165 -#define TK_CACHE 166 -#define TK_EXPLAIN 167 -#define TK_ANALYZE 168 -#define TK_VERBOSE 169 -#define TK_NK_BOOL 170 -#define TK_RATIO 171 -#define TK_NK_FLOAT 172 -#define TK_OUTPUTTYPE 173 -#define TK_AGGREGATE 174 -#define TK_BUFSIZE 175 -#define TK_STREAM 176 -#define TK_INTO 177 -#define TK_TRIGGER 178 -#define TK_AT_ONCE 179 -#define TK_WINDOW_CLOSE 180 -#define TK_IGNORE 181 -#define TK_EXPIRED 182 -#define TK_KILL 183 -#define TK_CONNECTION 184 -#define TK_TRANSACTION 185 -#define TK_BALANCE 186 -#define TK_VGROUP 187 -#define TK_MERGE 188 -#define TK_REDISTRIBUTE 189 -#define TK_SPLIT 190 -#define TK_DELETE 191 -#define TK_INSERT 192 -#define TK_NULL 193 -#define TK_NK_QUESTION 194 -#define TK_NK_ARROW 195 -#define TK_ROWTS 196 -#define TK_TBNAME 197 -#define TK_QSTART 198 -#define TK_QEND 199 -#define TK_QDURATION 200 -#define TK_WSTART 201 -#define TK_WEND 202 -#define TK_WDURATION 203 -#define TK_CAST 204 -#define TK_NOW 205 -#define TK_TODAY 206 -#define TK_TIMEZONE 207 -#define TK_CLIENT_VERSION 208 -#define TK_SERVER_VERSION 209 -#define TK_SERVER_STATUS 210 -#define TK_CURRENT_USER 211 -#define TK_COUNT 212 -#define TK_LAST_ROW 213 -#define TK_BETWEEN 214 -#define TK_IS 215 -#define TK_NK_LT 216 -#define TK_NK_GT 217 -#define TK_NK_LE 218 -#define TK_NK_GE 219 -#define TK_NK_NE 220 -#define TK_MATCH 221 -#define TK_NMATCH 222 -#define TK_CONTAINS 223 -#define TK_IN 224 -#define TK_JOIN 225 -#define TK_INNER 226 -#define TK_SELECT 227 -#define TK_DISTINCT 228 -#define TK_WHERE 229 -#define TK_PARTITION 230 -#define TK_BY 231 -#define TK_SESSION 232 -#define TK_STATE_WINDOW 233 -#define TK_SLIDING 234 -#define TK_FILL 235 -#define TK_VALUE 236 -#define TK_NONE 237 -#define TK_PREV 238 -#define TK_LINEAR 239 -#define TK_NEXT 240 -#define TK_HAVING 241 -#define TK_RANGE 242 -#define TK_EVERY 243 -#define TK_ORDER 244 -#define TK_SLIMIT 245 -#define TK_SOFFSET 246 -#define TK_LIMIT 247 -#define TK_OFFSET 248 -#define TK_ASC 249 -#define TK_NULLS 250 -#define TK_ABORT 251 -#define TK_AFTER 252 -#define TK_ATTACH 253 -#define TK_BEFORE 254 -#define TK_BEGIN 255 -#define TK_BITAND 256 -#define TK_BITNOT 257 -#define TK_BITOR 258 -#define TK_BLOCKS 259 -#define TK_CHANGE 260 -#define TK_COMMA 261 -#define TK_COMPACT 262 -#define TK_CONCAT 263 -#define TK_CONFLICT 264 -#define TK_COPY 265 -#define TK_DEFERRED 266 -#define TK_DELIMITERS 267 -#define TK_DETACH 268 -#define TK_DIVIDE 269 -#define TK_DOT 270 -#define TK_EACH 271 -#define TK_END 272 -#define TK_FAIL 273 -#define TK_FILE 274 -#define TK_FOR 275 -#define TK_GLOB 276 -#define TK_ID 277 -#define TK_IMMEDIATE 278 -#define TK_IMPORT 279 -#define TK_INITIALLY 280 -#define TK_INSTEAD 281 -#define TK_ISNULL 282 -#define TK_KEY 283 -#define TK_NK_BITNOT 284 -#define TK_NK_SEMI 285 -#define TK_NOTNULL 286 -#define TK_OF 287 -#define TK_PLUS 288 -#define TK_PRIVILEGE 289 -#define TK_RAISE 290 -#define TK_REPLACE 291 -#define TK_RESTRICT 292 -#define TK_ROW 293 -#define TK_SEMI 294 -#define TK_STAR 295 -#define TK_STATEMENT 296 -#define TK_STRING 297 -#define TK_TIMES 298 -#define TK_UPDATE 299 -#define TK_VALUES 300 -#define TK_VARIABLE 301 -#define TK_VIEW 302 -#define TK_VNODES 303 -#define TK_WAL 304 +#define TK_TSDB_PAGESIZE 74 +#define TK_PRECISION 75 +#define TK_REPLICA 76 +#define TK_STRICT 77 +#define TK_VGROUPS 78 +#define TK_SINGLE_STABLE 79 +#define TK_RETENTIONS 80 +#define TK_SCHEMALESS 81 +#define TK_WAL_LEVEL 82 +#define TK_WAL_FSYNC_PERIOD 83 +#define TK_WAL_RETENTION_PERIOD 84 +#define TK_WAL_RETENTION_SIZE 85 +#define TK_WAL_ROLL_PERIOD 86 +#define TK_WAL_SEGMENT_SIZE 87 +#define TK_STT_TRIGGER 88 +#define TK_TABLE_PREFIX 89 +#define TK_TABLE_SUFFIX 90 +#define TK_NK_COLON 91 +#define TK_TABLE 92 +#define TK_NK_LP 93 +#define TK_NK_RP 94 +#define TK_STABLE 95 +#define TK_ADD 96 +#define TK_COLUMN 97 +#define TK_MODIFY 98 +#define TK_RENAME 99 +#define TK_TAG 100 +#define TK_SET 101 +#define TK_NK_EQ 102 +#define TK_USING 103 +#define TK_TAGS 104 +#define TK_COMMENT 105 +#define TK_BOOL 106 +#define TK_TINYINT 107 +#define TK_SMALLINT 108 +#define TK_INT 109 +#define TK_INTEGER 110 +#define TK_BIGINT 111 +#define TK_FLOAT 112 +#define TK_DOUBLE 113 +#define TK_BINARY 114 +#define TK_TIMESTAMP 115 +#define TK_NCHAR 116 +#define TK_UNSIGNED 117 +#define TK_JSON 118 +#define TK_VARCHAR 119 +#define TK_MEDIUMBLOB 120 +#define TK_BLOB 121 +#define TK_VARBINARY 122 +#define TK_DECIMAL 123 +#define TK_MAX_DELAY 124 +#define TK_WATERMARK 125 +#define TK_ROLLUP 126 +#define TK_TTL 127 +#define TK_SMA 128 +#define TK_FIRST 129 +#define TK_LAST 130 +#define TK_SHOW 131 +#define TK_DATABASES 132 +#define TK_TABLES 133 +#define TK_STABLES 134 +#define TK_MNODES 135 +#define TK_MODULES 136 +#define TK_QNODES 137 +#define TK_FUNCTIONS 138 +#define TK_INDEXES 139 +#define TK_ACCOUNTS 140 +#define TK_APPS 141 +#define TK_CONNECTIONS 142 +#define TK_LICENCES 143 +#define TK_GRANTS 144 +#define TK_QUERIES 145 +#define TK_SCORES 146 +#define TK_TOPICS 147 +#define TK_VARIABLES 148 +#define TK_BNODES 149 +#define TK_SNODES 150 +#define TK_CLUSTER 151 +#define TK_TRANSACTIONS 152 +#define TK_DISTRIBUTED 153 +#define TK_CONSUMERS 154 +#define TK_SUBSCRIPTIONS 155 +#define TK_VNODES 156 +#define TK_LIKE 157 +#define TK_INDEX 158 +#define TK_FUNCTION 159 +#define TK_INTERVAL 160 +#define TK_TOPIC 161 +#define TK_AS 162 +#define TK_WITH 163 +#define TK_META 164 +#define TK_CONSUMER 165 +#define TK_GROUP 166 +#define TK_DESC 167 +#define TK_DESCRIBE 168 +#define TK_RESET 169 +#define TK_QUERY 170 +#define TK_CACHE 171 +#define TK_EXPLAIN 172 +#define TK_ANALYZE 173 +#define TK_VERBOSE 174 +#define TK_NK_BOOL 175 +#define TK_RATIO 176 +#define TK_NK_FLOAT 177 +#define TK_OUTPUTTYPE 178 +#define TK_AGGREGATE 179 +#define TK_BUFSIZE 180 +#define TK_STREAM 181 +#define TK_INTO 182 +#define TK_TRIGGER 183 +#define TK_AT_ONCE 184 +#define TK_WINDOW_CLOSE 185 +#define TK_IGNORE 186 +#define TK_EXPIRED 187 +#define TK_KILL 188 +#define TK_CONNECTION 189 +#define TK_TRANSACTION 190 +#define TK_BALANCE 191 +#define TK_VGROUP 192 +#define TK_MERGE 193 +#define TK_REDISTRIBUTE 194 +#define TK_SPLIT 195 +#define TK_DELETE 196 +#define TK_INSERT 197 +#define TK_NULL 198 +#define TK_NK_QUESTION 199 +#define TK_NK_ARROW 200 +#define TK_ROWTS 201 +#define TK_TBNAME 202 +#define TK_QSTART 203 +#define TK_QEND 204 +#define TK_QDURATION 205 +#define TK_WSTART 206 +#define TK_WEND 207 +#define TK_WDURATION 208 +#define TK_CAST 209 +#define TK_NOW 210 +#define TK_TODAY 211 +#define TK_TIMEZONE 212 +#define TK_CLIENT_VERSION 213 +#define TK_SERVER_VERSION 214 +#define TK_SERVER_STATUS 215 +#define TK_CURRENT_USER 216 +#define TK_COUNT 217 +#define TK_LAST_ROW 218 +#define TK_BETWEEN 219 +#define TK_IS 220 +#define TK_NK_LT 221 +#define TK_NK_GT 222 +#define TK_NK_LE 223 +#define TK_NK_GE 224 +#define TK_NK_NE 225 +#define TK_MATCH 226 +#define TK_NMATCH 227 +#define TK_CONTAINS 228 +#define TK_IN 229 +#define TK_JOIN 230 +#define TK_INNER 231 +#define TK_SELECT 232 +#define TK_DISTINCT 233 +#define TK_WHERE 234 +#define TK_PARTITION 235 +#define TK_BY 236 +#define TK_SESSION 237 +#define TK_STATE_WINDOW 238 +#define TK_SLIDING 239 +#define TK_FILL 240 +#define TK_VALUE 241 +#define TK_NONE 242 +#define TK_PREV 243 +#define TK_LINEAR 244 +#define TK_NEXT 245 +#define TK_HAVING 246 +#define TK_RANGE 247 +#define TK_EVERY 248 +#define TK_ORDER 249 +#define TK_SLIMIT 250 +#define TK_SOFFSET 251 +#define TK_LIMIT 252 +#define TK_OFFSET 253 +#define TK_ASC 254 +#define TK_NULLS 255 +#define TK_ABORT 256 +#define TK_AFTER 257 +#define TK_ATTACH 258 +#define TK_BEFORE 259 +#define TK_BEGIN 260 +#define TK_BITAND 261 +#define TK_BITNOT 262 +#define TK_BITOR 263 +#define TK_BLOCKS 264 +#define TK_CHANGE 265 +#define TK_COMMA 266 +#define TK_COMPACT 267 +#define TK_CONCAT 268 +#define TK_CONFLICT 269 +#define TK_COPY 270 +#define TK_DEFERRED 271 +#define TK_DELIMITERS 272 +#define TK_DETACH 273 +#define TK_DIVIDE 274 +#define TK_DOT 275 +#define TK_EACH 276 +#define TK_END 277 +#define TK_FAIL 278 +#define TK_FILE 279 +#define TK_FOR 280 +#define TK_GLOB 281 +#define TK_ID 282 +#define TK_IMMEDIATE 283 +#define TK_IMPORT 284 +#define TK_INITIALLY 285 +#define TK_INSTEAD 286 +#define TK_ISNULL 287 +#define TK_KEY 288 +#define TK_NK_BITNOT 289 +#define TK_NK_SEMI 290 +#define TK_NOTNULL 291 +#define TK_OF 292 +#define TK_PLUS 293 +#define TK_PRIVILEGE 294 +#define TK_RAISE 295 +#define TK_REPLACE 296 +#define TK_RESTRICT 297 +#define TK_ROW 298 +#define TK_SEMI 299 +#define TK_STAR 300 +#define TK_STATEMENT 301 +#define TK_STRING 302 +#define TK_TIMES 303 +#define TK_UPDATE 304 +#define TK_VALUES 305 +#define TK_VARIABLE 306 +#define TK_VIEW 307 +#define TK_WAL 308 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 3f26eee86ad3f1b4666c55283ad346f60a7b4f31..65ddc180d66e6ab8c56aec68ce2fb5064c104846 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -34,66 +34,69 @@ typedef struct SFuncExecEnv { int32_t calcMemSize; } SFuncExecEnv; -typedef bool (*FExecGetEnv)(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); -typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); +typedef bool (*FExecGetEnv)(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); +typedef bool (*FExecInit)(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResultCellInfo); typedef int32_t (*FExecProcess)(struct SqlFunctionCtx *pCtx); -typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock* pBlock); +typedef int32_t (*FExecFinalize)(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); typedef int32_t (*FScalarExecProcess)(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput); typedef int32_t (*FExecCombine)(struct SqlFunctionCtx *pDestCtx, struct SqlFunctionCtx *pSourceCtx); typedef struct SScalarFuncExecFuncs { - FExecGetEnv getEnv; + FExecGetEnv getEnv; FScalarExecProcess process; } SScalarFuncExecFuncs; typedef struct SFuncExecFuncs { - FExecGetEnv getEnv; - FExecInit init; - FExecProcess process; + FExecGetEnv getEnv; + FExecInit init; + FExecProcess process; FExecFinalize finalize; - FExecCombine combine; + FExecCombine combine; } SFuncExecFuncs; -#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results +#define MAX_INTERVAL_TIME_WINDOW 1000000 // maximum allowed time windows in final results #define TOP_BOTTOM_QUERY_LIMIT 100 #define FUNCTIONS_NAME_MAX_LENGTH 16 typedef struct SResultRowEntryInfo { - bool initialized:1; // output buffer has been initialized - bool complete:1; // query has completed - uint8_t isNullRes:6; // the result is null - uint16_t numOfRes; // num of output result in current buffer. NOT NULL RESULT + bool initialized : 1; // output buffer has been initialized + bool complete : 1; // query has completed + uint8_t isNullRes : 6; // the result is null + uint16_t numOfRes; // num of output result in current buffer. NOT NULL RESULT } SResultRowEntryInfo; // determine the real data need to calculated the result enum { - BLK_DATA_NOT_LOAD = 0x0, - BLK_DATA_SMA_LOAD = 0x1, + BLK_DATA_NOT_LOAD = 0x0, + BLK_DATA_SMA_LOAD = 0x1, BLK_DATA_DATA_LOAD = 0x3, - BLK_DATA_FILTEROUT = 0x4, // discard current data block since it is not qualified for filter + BLK_DATA_FILTEROUT = 0x4, // discard current data block since it is not qualified for filter }; enum { - MAIN_SCAN = 0x0u, - REVERSE_SCAN = 0x1u, // todo remove it - REPEAT_SCAN = 0x2u, //repeat scan belongs to the master scan - MERGE_STAGE = 0x20u, + MAIN_SCAN = 0x0u, + REVERSE_SCAN = 0x1u, // todo remove it + REPEAT_SCAN = 0x2u, // repeat scan belongs to the master scan + MERGE_STAGE = 0x20u, }; typedef struct SPoint1 { - int64_t key; - union{double val; char* ptr;}; + int64_t key; + union { + double val; + char *ptr; + }; } SPoint1; struct SqlFunctionCtx; struct SResultRowEntryInfo; -//for selectivity query, the corresponding tag value is assigned if the data is qualified +// for selectivity query, the corresponding tag value is assigned if the data is qualified typedef struct SSubsidiaryResInfo { - int16_t num; - int32_t rowLen; - char* buf; // serialize data buffer + int16_t num; + int32_t rowLen; + char *buf; // serialize data buffer struct SqlFunctionCtx **pCtx; } SSubsidiaryResInfo; @@ -106,69 +109,70 @@ typedef struct SResultDataInfo { } SResultDataInfo; #define GET_RES_INFO(ctx) ((ctx)->resultInfo) -#define GET_ROWCELL_INTERBUF(_c) ((void*) ((char*)(_c) + sizeof(SResultRowEntryInfo))) +#define GET_ROWCELL_INTERBUF(_c) ((void *)((char *)(_c) + sizeof(SResultRowEntryInfo))) typedef struct SInputColumnInfoData { - int32_t totalRows; // total rows in current columnar data - int32_t startRowIndex; // handle started row index - int32_t numOfRows; // the number of rows needs to be handled - int32_t numOfInputCols; // PTS is not included - bool colDataAggIsSet;// if agg is set or not - SColumnInfoData *pPTS; // primary timestamp column + int32_t totalRows; // total rows in current columnar data + int32_t startRowIndex; // handle started row index + int32_t numOfRows; // the number of rows needs to be handled + int32_t numOfInputCols; // PTS is not included + bool colDataAggIsSet; // if agg is set or not + SColumnInfoData *pPTS; // primary timestamp column SColumnInfoData **pData; SColumnDataAgg **pColumnDataAgg; - uint64_t uid; // table uid, used to set the tag value when building the final query result for selectivity functions. + uint64_t uid; // table uid, used to set the tag value when building the final query result for selectivity functions. } SInputColumnInfoData; typedef struct SSerializeDataHandle { - struct SDiskbasedBuf* pBuf; + struct SDiskbasedBuf *pBuf; int32_t currentPage; + void *pState; } SSerializeDataHandle; // sql function runtime context typedef struct SqlFunctionCtx { - SInputColumnInfoData input; - SResultDataInfo resDataInfo; - uint32_t order; // data block scanner order: asc|desc - uint8_t scanFlag; // record current running step, default: 0 - int16_t functionId; // function id - char *pOutput; // final result output buffer, point to sdata->data - int32_t numOfParams; - SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param - SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ - int32_t offset; - struct SResultRowEntryInfo *resultInfo; - SSubsidiaryResInfo subsidiaries; - SPoint1 start; - SPoint1 end; - SFuncExecFuncs fpSet; - SScalarFuncExecFuncs sfp; - struct SExprInfo *pExpr; - struct SSDataBlock *pSrcBlock; - struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity - SSerializeDataHandle saveHandle; - bool isStream; - - char udfName[TSDB_FUNC_NAME_LEN]; + SInputColumnInfoData input; + SResultDataInfo resDataInfo; + uint32_t order; // data block scanner order: asc|desc + uint8_t scanFlag; // record current running step, default: 0 + int16_t functionId; // function id + char *pOutput; // final result output buffer, point to sdata->data + int32_t numOfParams; + SFunctParam *param; // input parameter, e.g., top(k, 20), the number of results for top query is kept in param + SColumnInfoData *pTsOutput; // corresponding output buffer for timestamp of each result, e.g., top/bottom*/ + int32_t offset; + struct SResultRowEntryInfo *resultInfo; + SSubsidiaryResInfo subsidiaries; + SPoint1 start; + SPoint1 end; + SFuncExecFuncs fpSet; + SScalarFuncExecFuncs sfp; + struct SExprInfo *pExpr; + struct SSDataBlock *pSrcBlock; + struct SSDataBlock *pDstBlock; // used by indefinite rows function to set selectivity + SSerializeDataHandle saveHandle; + bool isStream; + + char udfName[TSDB_FUNC_NAME_LEN]; } SqlFunctionCtx; enum { - TEXPR_BINARYEXPR_NODE= 0x1, + TEXPR_BINARYEXPR_NODE = 0x1, TEXPR_UNARYEXPR_NODE = 0x2, }; typedef struct tExprNode { int32_t nodeType; union { - struct {// function node - char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor - int32_t functionId; - int32_t num; - struct SFunctionNode *pFunctNode; + struct { // function node + char functionName[FUNCTIONS_NAME_MAX_LENGTH]; // todo refactor + int32_t functionId; + int32_t num; + struct SFunctionNode *pFunctNode; } _function; struct { - struct SNode* pRootNode; + struct SNode *pRootNode; } _optrRoot; }; } tExprNode; @@ -182,17 +186,18 @@ struct SScalarParam { int32_t numOfRows; }; -void cleanupResultRowEntry(struct SResultRowEntryInfo* pCell); -int32_t getNumOfResult(SqlFunctionCtx* pCtx, int32_t num, SSDataBlock* pResBlock); -bool isRowEntryCompleted(struct SResultRowEntryInfo* pEntry); -bool isRowEntryInitialized(struct SResultRowEntryInfo* pEntry); +void cleanupResultRowEntry(struct SResultRowEntryInfo *pCell); +int32_t getNumOfResult(SqlFunctionCtx *pCtx, int32_t num, SSDataBlock *pResBlock); +bool isRowEntryCompleted(struct SResultRowEntryInfo *pEntry); +bool isRowEntryInitialized(struct SResultRowEntryInfo *pEntry); typedef struct SPoint { int64_t key; - void * val; + void *val; } SPoint; -int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* point1, SPoint* point2, int32_t inputType); +int32_t taosGetLinearInterpolationVal(SPoint *point, int32_t outputType, SPoint *point1, SPoint *point2, + int32_t inputType); /////////////////////////////////////////////////////////////////////////////////////////////////////////////////////// // udf api diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 3223d4cdb8dfd36284f3d36922451166226fdd3e..22e92b2e8031d4b2d604433f9f32db0bc3754b4f 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -64,6 +64,7 @@ typedef struct SDatabaseOptions { int64_t keep[3]; int32_t pages; int32_t pagesize; + int32_t tsdbPageSize; char precisionStr[3]; int8_t precision; int8_t replica; @@ -78,6 +79,12 @@ typedef struct SDatabaseOptions { int32_t walRetentionSize; int32_t walRollPeriod; int32_t walSegmentSize; + bool walRetentionPeriodIsSet; + bool walRetentionSizeIsSet; + bool walRollPeriodIsSet; + int32_t sstTrigger; + int32_t tablePrefix; + int32_t tableSuffix; } SDatabaseOptions; typedef struct SCreateDatabaseStmt { @@ -268,6 +275,12 @@ typedef struct SShowDnodeVariablesStmt { SNode* pDnodeId; } SShowDnodeVariablesStmt; +typedef struct SShowVnodesStmt { + ENodeType type; + SNode* pDnodeId; + SNode* pDnodeEndpoint; +} SShowVnodesStmt; + typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType; typedef struct SIndexOptions { diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 5743d3360857dab460841d89e50360ba53d36b39..6500d3d1831e817c497406e574b721594e63e209 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -183,12 +183,12 @@ typedef enum ENodeType { QUERY_NODE_SHOW_DNODE_VARIABLES_STMT, QUERY_NODE_SHOW_TRANSACTIONS_STMT, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT, + QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT, QUERY_NODE_SHOW_CREATE_STABLE_STMT, QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, - QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_SCORES_STMT, QUERY_NODE_KILL_CONNECTION_STMT, QUERY_NODE_KILL_QUERY_STMT, @@ -244,6 +244,7 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE, QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, QUERY_NODE_PHYSICAL_PLAN_PARTITION, + QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION, QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC, QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC, QUERY_NODE_PHYSICAL_PLAN_DISPATCH, @@ -319,6 +320,9 @@ int32_t nodesStringToNode(const char* pStr, SNode** pNode); int32_t nodesListToString(const SNodeList* pList, bool format, char** pStr, int32_t* pLen); int32_t nodesStringToList(const char* pStr, SNodeList** pList); +int32_t nodesNodeToMsg(const SNode* pNode, char** pMsg, int32_t* pLen); +int32_t nodesMsgToNode(const char* pStr, int32_t len, SNode** pNode); + int32_t nodesNodeToSQL(SNode* pNode, char* buf, int32_t bufSize, int32_t* len); char* nodesGetNameFromColumnNode(SNode* pNode); int32_t nodesGetOutputNumFromSlotList(SNodeList* pSlots); diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index 5abad5cccc87b8c3caea4d4bb64fe1c49c553814..07994d65d262eba7cabb4c575e8d77b5efa69245 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -151,6 +151,8 @@ typedef struct SVnodeModifyLogicNode { SArray* pDataBlocks; SVgDataBlocks* pVgDataBlocks; SNode* pAffectedRows; // SColumnNode + SNode* pStartTs; // SColumnNode + SNode* pEndTs; // SColumnNode uint64_t tableId; uint64_t stableId; int8_t tableType; // table type @@ -489,6 +491,8 @@ typedef struct SPartitionPhysiNode { SNodeList* pTargets; } SPartitionPhysiNode; +typedef SPartitionPhysiNode SStreamPartitionPhysiNode; + typedef struct SDataSinkNode { ENodeType type; SDataBlockDescNode* pInputDataBlockDesc; @@ -524,6 +528,8 @@ typedef struct SDataDeleterNode { char tsColName[TSDB_COL_NAME_LEN]; STimeWindow deleteTimeRange; SNode* pAffectedRows; + SNode* pStartTs; + SNode* pEndTs; } SDataDeleterNode; typedef struct SSubplan { diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 3a1eaf289e4ba245544b985e893f746845c37c88..e90c994e8f2770fa95da82c71f5cc9ba16b1425f 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -315,6 +315,8 @@ typedef struct SDeleteStmt { SNode* pFromTable; // FROM clause SNode* pWhere; // WHERE clause SNode* pCountFunc; // count the number of rows affected + SNode* pFirstFunc; // the start timestamp when the data was actually deleted + SNode* pLastFunc; // the end timestamp when the data was actually deleted SNode* pTagCond; // pWhere divided into pTagCond and timeRange STimeWindow timeRange; uint8_t precision; diff --git a/include/libs/planner/planner.h b/include/libs/planner/planner.h index 05caa7a7bb56617ef34c03e3646f85ac98f65a56..e03ac3811a11b3927531a6250f5b41fb876c0f1c 100644 --- a/include/libs/planner/planner.h +++ b/include/libs/planner/planner.h @@ -52,10 +52,14 @@ int32_t qSetSubplanExecutionNode(SSubplan* pSubplan, int32_t groupId, SDownstrea void qClearSubplanExecutionNode(SSubplan* pSubplan); -// Convert to subplan to string for the scheduler to send to the executor +// Convert to subplan to display string for the scheduler to send to the executor int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen); int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan); +// Convert to subplan to msg for the scheduler to send to the executor +int32_t qSubPlanToMsg(const SSubplan* pSubplan, char** pStr, int32_t* pLen); +int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan); + char* qQueryPlanToString(const SQueryPlan* pPlan); SQueryPlan* qStringToQueryPlan(const char* pStr); diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index bf89e98ce37fd53301e1a3dcd28284a1ed44208a..e9f3864f6738bc7714db49d8f2373900fe10bfb6 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -117,6 +117,8 @@ typedef struct STableMeta { typedef struct SDBVgInfo { int32_t vgVersion; + int16_t hashPrefix; + int16_t hashSuffix; int8_t hashMethod; int32_t numOfTable; // DB's table num, unit is TSDB_TABLE_NUM_UNIT SHashObj* vgHash; // key:vgId, value:SVgroupInfo diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h new file mode 100644 index 0000000000000000000000000000000000000000..df195443963e348b54ee4d8c30db64fdc933f1ab --- /dev/null +++ b/include/libs/stream/streamState.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tdatablock.h" +#include "tdbInt.h" + +#ifdef __cplusplus +extern "C" { +#endif + +#ifndef _STREAM_STATE_H_ +#define _STREAM_STATE_H_ + +typedef struct SStreamTask SStreamTask; + +// incremental state storage +typedef struct { + SStreamTask* pOwner; + TDB* db; + TTB* pStateDb; + TTB* pFuncStateDb; + TXN txn; +} SStreamState; + +SStreamState* streamStateOpen(char* path, SStreamTask* pTask); +void streamStateClose(SStreamState* pState); +int32_t streamStateBegin(SStreamState* pState); +int32_t streamStateCommit(SStreamState* pState); +int32_t streamStateAbort(SStreamState* pState); + +typedef struct { + TBC* pCur; +} SStreamStateCur; + +#if 1 +int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen); +int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key); + +int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); +int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateDel(SStreamState* pState, const SWinKey* key); +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); +int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal); +void streamFreeVal(void* val); + +SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); +SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); +SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key); +void streamStateFreeCur(SStreamStateCur* pCur); + +int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); + +int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); +int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); + +int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); +int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); + +#endif + +#ifdef __cplusplus +} +#endif + +#endif /* ifndef _STREAM_STATE_H_ */ diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 2c275090080f73577cd28b3e10b3f1e102b4556e..554d66d62120977d45bba05988ff495482ef8246 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -16,6 +16,7 @@ #include "executor.h" #include "os.h" #include "query.h" +#include "streamState.h" #include "tdatablock.h" #include "tdbInt.h" #include "tmsg.h" @@ -263,14 +264,6 @@ typedef struct { SArray* checkpointVer; } SStreamRecoveringState; -// incremental state storage -typedef struct { - SStreamTask* pOwner; - TDB* db; - TTB* pStateDb; - TXN txn; -} SStreamState; - typedef struct SStreamTask { int64_t streamId; int32_t taskId; @@ -540,37 +533,6 @@ int32_t streamMetaCommit(SStreamMeta* pMeta); int32_t streamMetaRollBack(SStreamMeta* pMeta); int32_t streamLoadTasks(SStreamMeta* pMeta); -SStreamState* streamStateOpen(char* path, SStreamTask* pTask); -void streamStateClose(SStreamState* pState); -int32_t streamStateBegin(SStreamState* pState); -int32_t streamStateCommit(SStreamState* pState); -int32_t streamStateAbort(SStreamState* pState); - -typedef struct { - TBC* pCur; -} SStreamStateCur; - -#if 1 -int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen); -int32_t streamStateGet(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen); -int32_t streamStateDel(SStreamState* pState, const SWinKey* key); -void streamFreeVal(void* val); - -SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key); -SStreamStateCur* streamStateSeekKeyNext(SStreamState* pState, const SWinKey* key); -SStreamStateCur* streamStateSeekKeyPrev(SStreamState* pState, const SWinKey* key); -void streamStateFreeCur(SStreamStateCur* pCur); - -int32_t streamStateGetKVByCur(SStreamStateCur* pCur, SWinKey* pKey, const void** pVal, int32_t* pVLen); - -int32_t streamStateSeekFirst(SStreamState* pState, SStreamStateCur* pCur); -int32_t streamStateSeekLast(SStreamState* pState, SStreamStateCur* pCur); - -int32_t streamStateCurNext(SStreamState* pState, SStreamStateCur* pCur); -int32_t streamStateCurPrev(SStreamState* pState, SStreamStateCur* pCur); - -#endif - #ifdef __cplusplus } #endif diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 6c95c3c6d72929045bd780056811c1938864717b..de2271554d2b9166ec240ea07e91dea9d017ef92 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -444,6 +444,70 @@ void syncAppendEntriesReplyPrint2(char* s, const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog(const SyncAppendEntriesReply* pMsg); void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg); +// --------------------------------------------- +typedef struct SyncHeartbeat { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncIndex commitIndex; + SyncTerm privateTerm; +} SyncHeartbeat; + +SyncHeartbeat* syncHeartbeatBuild(int32_t vgId); +void syncHeartbeatDestroy(SyncHeartbeat* pMsg); +void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen); +void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg); +char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len); +SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len); +void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg); +void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg); +SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg); +char* syncHeartbeat2Str(const SyncHeartbeat* pMsg); + +// for debug ---------------------- +void syncHeartbeatPrint(const SyncHeartbeat* pMsg); +void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg); +void syncHeartbeatLog(const SyncHeartbeat* pMsg); +void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg); + +// --------------------------------------------- +typedef struct SyncHeartbeatReply { + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SRaftId srcId; + SRaftId destId; + + // private data + SyncTerm term; + SyncTerm privateTerm; + int64_t startTime; +} SyncHeartbeatReply; + +SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId); +void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg); +void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen); +void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg); +char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len); +SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len); +void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg); +void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg); +SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg); +cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg); +char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg); + +// for debug ---------------------- +void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg); +void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg); + // --------------------------------------------- typedef struct SyncApplyMsg { uint32_t bytes; diff --git a/include/libs/tfs/tfs.h b/include/libs/tfs/tfs.h index 1dc154ce484e1a923852f877bdae4ec37d62b1b4..6f71fd4cd02a241b9d4a43efbfd92bc99b3c91a8 100644 --- a/include/libs/tfs/tfs.h +++ b/include/libs/tfs/tfs.h @@ -69,6 +69,14 @@ void tfsUpdateSize(STfs *pTfs); */ SDiskSize tfsGetSize(STfs *pTfs); +/** + * @brief Get level of multi-tier storage. + * + * @param pTfs + * @return int32_t + */ +int32_t tfsGetLevel(STfs *pTfs); + /** * @brief Allocate an existing available tier level from fs. * diff --git a/include/os/osDir.h b/include/os/osDir.h index 9019d4f80240b2335824cb5626488bf4d0957f06..95b1a6ee1d00ab18e31522063102ff0ec9a2bab8 100644 --- a/include/os/osDir.h +++ b/include/os/osDir.h @@ -56,6 +56,7 @@ void taosRemoveDir(const char *dirname); bool taosDirExist(const char *dirname); int32_t taosMkDir(const char *dirname); int32_t taosMulMkDir(const char *dirname); +int32_t taosMulModeMkDir(const char *dirname, int mode); void taosRemoveOldFiles(const char *dirname, int32_t keepDays); int32_t taosExpandDir(const char *dirname, char *outname, int32_t maxlen); int32_t taosRealPath(char *dirname, char *realPath, int32_t maxlen); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index e39172d74e52e852f0fa1812634e494d61ac6213..01d42d6950dd2b7cdac549fcff14beef6e127db0 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -285,6 +285,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_TOPIC_SUBSCRIBED TAOS_DEF_ERROR_CODE(0, 0x03EB) #define TSDB_CODE_MND_CGROUP_USED TAOS_DEF_ERROR_CODE(0, 0x03EC) #define TSDB_CODE_MND_TOPIC_MUST_BE_DELETED TAOS_DEF_ERROR_CODE(0, 0x03ED) +#define TSDB_CODE_MND_IN_REBALANCE TAOS_DEF_ERROR_CODE(0, 0x03EF) // mnode-stream #define TSDB_CODE_MND_STREAM_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x03F0) @@ -577,6 +578,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_FUNC_FUNTION_PARA_TYPE TAOS_DEF_ERROR_CODE(0, 0x2802) #define TSDB_CODE_FUNC_FUNTION_PARA_VALUE TAOS_DEF_ERROR_CODE(0, 0x2803) #define TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION TAOS_DEF_ERROR_CODE(0, 0x2804) +#define TSDB_CODE_FUNC_DUP_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x2805) //udf #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) @@ -616,6 +618,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP TAOS_DEF_ERROR_CODE(0, 0x3155) #define TSDB_CODE_RSMA_EMPTY_INFO TAOS_DEF_ERROR_CODE(0, 0x3156) #define TSDB_CODE_RSMA_INVALID_SCHEMA TAOS_DEF_ERROR_CODE(0, 0x3157) +#define TSDB_CODE_RSMA_REGEX_MATCH TAOS_DEF_ERROR_CODE(0, 0x3158) //index #define TSDB_CODE_INDEX_REBUILDING TAOS_DEF_ERROR_CODE(0, 0x3200) diff --git a/include/util/tdef.h b/include/util/tdef.h index b17b8cbc66a4aea7a9b3deaec571cee32464ec01..9aa6985e298489cc0dab20ba1917cbacae2f8e93 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -300,6 +300,9 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_PAGES_PER_VNODE 256 #define TSDB_MIN_PAGESIZE_PER_VNODE 1 // unit KB #define TSDB_MAX_PAGESIZE_PER_VNODE 16384 +#define TSDB_DEFAULT_TSDB_PAGESIZE 4 +#define TSDB_MIN_TSDB_PAGESIZE 1 // unit KB +#define TSDB_MAX_TSDB_PAGESIZE 16384 #define TSDB_DEFAULT_PAGESIZE_PER_VNODE 4 #define TSDB_MIN_DAYS_PER_FILE 60 // unit minute #define TSDB_MAX_DAYS_PER_FILE (3650 * 1440) @@ -359,15 +362,27 @@ typedef enum ELogicConditionType { #define TSDB_DB_SCHEMALESS_ON 1 #define TSDB_DB_SCHEMALESS_OFF 0 #define TSDB_DEFAULT_DB_SCHEMALESS TSDB_DB_SCHEMALESS_OFF - -#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 -#define TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD (24 * 60 * 60 * 4) -#define TSDB_DB_MIN_WAL_RETENTION_SIZE -1 -#define TSDB_DEFAULT_DB_WAL_RETENTION_SIZE -1 -#define TSDB_DB_MIN_WAL_ROLL_PERIOD 0 -#define TSDB_DEFAULT_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1) -#define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0 -#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0 +#define TSDB_MIN_STT_TRIGGER 1 +#define TSDB_MAX_STT_TRIGGER 16 +#define TSDB_DEFAULT_SST_TRIGGER 8 +#define TSDB_MIN_HASH_PREFIX 0 +#define TSDB_MAX_HASH_PREFIX 128 +#define TSDB_DEFAULT_HASH_PREFIX 0 +#define TSDB_MIN_HASH_SUFFIX 0 +#define TSDB_MAX_HASH_SUFFIX 128 +#define TSDB_DEFAULT_HASH_SUFFIX 0 + +#define TSDB_DB_MIN_WAL_RETENTION_PERIOD -1 +#define TSDB_REP_DEF_DB_WAL_RET_PERIOD 0 +#define TSDB_REPS_DEF_DB_WAL_RET_PERIOD (24 * 60 * 60 * 4) +#define TSDB_DB_MIN_WAL_RETENTION_SIZE -1 +#define TSDB_REP_DEF_DB_WAL_RET_SIZE 0 +#define TSDB_REPS_DEF_DB_WAL_RET_SIZE -1 +#define TSDB_DB_MIN_WAL_ROLL_PERIOD 0 +#define TSDB_REP_DEF_DB_WAL_ROLL_PERIOD 0 +#define TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD (24 * 60 * 60 * 1) +#define TSDB_DB_MIN_WAL_SEGMENT_SIZE 0 +#define TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE 0 #define TSDB_MIN_ROLLUP_MAX_DELAY 1 // unit millisecond #define TSDB_MAX_ROLLUP_MAX_DELAY (15 * 60 * 1000) @@ -386,7 +401,7 @@ typedef enum ELogicConditionType { #define TSDB_DEFAULT_EXPLAIN_VERBOSE false -#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16*1024) +#define TSDB_EXPLAIN_RESULT_ROW_SIZE (16 * 1024) #define TSDB_EXPLAIN_RESULT_COLUMN_NAME "QUERY_PLAN" #define TSDB_MAX_FIELD_LEN 16384 diff --git a/include/util/trbtree.h b/include/util/trbtree.h new file mode 100644 index 0000000000000000000000000000000000000000..f6d37e3d753de71fdf312b795935cb9014149f23 --- /dev/null +++ b/include/util/trbtree.h @@ -0,0 +1,78 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#ifndef _TD_UTIL_RBTREE_H_ +#define _TD_UTIL_RBTREE_H_ + +#include "os.h" + +#ifdef __cplusplus +extern "C" { +#endif + +typedef struct SRBTree SRBTree; +typedef struct SRBTreeNode SRBTreeNode; +typedef struct SRBTreeIter SRBTreeIter; + +typedef int32_t (*tRBTreeCmprFn)(const void *, const void *); + +// SRBTree ============================================= +#define tRBTreeMin(T) ((T)->min == ((T)->NIL) ? NULL : (T)->min) +#define tRBTreeMax(T) ((T)->max == ((T)->NIL) ? NULL : (T)->max) + +void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn); +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z); +void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z); +SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey); +SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey); + +// SRBTreeIter ============================================= +#define tRBTreeIterCreate(tree, ascend) \ + (SRBTreeIter) { .asc = (ascend), .pTree = (tree), .pNode = (ascend) ? (tree)->min : (tree)->max } + +SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter); + +// STRUCT ============================================= +typedef enum { RED, BLACK } ECOLOR; +struct SRBTreeNode { + ECOLOR color; + SRBTreeNode *parent; + SRBTreeNode *left; + SRBTreeNode *right; +}; + +#define RBTREE_NODE_PAYLOAD(N) ((const void *)&(N)[1]) + +struct SRBTree { + tRBTreeCmprFn cmprFn; + int64_t n; + SRBTreeNode *root; + SRBTreeNode *min; + SRBTreeNode *max; + SRBTreeNode *NIL; + SRBTreeNode NILNODE; +}; + +struct SRBTreeIter { + int8_t asc; + SRBTree *pTree; + SRBTreeNode *pNode; +}; + +#ifdef __cplusplus +} +#endif + +#endif /*_TD_UTIL_RBTREE_H_*/ \ No newline at end of file diff --git a/include/util/tsched.h b/include/util/tsched.h index 347cacd19185b0891deadfdbdc6a0b42b8a71d18..379456afe64ebfd225bcec8dc873a23c54e6e29b 100644 --- a/include/util/tsched.h +++ b/include/util/tsched.h @@ -31,7 +31,6 @@ typedef struct SSchedMsg { void *thandle; } SSchedMsg; - typedef struct { char label[TSDB_LABEL_LEN]; tsem_t emptySem; @@ -48,7 +47,6 @@ typedef struct { void *pTimer; } SSchedQueue; - /** * Create a thread-safe ring-buffer based task queue and return the instance. A thread * pool will be created to consume the messages in the queue. @@ -57,7 +55,7 @@ typedef struct { * @param label the label of the queue * @return the created queue scheduler */ -void *taosInitScheduler(int32_t capacity, int32_t numOfThreads, const char *label, SSchedQueue* pSched); +void *taosInitScheduler(int32_t capacity, int32_t numOfThreads, const char *label, SSchedQueue *pSched); /** * Create a thread-safe ring-buffer based task queue and return the instance. @@ -83,7 +81,7 @@ void taosCleanUpScheduler(void *queueScheduler); * @param queueScheduler the queue scheduler instance * @param pMsg the message for the task */ -void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); +int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg); #ifdef __cplusplus } diff --git a/include/util/tutil.h b/include/util/tutil.h index 6a1a40f14ccb865533f117524ffdfef3c84e20ad..32a88b37ecffc16c2a222d73d9fe452e6fcaacac 100644 --- a/include/util/tutil.h +++ b/include/util/tutil.h @@ -20,6 +20,7 @@ #include "tcrc32c.h" #include "tdef.h" #include "tmd5.h" +#include "thash.h" #ifdef __cplusplus extern "C" { @@ -68,6 +69,19 @@ static FORCE_INLINE void taosEncryptPass_c(uint8_t *inBuf, size_t len, char *tar memcpy(target, buf, TSDB_PASSWORD_LEN); } +static FORCE_INLINE int32_t taosGetTbHashVal(const char *tbname, int32_t tblen, int32_t method, int32_t prefix, + int32_t suffix) { + if (prefix == 0 && suffix == 0) { + return MurmurHash3_32(tbname, tblen); + } else { + if (tblen <= (prefix + suffix)) { + return MurmurHash3_32(tbname, tblen); + } else { + return MurmurHash3_32(tbname + prefix, tblen - prefix - suffix); + } + } +} + #ifdef __cplusplus } #endif diff --git a/packaging/MPtestJenkinsfile b/packaging/MPtestJenkinsfile index 77f642180a59932d8f8e4deaa2d91ac37e7268fc..5b793459164baec2791107842bddb3f0bb90b2df 100644 --- a/packaging/MPtestJenkinsfile +++ b/packaging/MPtestJenkinsfile @@ -5,13 +5,6 @@ def sync_source(branch_name) { echo ''' + branch_name + ''' ''' sh ''' - cd ${TDINTERNAL_ROOT_DIR} - git reset --hard - git fetch || git fetch - git checkout ''' + branch_name + ''' -f - git branch - git pull || git pull - git log | head -n 20 cd ${TDENGINE_ROOT_DIR} git reset --hard git fetch || git fetch @@ -64,17 +57,12 @@ pipeline { defaultValue:'2.1.2', description: 'This number of baseVerison is generally not modified.Now it is 3.0.0.1' ) - string ( - name:'nasPassword', - defaultValue:'password', - description: 'the pasword of the NAS server which has installPackage-192.168.1.131' - ) } environment{ WORK_DIR = '/var/lib/jenkins/workspace' TDINTERNAL_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal' TDENGINE_ROOT_DIR = '/var/lib/jenkins/workspace/TDinternal/community' - BRANCH_NAME = '3.0' + BRANCH_NAME = 'test/chr/TD-14699' TD_SERVER_TAR = "TDengine-server-${version}-Linux-x64.tar.gz" BASE_TD_SERVER_TAR = "TDengine-server-${baseVersion}-Linux-x64.tar.gz" @@ -107,7 +95,7 @@ pipeline { } stages { - stage ('RUN') { + stage ('Test Server') { parallel { stage('ubuntu16') { agent{label " ubuntu16 "} @@ -116,17 +104,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' } @@ -139,24 +127,21 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client ${nasPassword} + bash testpackage.sh ${TD_SERVER_DEB} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py + dpkg -r tdengine ''' + } } } @@ -167,17 +152,17 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' } @@ -190,28 +175,23 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_TAR} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_LITE_TAR} ${version} ${BASE_TD_SERVER_LITE_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server ${nasPassword} - python3 checkPackageRuning.py - ''' - sh ''' - cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client ${nasPassword} + bash testpackage.sh ${TD_SERVER_RPM} ${version} ${BASE_TD_SERVER_TAR} ${baseVersion} server python3 checkPackageRuning.py - ''' + sudo rpm -e tdengine + ''' } } - } - + } stage('arm64') { agent{label 'linux_arm64'} steps { @@ -219,18 +199,53 @@ pipeline { sync_source("${BRANCH_NAME}") sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server ${nasPassword} + bash testpackage.sh ${TD_SERVER_ARM_TAR} ${version} ${BASE_TD_SERVER_ARM_TAR} ${baseVersion} server python3 checkPackageRuning.py ''' + } + } + } + } + } + stage ('Test Client') { + parallel { + stage('ubuntu18') { + agent{label " ubuntu18 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ sh ''' cd ${TDENGINE_ROOT_DIR}/packaging - bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client ${nasPassword} - python3 checkPackageRuning.py - ''' + bash testpackage.sh ${TD_CLIENT_TAR} ${version} ${BASE_TD_CLIENT_TAR} ${baseVersion} client + python3 checkPackageRuning.py 192.168.0.21 + ''' } } } + stage('centos8') { + agent{label " centos8_3 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh ${TD_CLIENT_LITE_TAR} ${version} ${BASE_TD_CLIENT_LITE_TAR} ${baseVersion} client + python3 checkPackageRuning.py 192.168.0.24 + ''' + } + } + } } - } - } + } + stage('arm64-client') { + agent{label " linux_arm64 "} + steps { + timeout(time: 30, unit: 'MINUTES'){ + sh ''' + cd ${TDENGINE_ROOT_DIR}/packaging + bash testpackage.sh ${TD_CLIENT_ARM_TAR} ${version} ${BASE_TD_CLIENT_ARM_TAR} ${baseVersion} client + python3 checkPackageRuning.py 192.168.0.21 + ''' + } + } + } + } } \ No newline at end of file diff --git a/packaging/checkPackageRuning.py b/packaging/checkPackageRuning.py index c0d1e8b86c3df2150b7f434e899c545439ab0477..2edeeb6dbbb682bb06150e30803a7f05c170a5b1 100755 --- a/packaging/checkPackageRuning.py +++ b/packaging/checkPackageRuning.py @@ -19,12 +19,19 @@ import subprocess # from this import d import time + +if( len(sys.argv)>1 ): + serverHost=sys.argv[1] +else: + serverHost="localhost" + + # install taospy out = subprocess.getoutput("pip3 show taospy|grep Version| awk -F ':' '{print $2}' ") print("taospy version %s "%out) if (out == "" ): - os.system("pip install git+https://github.com/taosdata/taos-connector-python.git") + os.system("pip3 install git+https://github.com/taosdata/taos-connector-python.git") print("install taos python connector") else: os.system("pip3 install --upgrade taospy ") @@ -32,19 +39,19 @@ else: # start taosd prepare -os.system("rm -rf /var/lib/taos/*") -os.system("systemctl restart taosd ") +# os.system("rm -rf /var/lib/taos/*") +# os.system("systemctl restart taosd ") # wait a moment ,at least 5 seconds time.sleep(5) # prepare data by taosBenchmark -os.system("taosBenchmark -y -n 100 -t 100") +os.system("taosBenchmark -y -n 100 -t 100 -h %s "%serverHost ) import taos -conn = taos.connect(host="localhost", +conn = taos.connect(host="%s"%serverHost, user="root", password="taosdata", database="test", @@ -80,15 +87,15 @@ os.system("rm -rf /tmp/dumpdata/*") # dump data out print("taosdump dump out data") -os.system("taosdump -o /tmp/dumpdata -D test -y ") +os.system("taosdump -o /tmp/dumpdata -D test -y -h %s "%serverHost) # drop database of test print("drop database test") -os.system(" taos -s ' drop database test ;' ") +os.system(" taos -s ' drop database test ;' -h %s "%serverHost) # dump data in print("taosdump dump data in") -os.system("taosdump -i /tmp/dumpdata -y ") +os.system("taosdump -i /tmp/dumpdata -y -h %s "%serverHost) result = conn.query("SELECT count(*) from test.meters") diff --git a/packaging/deb/makedeb.sh b/packaging/deb/makedeb.sh index 3db9005f95a3027c42dd05b9f28d448ade5852cb..94a24a41487e8d7b82571bcc524392e4335d7fae 100755 --- a/packaging/deb/makedeb.sh +++ b/packaging/deb/makedeb.sh @@ -45,6 +45,7 @@ mkdir -p ${pkg_dir}${install_home_path}/include mkdir -p ${pkg_dir}${install_home_path}/script cp ${compile_dir}/../packaging/cfg/taos.cfg ${pkg_dir}${install_home_path}/cfg +cp ${compile_dir}/../packaging/cfg/taosd.service ${pkg_dir}${install_home_path}/cfg if [ -f "${compile_dir}/test/cfg/taosadapter.toml" ]; then cp ${compile_dir}/test/cfg/taosadapter.toml ${pkg_dir}${install_home_path}/cfg || : fi diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh index 1f51378c91d14b5fcfd1eb4cca87a6cd472161cc..3579f813e5b6ce91f0daa1fd230af14a4bf3d4b9 100755 --- a/packaging/debRpmAutoInstall.sh +++ b/packaging/debRpmAutoInstall.sh @@ -11,3 +11,5 @@ expect "*one:" send "\r" expect "*skip:" send "\r" + +expect eof \ No newline at end of file diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 56da9e59be9379af9a1b96eadc88fe0e052e9863..794b3968fe4df178e3da91d2ebbd1512e288a57e 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -7,7 +7,6 @@ originPackageName=$3 originversion=$4 testFile=$5 subFile="taos.tar.gz" -password=$6 # Color setting RED='\033[41;30m' @@ -68,11 +67,37 @@ fi } +function wgetFile { + +file=$1 + +if [ ! -f ${file} ];then + echoColor BD "wget https://www.taosdata.com/assets-download/3.0/${file}" + wget https://www.taosdata.com/assets-download/3.0/${file} +else + echoColor YD "${file} already exists " +fi +} + +function newPath { + +buildPath=$1 + +if [ ! -d ${buildPath} ] ;then + echoColor BD "mkdir -p ${buildPath}" + mkdir -p ${buildPath} +else + echoColor YD "${buildPath} already exists" +fi + +} + + echoColor G "===== install basesoft =====" cmdInstall tree cmdInstall wget -cmdInstall sshpass +cmdInstall expect echoColor G "===== Uninstall all components of TDeingne =====" @@ -97,45 +122,28 @@ echoColor G "===== new workroom path =====" installPath="/usr/local/src/packageTest" oriInstallPath="/usr/local/src/packageTest/3.1" -if [ ! -d ${installPath} ] ;then - echoColor BD "mkdir -p ${installPath}" - mkdir -p ${installPath} -else - echoColor YD "${installPath} already exists" -fi +newPath ${installPath} -if [ -d ${installPath}/${tdPath} ] ;then - echoColor BD "rm -rf ${installPath}/${tdPath}/*" - rm -rf ${installPath}/${tdPath}/* -fi +newPath ${oriInstallPath} -if [ ! -d ${oriInstallPath} ] ;then - echoColor BD "mkdir -p ${oriInstallPath}" - mkdir -p ${oriInstallPath} -else - echoColor YD "${oriInstallPath} already exists" -fi if [ -d ${oriInstallPath}/${originTdpPath} ] ;then echoColor BD "rm -rf ${oriInstallPath}/${originTdpPath}/*" rm -rf ${oriInstallPath}/${originTdpPath}/* fi +if [ -d ${installPath}/${tdPath} ] ;then + echoColor BD "rm -rf ${installPath}/${tdPath}/*" + rm -rf ${installPath}/${tdPath}/* +fi echoColor G "===== download installPackage =====" -# cd ${installPath} -# wget https://www.taosdata.com/assets-download/3.0/${packgeName} -# cd ${oriInstallPath} -# wget https://www.taosdata.com/assets-download/3.0/${originPackageName} +cd ${installPath} && wgetFile ${packgeName} +cd ${oriInstallPath} && wgetFile ${originPackageName} cd ${installPath} cp -r ${scriptDir}/debRpmAutoInstall.sh . -if [ ! -f {packgeName} ];then - echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName} ." - sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/${packgeName} . -fi - packageSuffix=$(echo ${packgeName} | awk -F '.' '{print $NF}') @@ -181,8 +189,7 @@ elif [[ ${packgeName} =~ "tar" ]];then cd ${oriInstallPath} if [ ! -f {originPackageName} ];then echoColor YD "download base installPackage" - echoColor BD "sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} ." - sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${originversion}/community/${originPackageName} . + wgetFile ${originPackageName} fi echoColor YD "unzip the base installation package" echoColor BD "tar -xf ${originPackageName}" && tar -xf ${originPackageName} @@ -222,24 +229,45 @@ fi cd ${installPath} -if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then - echoColor G "===== install taos-tools when package is lite or client =====" - cd ${installPath} - sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz . - # wget https://www.taosdata.com/assets-download/3.0/taosTools-2.1.2-Linux-x64.tar.gz - tar xf taosTools-2.1.2-Linux-x64.tar.gz - cd taosTools-2.1.2 && bash install-taostools.sh -elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "deb" ]] ;then +if [[ ${packgeName} =~ "Lite" ]] || ([[ ${packgeName} =~ "x64" ]] && [[ ${packgeName} =~ "client" ]]) || ([[ ${packgeName} =~ "deb" ]] && [[ ${packgeName} =~ "server" ]]) || ([[ ${packgeName} =~ "rpm" ]] && [[ ${packgeName} =~ "server" ]]) ;then echoColor G "===== install taos-tools when package is lite or client =====" cd ${installPath} - sshpass -p ${password} scp -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz . - tar xf taosTools-2.1.2-Linux-x64.tar.gz - cd taosTools-2.1.2 && bash install-taostools.sh -elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "rpm" ]] ;then - echoColor G "===== install taos-tools when package is lite or client =====" + wgetFile taosTools-2.1.3-Linux-x64.tar.gz . + tar xf taosTools-2.1.3-Linux-x64.tar.gz + cd taosTools-2.1.3 && bash install-taostools.sh +elif ([[ ${packgeName} =~ "arm64" ]] && [[ ${packgeName} =~ "client" ]]);then + echoColor G "===== install taos-tools arm when package is arm64-client =====" cd ${installPath} - sshpass -p ${password} scp -oStrictHostKeyChecking=no -oStrictHostKeyChecking=no 192.168.1.131:/nas/TDengine3/v${version}/community/taosTools-2.1.2-Linux-x64.tar.gz . - tar xf taosTools-2.1.2-Linux-x64.tar.gz - cd taosTools-2.1.2 && bash install-taostools.sh + wgetFile taosTools-2.1.3-Linux-arm64.tar.gz . + tar xf taosTools-2.1.3-Linux-arm64.tar.gz + cd taosTools-2.1.3 && bash install-taostools.sh fi +echoColor G "===== start TDengine =====" + +if [[ ${packgeName} =~ "server" ]] ;then + echoColor BD " rm -rf /var/lib/taos/* && systemctl restart taosd " + rm -rf /var/lib/taos/* + systemctl restart taosd +fi + +# if ([[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "tar" ]]) || [[ ${packgeName} =~ "client" ]] ;then +# echoColor G "===== install taos-tools when package is lite or client =====" +# cd ${installPath} +# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . +# tar xf taosTools-2.1.2-Linux-x64.tar.gz +# cd taosTools-2.1.2 && bash install-taostools.sh +# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "deb" ]] ;then +# echoColor G "===== install taos-tools when package is lite or client =====" +# cd ${installPath} +# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . +# tar xf taosTools-2.1.2-Linux-x64.tar.gz +# cd taosTools-2.1.2 && bash install-taostools.sh +# elif [[ ${packgeName} =~ "Lite" ]] && [[ ${packgeName} =~ "rpm" ]] ;then +# echoColor G "===== install taos-tools when package is lite or client =====" +# cd ${installPath} +# wgetFile taosTools-2.1.2-Linux-x64.tar.gz . +# tar xf taosTools-2.1.2-Linux-x64.tar.gz +# cd taosTools-2.1.2 && bash install-taostools.sh +# fi + diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 39606ead300c8c603b9f25360d19e3af49b642ff..5088e9bdac6e3da77cfd5c84cf33b9f541db8a07 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -33,6 +33,7 @@ adapterName="taosadapter" benchmarkName="taosBenchmark" dumpName="taosdump" demoName="taosdemo" +xname="taosx" data_dir=${dataDir} log_dir=${logDir} @@ -199,6 +200,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${demoName} || : ${csudo}rm -f ${bin_link_dir}/${benchmarkName} || : ${csudo}rm -f ${bin_link_dir}/${dumpName} || : + ${csudo}rm -f ${bin_link_dir}/${xname} || : ${csudo}rm -f ${bin_link_dir}/set_core || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : @@ -212,6 +214,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${demoName} || : [ -x ${install_main_dir}/bin/${benchmarkName} ] && ${csudo}ln -s ${install_main_dir}/bin/${benchmarkName} ${bin_link_dir}/${benchmarkName} || : [ -x ${install_main_dir}/bin/${dumpName} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${dumpName} || : + [ -x ${install_main_dir}/bin/${xname} ] && ${csudo}ln -s ${install_main_dir}/bin/${dumpName} ${bin_link_dir}/${xname} || : [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : @@ -837,14 +840,20 @@ function updateProduct() { echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" - echo -e "${GREEN_DARK}To configure Adapter (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml" + [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" else - echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi if [ ${openresty_work} = 'true' ]; then @@ -923,14 +932,20 @@ function installProduct() { # Ask if to start the service echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${cfg_install_dir}/${configFile}" - echo -e "${GREEN_DARK}To configure ${adapterName} (if has) ${NC}: edit ${cfg_install_dir}/${adapterName}.toml" + [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" else - echo -e "${GREEN_DARK}To start Adapter (if has)${NC}: ${adapterName} &${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi if [ ! -z "$firstEp" ]; then diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index f554942ce330767b45ef5ec2ee0df422c273c5ed..d1e7a222cc1739cca1e90fe268f8512326e36ae5 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -172,6 +172,7 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/udfd || : ${csudo}rm -f ${bin_link_dir}/taosdemo || : ${csudo}rm -f ${bin_link_dir}/taosdump || : + ${csudo}rm -f ${bin_link_dir}/taosx || : if [ "$osType" != "Darwin" ]; then ${csudo}rm -f ${bin_link_dir}/perfMonitor || : @@ -184,6 +185,7 @@ function install_bin() { [ -f ${binary_dir}/build/bin/taosdump ] && ${csudo}cp -r ${binary_dir}/build/bin/taosdump ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/taosadapter ] && ${csudo}cp -r ${binary_dir}/build/bin/taosadapter ${install_main_dir}/bin || : [ -f ${binary_dir}/build/bin/udfd ] && ${csudo}cp -r ${binary_dir}/build/bin/udfd ${install_main_dir}/bin || : + [ -f ${binary_dir}/build/bin/taosx ] && ${csudo}cp -r ${binary_dir}/build/bin/taosx ${install_main_dir}/bin || : ${csudo}cp -r ${binary_dir}/build/bin/${serverName} ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : @@ -199,6 +201,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || : [ -x ${install_main_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : @@ -215,6 +218,7 @@ function install_bin() { [ -x ${install_main_dir}/bin/udfd ] || [ -x ${install_main_2_dir}/bin/udfd ] && ${csudo}ln -s ${install_main_dir}/bin/udfd ${bin_link_dir}/udfd || ${csudo}ln -s ${install_main_2_dir}/bin/udfd || : [ -x ${install_main_dir}/bin/taosdump ] || [ -x ${install_main_2_dir}/bin/taosdump ] && ${csudo}ln -s ${install_main_dir}/bin/taosdump ${bin_link_dir}/taosdump || ln -s ${install_main_2_dir}/bin/taosdump ${bin_link_dir}/taosdump || : [ -x ${install_main_dir}/bin/taosdemo ] || [ -x ${install_main_2_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || ln -s ${install_main_2_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : + [ -x ${install_main_dir}/bin/taosx ] || [ -x ${install_main_2_dir}/bin/taosx ] && ${csudo}ln -s ${install_main_dir}/bin/taosx ${bin_link_dir}/taosx || ln -s ${install_main_2_dir}/bin/taosx ${bin_link_dir}/taosx || : fi } @@ -605,14 +609,20 @@ function update_TDengine() { echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" - echo -e "${GREEN_DARK}To configure Taos Adapter (if has) ${NC}: edit ${configDir}/taosadapter.toml" + [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adatper ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" else - echo -e "${GREEN_DARK}To start Taos Adapter (if has)${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName}${NC} in shell${NC}" @@ -645,14 +655,20 @@ function install_TDengine() { echo -e "\033[44;32;1m${productName} is installed successfully!${NC}" echo echo -e "${GREEN_DARK}To configure ${productName} ${NC}: edit ${configDir}/${configFile}" - echo -e "${GREEN_DARK}To configure taosadapter (if has) ${NC}: edit ${configDir}/taosadapter.toml" + [ -f ${configDir}/taosadapter.toml ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To configure Taos Adapter ${NC}: edit ${configDir}/taosadapter.toml" if ((${service_mod} == 0)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}systemctl start ${serverName}${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}systemctl start taosadapter ${NC}" elif ((${service_mod} == 1)); then echo -e "${GREEN_DARK}To start ${productName} ${NC}: ${csudo}service ${serverName} start${NC}" + [ -f ${service_config_dir}/taosadapter.service ] && [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: ${csudo}service taosadapter start${NC}" else - echo -e "${GREEN_DARK}To start Taos Adapter (if has)${NC}: taosadapter &${NC}" echo -e "${GREEN_DARK}To start ${productName} ${NC}: ./${serverName}${NC}" + [ -f ${installDir}/bin/taosadapter ] && \ + echo -e "${GREEN_DARK}To start Taos Adapter ${NC}: taosadapter &${NC}" fi echo -e "${GREEN_DARK}To access ${productName} ${NC}: use ${GREEN_UNDERLINE}${clientName}${NC} in shell${NC}" diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index f5e3bf18822676f54ee2f20412b5ebb4ce57fd3a..2305b96b3663c3c52ca84988518de3fd00769af8 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -80,10 +80,12 @@ else ${build_dir}/bin/taosBenchmark \ ${build_dir}/bin/TDinsight.sh \ $tdinsight_caches" + [ -f ${build_dir}/bin/taosx ] && taosx_bin="${build_dir}/bin/taosx" bin_files="${build_dir}/bin/${serverName} \ ${build_dir}/bin/${clientName} \ ${taostools_bin_files} \ + ${taosx_bin} \ ${build_dir}/bin/taosadapter \ ${build_dir}/bin/udfd \ ${script_dir}/remove.sh \ diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index e1cf078d577337ea4c9d66552ca158b8e01d14ad..9a0639e1b6646f8452d6e78b08692e08833b8717 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -489,7 +489,7 @@ int taos_options_imp(TSDB_OPTION option, const char *str) { */ uint64_t generateRequestId() { static uint64_t hashId = 0; - static int32_t requestSerialId = 0; + static uint32_t requestSerialId = 0; if (hashId == 0) { char uid[64] = {0}; @@ -508,7 +508,8 @@ uint64_t generateRequestId() { while (true) { int64_t ts = taosGetTimestampMs(); uint64_t pid = taosGetPId(); - int32_t val = atomic_add_fetch_32(&requestSerialId, 1); + uint32_t val = atomic_add_fetch_32(&requestSerialId, 1); + if (val >= 0xFFFF) atomic_store_32(&requestSerialId, 0); id = ((hashId & 0x0FFF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); if (id) { diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index 56e3527f9684e17f58c7b0fca5bb7a3fa6210d1c..e484e3c59c0e48535f25ad0640e7a93092603f73 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -73,6 +73,8 @@ static int32_t hbProcessDBInfoRsp(void *value, int32_t valueLen, struct SCatalog vgInfo->vgVersion = rsp->vgVersion; vgInfo->hashMethod = rsp->hashMethod; + vgInfo->hashPrefix = rsp->hashPrefix; + vgInfo->hashSuffix = rsp->hashSuffix; vgInfo->vgHash = taosHashInit(rsp->vgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); if (NULL == vgInfo->vgHash) { taosMemoryFree(vgInfo); @@ -412,6 +414,9 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { int32_t code = hbBuildQueryDesc(hbBasic, pTscObj); if (code) { releaseTscObj(connKey->tscRid); + if (hbBasic->queryDesc) { + taosArrayDestroyEx(hbBasic->queryDesc, tFreeClientHbQueryDesc); + } taosMemoryFree(hbBasic); return code; } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 6a8e58efb401a884e922e9d2009b64ca4dfaab95..9ff436f64e4dd23d22f98d5300fe3d3d94abc7d9 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -877,6 +877,7 @@ void schedulerExecCb(SExecResult* pResult, void* param, int32_t code) { pRequest->metric.resultReady = taosGetTimestampUs(); if (pResult) { + destroyQueryExecRes(&pRequest->body.resInfo.execRes); memcpy(&pRequest->body.resInfo.execRes, pResult, sizeof(*pResult)); } @@ -1408,6 +1409,7 @@ int32_t doProcessMsgFromServer(void* param) { pSendInfo->fp(pSendInfo->param, &buf, pMsg->code); rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pSendInfo); + taosMemoryFree(arg); return TSDB_CODE_SUCCESS; } @@ -1423,7 +1425,12 @@ void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { arg->msg = *pMsg; arg->pEpset = tEpSet; - taosAsyncExec(doProcessMsgFromServer, arg, NULL); + if (0 != taosAsyncExec(doProcessMsgFromServer, arg, NULL)) { + tscError("failed to sched msg to tsc, tsc ready to quit"); + rpcFreeCont(pMsg->pCont); + taosMemoryFree(arg->pEpset); + taosMemoryFree(arg); + } } TAOS* taos_connect_auth(const char* ip, const char* user, const char* auth, const char* db, uint16_t port) { @@ -1696,7 +1703,12 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int break; } } - if (!needConvert) return TSDB_CODE_SUCCESS; + + if (!needConvert) { + return TSDB_CODE_SUCCESS; + } + + tscDebug("start to convert form json format string"); char* p = (char*)pResultInfo->pData; int32_t dataLen = estimateJsonLen(pResultInfo, numOfCols, numOfRows); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 73ff36a2994684e3f4da76afa6f56876486bcdce..d35d74b488f0e81de37c103a0a76677a0699bef3 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -871,11 +871,13 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { if (code != TSDB_CODE_SUCCESS) { pRequest->code = code; + taosMemoryFreeClear(pResultInfo->pData); pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); return; } if (pRequest->code != TSDB_CODE_SUCCESS) { + taosMemoryFreeClear(pResultInfo->pData); pRequest->body.fetchFp(pRequest->body.param, pRequest, 0); return; } diff --git a/source/client/src/taosx.c b/source/client/src/clientRawBlockWrite.c similarity index 97% rename from source/client/src/taosx.c rename to source/client/src/clientRawBlockWrite.c index f016120a1f5cbe1b4baf4e42f76a0fea7c80d353..c135965f07454c46e20780d029c18b2359110877 100644 --- a/source/client/src/taosx.c +++ b/source/client/src/clientRawBlockWrite.c @@ -765,23 +765,25 @@ static int32_t taosCreateTable(TAOS* taos, void* meta, int32_t metaLen) { } taosArrayPush(pRequest->tableList, &pName); + pCreateReq->flags |= TD_CREATE_IF_NOT_EXISTS; // change tag cid to new cid - if(pCreateReq->type == TSDB_CHILD_TABLE){ + if (pCreateReq->type == TSDB_CHILD_TABLE) { STableMeta* pTableMeta = NULL; SName sName = {0}; toName(pTscObj->acctId, pRequest->pDb, pCreateReq->ctb.name, &sName); code = catalogGetTableMeta(pCatalog, &conn, &sName, &pTableMeta); - if(code != TSDB_CODE_SUCCESS){ + if (code != TSDB_CODE_SUCCESS) { uError("taosCreateTable:catalogGetTableMeta failed. table name: %s", pCreateReq->ctb.name); goto end; } - for(int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++){ + for (int32_t i = 0; i < taosArrayGetSize(pCreateReq->ctb.tagName); i++) { char* tName = taosArrayGet(pCreateReq->ctb.tagName, i); - for(int32_t j = pTableMeta->tableInfo.numOfColumns; j < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; j++){ - SSchema *tag = &pTableMeta->schema[j]; - if(strcmp(tag->name, tName) == 0 && tag->type != TSDB_DATA_TYPE_JSON){ - tTagSetCid((STag *)pCreateReq->ctb.pTag, i, tag->colId); + for (int32_t j = pTableMeta->tableInfo.numOfColumns; + j < pTableMeta->tableInfo.numOfColumns + pTableMeta->tableInfo.numOfTags; j++) { + SSchema* tag = &pTableMeta->schema[j]; + if (strcmp(tag->name, tName) == 0 && tag->type != TSDB_DATA_TYPE_JSON) { + tTagSetCid((STag*)pCreateReq->ctb.pTag, i, tag->colId); } } } @@ -1322,12 +1324,12 @@ end: return code; } -static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) { - int32_t code = TSDB_CODE_SUCCESS; - SHashObj* pVgHash = NULL; - SQuery* pQuery = NULL; - SMqRspObj rspObj = {0}; - SDecoder decoder = {0}; +static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { + int32_t code = TSDB_CODE_SUCCESS; + SHashObj* pVgHash = NULL; + SQuery* pQuery = NULL; + SMqRspObj rspObj = {0}; + SDecoder decoder = {0}; STableMeta* pTableMeta = NULL; terrno = TSDB_CODE_SUCCESS; @@ -1405,7 +1407,7 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) { } code = catalogGetTableMeta(pCatalog, &conn, &pName, &pTableMeta); - if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST){ + if (code == TSDB_CODE_PAR_TABLE_NOT_EXIST) { uError("WriteRaw:catalogGetTableMeta table not exist. table name: %s", tbName); code = TSDB_CODE_SUCCESS; continue; @@ -1466,7 +1468,7 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) { } // pSW->pSchema should be same as pTableMeta->schema -// ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); + // ASSERT(pSW->nCols == pTableMeta->tableInfo.numOfColumns); uint64_t suid = (TSDB_NORMAL_TABLE == pTableMeta->tableType ? 0 : pTableMeta->suid); uint64_t uid = pTableMeta->uid; int16_t sver = pTableMeta->sversion; @@ -1494,10 +1496,10 @@ static int32_t tmqWriteRaw(TAOS* taos, void* data, int32_t dataLen) { int32_t offset = 0; for (int32_t k = 0; k < pTableMeta->tableInfo.numOfColumns; k++) { const SSchema* pColumn = &pTableMeta->schema[k]; - int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); - if(!index){ + int32_t* index = taosHashGet(schemaHash, pColumn->name, strlen(pColumn->name)); + if (!index) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); - }else{ + } else { char* colData = rspObj.resInfo.row[*index]; if (!colData) { tdAppendColValToRow(&rb, pColumn->colId, pColumn->type, TD_VTYPE_NULL, NULL, false, offset, k); @@ -1668,7 +1670,7 @@ int32_t tmq_write_raw(TAOS* taos, tmq_raw_data raw) { } else if (raw.raw_type == TDMT_VND_DELETE) { return taosDeleteData(taos, raw.raw, raw.raw_len); } else if (raw.raw_type == RES_TYPE__TMQ) { - return tmqWriteRaw(taos, raw.raw, raw.raw_len); + return tmqWriteRawDataImpl(taos, raw.raw, raw.raw_len); } return TSDB_CODE_INVALID_PARA; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 23dc0c08645ecbaa60ff2a9c2107c7fc6c439cae..f2493f6c57daf5654524f2c0e4f49d14df40b022 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -537,7 +537,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = smlSendMetaMsg(info, &pName, pColumns, pTags, pTableMeta, action); if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, superTable); + uError("SML:0x%" PRIx64 " smlSendMetaMsg failed. can not create %s", info->id, pName.tname); goto end; } } @@ -547,6 +547,8 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { goto end; } needCheckMeta = true; + taosHashCleanup(hashTmp); + hashTmp = NULL; } else { uError("SML:0x%" PRIx64 " load table meta error: %s", info->id, tstrerror(code)); goto end; @@ -555,7 +557,7 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = catalogGetSTableMeta(info->pCatalog, &conn, &pName, &pTableMeta); if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, (char *)superTable); + uError("SML:0x%" PRIx64 " catalogGetSTableMeta failed. super table name %s", info->id, pName.tname); goto end; } @@ -563,12 +565,12 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { code = smlCheckMeta(&(pTableMeta->schema[pTableMeta->tableInfo.numOfColumns]), pTableMeta->tableInfo.numOfTags, sTableData->tags, true); if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, (char *)superTable); + uError("SML:0x%" PRIx64 " check tag failed. super table name %s", info->id, pName.tname); goto end; } code = smlCheckMeta(&(pTableMeta->schema[0]), pTableMeta->tableInfo.numOfColumns, sTableData->cols, false); if (code != TSDB_CODE_SUCCESS) { - uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, (char *)superTable); + uError("SML:0x%" PRIx64 " check cols failed. super table name %s", info->id, pName.tname); goto end; } } @@ -576,7 +578,6 @@ static int32_t smlModifyDBSchemas(SSmlHandle *info) { sTableData->tableMeta = pTableMeta; tableMetaSml = (SSmlSTableMeta **)taosHashIterate(info->superTables, tableMetaSml); - taosHashCleanup(hashTmp); } return 0; @@ -1559,7 +1560,7 @@ cleanup: /************* TSDB_SML_JSON_PROTOCOL function start **************/ static int32_t smlJsonCreateSring(const char **output, char *input, int32_t inputLen) { - *output = (const char *)taosMemoryMalloc(inputLen); + *output = (const char *)taosMemoryCalloc(1, inputLen); if (*output == NULL) { return TSDB_CODE_TSC_OUT_OF_MEMORY; } @@ -2450,9 +2451,11 @@ static void smlInsertCallback(void *param, void *res, int32_t code) { uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); // lock taosThreadSpinLock(&info->params->lock); - info->params->request->body.resInfo.numOfRows += rows; if (code != TSDB_CODE_SUCCESS) { info->params->request->code = code; + info->params->request->body.resInfo.numOfRows += rows; + }else{ + info->params->request->body.resInfo.numOfRows += info->affectedRows; } taosThreadSpinUnlock(&info->params->lock); // unlock diff --git a/source/client/src/tmq.c b/source/client/src/clientTmq.c similarity index 100% rename from source/client/src/tmq.c rename to source/client/src/clientTmq.c diff --git a/source/common/src/systable.c b/source/common/src/systable.c index dffef21ac49502534e0b42deaf9359c8562d0997..496806d87794416e19cd4338659b87ee6f520792 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -90,7 +90,7 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "minrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "maxrows", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "comp", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, - {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "precision", .bytes = 2 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "retentions", .bytes = 60 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "single_stable", .bytes = 1, .type = TSDB_DATA_TYPE_BOOL, .sysInfo = true}, @@ -102,6 +102,10 @@ static const SSysDbTableSchema userDBSchema[] = { {.name = "wal_retention_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, {.name = "wal_roll_period", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "wal_segment_size", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = true}, + {.name = "stt_trigger", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "table_prefix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "table_suffix", .bytes = 2, .type = TSDB_DATA_TYPE_SMALLINT, .sysInfo = true}, + {.name = "tsdb_pagesize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, }; static const SSysDbTableSchema userFuncSchema[] = { @@ -206,6 +210,7 @@ static const SSysDbTableSchema vgroupsSchema[] = { {.name = "v3_dnode", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "v3_status", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, {.name = "status", .bytes = 12 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "cacheload", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "nfiles", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "file_size", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, {.name = "tsma", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, @@ -256,6 +261,15 @@ static const SSysDbTableSchema subscriptionSchema[] = { {.name = "consumer_id", .bytes = 8, .type = TSDB_DATA_TYPE_BIGINT, .sysInfo = false}, }; +static const SSysDbTableSchema vnodesSchema[] = { + {.name = "vgroup_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "replica", .bytes = 1, .type = TSDB_DATA_TYPE_TINYINT, .sysInfo = true}, + {.name = "status", .bytes = 9 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "db_name", .bytes = SYSTABLE_SCH_DB_NAME_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = true}, + {.name = "dnode_ep", .bytes = TSDB_EP_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = true}, +}; + static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_DNODES, dnodesSchema, tListLen(dnodesSchema), true}, {TSDB_INS_TABLE_MNODES, mnodesSchema, tListLen(mnodesSchema), true}, @@ -279,6 +293,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_TOPICS, topicSchema, tListLen(topicSchema), false}, {TSDB_INS_TABLE_SUBSCRIPTIONS, subscriptionSchema, tListLen(subscriptionSchema), false}, {TSDB_INS_TABLE_STREAMS, streamSchema, tListLen(streamSchema), false}, + {TSDB_INS_TABLE_VNODES, vnodesSchema, tListLen(vnodesSchema), true}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index c7f372f17b3c174290396c0e0ca49229ff8df73b..16b8e55cf7a1fe57d9e984ad6d434297794aa07d 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -140,7 +140,8 @@ int32_t colDataReserve(SColumnInfoData* pColumnInfoData, size_t newSize) { return TSDB_CODE_SUCCESS; } -static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, int32_t itemLen, int32_t numOfRows) { +static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t currentRow, const char* pData, + int32_t itemLen, int32_t numOfRows) { ASSERT(pColumnInfoData->info.bytes >= itemLen); size_t start = 1; @@ -148,21 +149,23 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren memcpy(pColumnInfoData->pData, pData, itemLen); int32_t t = 0; - int32_t count = log(numOfRows)/log(2); - while(t < count) { + int32_t count = log(numOfRows) / log(2); + while (t < count) { int32_t xlen = 1 << t; - memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData, xlen * itemLen); + memcpy(pColumnInfoData->pData + start * itemLen + pColumnInfoData->varmeta.length, pColumnInfoData->pData, + xlen * itemLen); t += 1; start += xlen; } // the tail part if (numOfRows > start) { - memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData, (numOfRows - start) * itemLen); + memcpy(pColumnInfoData->pData + start * itemLen + currentRow * itemLen, pColumnInfoData->pData, + (numOfRows - start) * itemLen); } if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - for(int32_t i = 0; i < numOfRows; ++i) { + for (int32_t i = 0; i < numOfRows; ++i) { pColumnInfoData->varmeta.offset[i + currentRow] = pColumnInfoData->varmeta.length + i * itemLen; } @@ -170,7 +173,8 @@ static void doCopyNItems(struct SColumnInfoData* pColumnInfoData, int32_t curren } } -int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, uint32_t numOfRows) { +int32_t colDataAppendNItems(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, + uint32_t numOfRows) { ASSERT(pData != NULL && pColumnInfoData != NULL); int32_t len = pColumnInfoData->info.bytes; @@ -278,7 +282,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int } else { if (finalNumOfRows > *capacity || (numOfRow1 == 0 && pColumnInfoData->info.bytes != 0)) { // all data may be null, when the pColumnInfoData->info.type == 0, bytes == 0; -// ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); + // ASSERT(finalNumOfRows * pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -557,7 +561,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { } int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { - int32_t numOfRows = *(int32_t*) buf; + int32_t numOfRows = *(int32_t*)buf; blockDataEnsureCapacity(pBlock, numOfRows); pBlock->info.rows = numOfRows; @@ -676,7 +680,8 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { * @return */ size_t blockDataGetSerialMetaSize(uint32_t numOfCols) { - // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column length | + // | version | total length | total rows | total columns | flag seg| block group id | column schema | each column + // length | return sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(int32_t) + sizeof(uint64_t) + numOfCols * (sizeof(int8_t) + sizeof(int32_t)) + numOfCols * sizeof(int32_t); } @@ -1302,6 +1307,40 @@ int32_t copyDataBlock(SSDataBlock* dst, const SSDataBlock* src) { return TSDB_CODE_SUCCESS; } +SSDataBlock* createSpecialDataBlock(EStreamType type) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = false; + pBlock->info.groupId = 0; + pBlock->info.rows = 0; + pBlock->info.type = type; + pBlock->info.rowSize = + sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); + pBlock->info.watermark = INT64_MIN; + + pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; + infoData.info.bytes = sizeof(TSKEY); + // window start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // window end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_UBIGINT; + infoData.info.bytes = sizeof(uint64_t); + // uid + taosArrayPush(pBlock->pDataBlock, &infoData); + // group id + taosArrayPush(pBlock->pDataBlock, &infoData); + + // calculate start ts + taosArrayPush(pBlock->pDataBlock, &infoData); + // calculate end ts + taosArrayPush(pBlock->pDataBlock, &infoData); + + return pBlock; +} + SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { if (pDataBlock == NULL) { return NULL; @@ -1426,7 +1465,7 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { } void colDataDestroy(SColumnInfoData* pColData) { - if(!pColData) return; + if (!pColData) return; if (IS_VAR_DATA_TYPE(pColData->info.type)) { taosMemoryFreeClear(pColData->varmeta.offset); } else { @@ -1693,7 +1732,7 @@ static char* formatTimestamp(char* buf, int64_t val, int precision) { } struct tm ptm = {0}; taosLocalTime(&tt, &ptm); - size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", &ptm); if (precision == TSDB_TIME_PRECISION_NANO) { sprintf(buf + pos, ".%09d", ms); @@ -1847,20 +1886,20 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) break; case TSDB_DATA_TYPE_VARCHAR: { memset(pBuf, 0, sizeof(pBuf)); - char* pData = colDataGetVarData(pColInfoData, j); + char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memcpy(pBuf, varDataVal(pData), dataSize); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) return dumpBuf; - } break; + } break; case TSDB_DATA_TYPE_NCHAR: { - char* pData = colDataGetVarData(pColInfoData, j); + char* pData = colDataGetVarData(pColInfoData, j); int32_t dataSize = TMIN(sizeof(pBuf), varDataLen(pData)); memset(pBuf, 0, sizeof(pBuf)); - taosUcs4ToMbs((TdUcs4 *)varDataVal(pData), dataSize, pBuf); + taosUcs4ToMbs((TdUcs4*)varDataVal(pData), dataSize, pBuf); len += snprintf(dumpBuf + len, size - len, " %15s |", pBuf); if (len >= size - 1) return dumpBuf; - } break; + } break; } } len += snprintf(dumpBuf + len, size - len, "\n"); @@ -1877,7 +1916,7 @@ char* dumpBlockData(SSDataBlock* pDataBlock, const char* flag, char** pDataBuf) * @param pDataBlocks * @param vgId * @param suid - * + * */ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataBlock, STSchema* pTSchema, int32_t vgId, tb_uid_t suid) { @@ -1904,8 +1943,8 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB tdSRowInit(&rb, pTSchema->version); for (int32_t i = 0; i < sz; ++i) { - int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); - int32_t rows = pDataBlock->info.rows; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; // int32_t rowSize = pDataBlock->info.rowSize; // int64_t groupId = pDataBlock->info.groupId; @@ -1926,7 +1965,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SSDataBlock* pDataB msgLen += sizeof(SSubmitBlk); int32_t dataLen = 0; - for (int32_t j = 0; j < rows; ++j) { // iterate by row + for (int32_t j = 0; j < rows; ++j) { // iterate by row tdSRowResetBuf(&rb, POINTER_SHIFT(pDataBuf, msgLen + dataLen)); // set row buf bool isStartKey = false; int32_t offset = 0; @@ -2081,6 +2120,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ int32_t* rows = (int32_t*)data; *rows = pBlock->info.rows; data += sizeof(int32_t); + ASSERT(*rows > 0); int32_t* cols = (int32_t*)data; *cols = numOfCols; @@ -2089,7 +2129,7 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ // flag segment. // the inital bit is for column info int32_t* flagSegment = (int32_t*)data; - *flagSegment = (1<<31); + *flagSegment = (1 << 31); data += sizeof(int32_t); @@ -2144,12 +2184,14 @@ void blockEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_ *actualLen = *dataLen; *groupId = pBlock->info.groupId; + ASSERT(*dataLen > 0); + uDebug("build data block, actualLen:%d, rows:%d, cols:%d", *dataLen, *rows, *cols); } const char* blockDecode(SSDataBlock* pBlock, const char* pData) { const char* pStart = pData; - int32_t version = *(int32_t*) pStart; + int32_t version = *(int32_t*)pStart; pStart += sizeof(int32_t); ASSERT(version == 1); @@ -2158,7 +2200,7 @@ const char* blockDecode(SSDataBlock* pBlock, const char* pData) { pStart += sizeof(int32_t); // total rows sizeof(int32_t) - int32_t numOfRows = *(int32_t*)pStart; + int32_t numOfRows = *(int32_t*)pStart; pStart += sizeof(int32_t); // total columns sizeof(int32_t) diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index b40f449a0550140784250b9c2250d191552e4652..15a369fe404da6e279afe14bdaf3e68ac6a07f8a 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "tdataformat.h" +#include "tRealloc.h" #include "tcoding.h" #include "tdatablock.h" #include "tlog.h" @@ -680,7 +681,7 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 **ppRow) { return n; } -// STSchema +// STSchema ======================================== int32_t tTSchemaCreate(int32_t sver, SSchema *pSchema, int32_t ncols, STSchema **ppTSchema) { *ppTSchema = (STSchema *)taosMemoryMalloc(sizeof(STSchema) + sizeof(STColumn) * ncols); if (*ppTSchema == NULL) { @@ -720,9 +721,7 @@ void tTSchemaDestroy(STSchema *pTSchema) { if (pTSchema) taosMemoryFree(pTSchema); } -// STSRowBuilder - -// STag +// STag ======================================== static int tTagValCmprFn(const void *p1, const void *p2) { if (((STagVal *)p1)->cid < ((STagVal *)p2)->cid) { return -1; @@ -1172,4 +1171,495 @@ STSchema *tdGetSchemaFromBuilder(STSchemaBuilder *pBuilder) { return pSchema; } -#endif \ No newline at end of file +#endif + +// SColData ======================================== +void tColDataDestroy(void *ph) { + SColData *pColData = (SColData *)ph; + + tFree(pColData->pBitMap); + tFree((uint8_t *)pColData->aOffset); + tFree(pColData->pData); +} + +void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { + pColData->cid = cid; + pColData->type = type; + pColData->smaOn = smaOn; + tColDataClear(pColData); +} + +void tColDataClear(SColData *pColData) { + pColData->nVal = 0; + pColData->flag = 0; + pColData->nData = 0; +} + +static FORCE_INLINE int32_t tColDataPutValue(SColData *pColData, SColVal *pColVal) { + int32_t code = 0; + + if (IS_VAR_DATA_TYPE(pColData->type)) { + code = tRealloc((uint8_t **)(&pColData->aOffset), sizeof(int32_t) * (pColData->nVal + 1)); + if (code) goto _exit; + pColData->aOffset[pColData->nVal] = pColData->nData; + + if (pColVal->value.nData) { + code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); + if (code) goto _exit; + memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); + pColData->nData += pColVal->value.nData; + } + } else { + ASSERT(pColData->nData == tDataTypes[pColData->type].bytes * pColData->nVal); + code = tRealloc(&pColData->pData, pColData->nData + tDataTypes[pColData->type].bytes); + if (code) goto _exit; + pColData->nData += tPutValue(pColData->pData + pColData->nData, &pColVal->value, pColVal->type); + } + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue0(SColData *pColData, SColVal *pColVal) { // 0 + int32_t code = 0; + + if (pColVal->isNone) { + pColData->flag = HAS_NONE; + } else if (pColVal->isNull) { + pColData->flag = HAS_NULL; + } else { + pColData->flag = HAS_VALUE; + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue1(SColData *pColData, SColVal *pColVal) { // HAS_NONE + int32_t code = 0; + + if (!pColVal->isNone) { + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + + code = tRealloc(&pColData->pBitMap, nBit); + if (code) goto _exit; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + if (pColVal->isNull) { + pColData->flag |= HAS_NULL; + } else { + pColData->flag |= HAS_VALUE; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) goto _exit; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) goto _exit; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue2(SColData *pColData, SColVal *pColVal) { // HAS_NULL + int32_t code = 0; + + if (!pColVal->isNull) { + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) goto _exit; + + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else { + pColData->flag |= HAS_VALUE; + + memset(pColData->pBitMap, 0, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) goto _exit; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) goto _exit; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue3(SColData *pColData, SColVal *pColVal) { // HAS_NULL|HAS_NONE + int32_t code = 0; + + if (pColVal->isNone) { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else if (pColVal->isNull) { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + } else { + pColData->flag |= HAS_VALUE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal)); + } + SET_BIT2(pBitMap, pColData->nVal, 2); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + + if (pColData->nVal) { + if (IS_VAR_DATA_TYPE(pColData->type)) { + int32_t nOffset = sizeof(int32_t) * pColData->nVal; + code = tRealloc((uint8_t **)(&pColData->aOffset), nOffset); + if (code) goto _exit; + memset(pColData->aOffset, 0, nOffset); + } else { + pColData->nData = tDataTypes[pColData->type].bytes * pColData->nVal; + code = tRealloc(&pColData->pData, pColData->nData); + if (code) goto _exit; + memset(pColData->pData, 0, pColData->nData); + } + } + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue4(SColData *pColData, SColVal *pColVal) { // HAS_VALUE + int32_t code = 0; + + if (pColVal->isNone || pColVal->isNull) { + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + } else { + pColData->flag |= HAS_NULL; + } + + int32_t nBit = BIT1_SIZE(pColData->nVal + 1); + code = tRealloc(&pColData->pBitMap, nBit); + if (code) goto _exit; + + memset(pColData->pBitMap, 255, nBit); + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } else { + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + } + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue5(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NONE + int32_t code = 0; + + if (pColVal->isNull) { + pColData->flag |= HAS_NULL; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 0); + } + SET_BIT2(pBitMap, pColData->nVal, 1); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + } else { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + if (pColVal->isNone) { + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else { + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + } + } + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue6(SColData *pColData, SColVal *pColVal) { // HAS_VALUE|HAS_NULL + int32_t code = 0; + + if (pColVal->isNone) { + pColData->flag |= HAS_NONE; + + uint8_t *pBitMap = NULL; + code = tRealloc(&pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { + SET_BIT2(pBitMap, iVal, GET_BIT1(pColData->pBitMap, iVal) ? 2 : 1); + } + SET_BIT2(pBitMap, pColData->nVal, 0); + + tFree(pColData->pBitMap); + pColData->pBitMap = pBitMap; + } else { + code = tRealloc(&pColData->pBitMap, BIT1_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + if (pColVal->isNull) { + SET_BIT1(pColData->pBitMap, pColData->nVal, 0); + } else { + SET_BIT1(pColData->pBitMap, pColData->nVal, 1); + } + } + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + + pColData->nVal++; + +_exit: + return code; +} +static FORCE_INLINE int32_t tColDataAppendValue7(SColData *pColData, + SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE + int32_t code = 0; + + code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal + 1)); + if (code) goto _exit; + + if (pColVal->isNone) { + SET_BIT2(pColData->pBitMap, pColData->nVal, 0); + } else if (pColVal->isNull) { + SET_BIT2(pColData->pBitMap, pColData->nVal, 1); + } else { + SET_BIT2(pColData->pBitMap, pColData->nVal, 2); + } + code = tColDataPutValue(pColData, pColVal); + if (code) goto _exit; + + pColData->nVal++; + +_exit: + return code; +} +static int32_t (*tColDataAppendValueImpl[])(SColData *pColData, SColVal *pColVal) = { + tColDataAppendValue0, // 0 + tColDataAppendValue1, // HAS_NONE + tColDataAppendValue2, // HAS_NULL + tColDataAppendValue3, // HAS_NULL|HAS_NONE + tColDataAppendValue4, // HAS_VALUE + tColDataAppendValue5, // HAS_VALUE|HAS_NONE + tColDataAppendValue6, // HAS_VALUE|HAS_NULL + tColDataAppendValue7 // HAS_VALUE|HAS_NULL|HAS_NONE +}; +int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { + ASSERT(pColData->cid == pColVal->cid && pColData->type == pColVal->type); + return tColDataAppendValueImpl[pColData->flag](pColData, pColVal); +} + +static FORCE_INLINE void tColDataGetValue1(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NONE + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); +} +static FORCE_INLINE void tColDataGetValue2(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); +} +static FORCE_INLINE void tColDataGetValue3(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_NULL|HAS_NONE + switch (GET_BIT1(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + break; + case 1: + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + break; + default: + ASSERT(0); + } +} +static FORCE_INLINE void tColDataGetValue4(SColData *pColData, int32_t iVal, SColVal *pColVal) { // HAS_VALUE + SValue value; + if (IS_VAR_DATA_TYPE(pColData->type)) { + if (iVal + 1 < pColData->nVal) { + value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal]; + } else { + value.nData = pColData->nData - pColData->aOffset[iVal]; + } + value.pData = pColData->pData + pColData->aOffset[iVal]; + } else { + tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); + } + *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); +} +static FORCE_INLINE void tColDataGetValue5(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_VALUE|HAS_NONE + switch (GET_BIT1(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + break; + case 1: + tColDataGetValue4(pColData, iVal, pColVal); + break; + default: + ASSERT(0); + } +} +static FORCE_INLINE void tColDataGetValue6(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_VALUE|HAS_NULL + switch (GET_BIT1(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + break; + case 1: + tColDataGetValue4(pColData, iVal, pColVal); + break; + default: + ASSERT(0); + } +} +static FORCE_INLINE void tColDataGetValue7(SColData *pColData, int32_t iVal, + SColVal *pColVal) { // HAS_VALUE|HAS_NULL|HAS_NONE + switch (GET_BIT2(pColData->pBitMap, iVal)) { + case 0: + *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); + break; + case 1: + *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); + break; + case 2: + tColDataGetValue4(pColData, iVal, pColVal); + break; + default: + ASSERT(0); + } +} +static void (*tColDataGetValueImpl[])(SColData *pColData, int32_t iVal, SColVal *pColVal) = { + NULL, // 0 + tColDataGetValue1, // HAS_NONE + tColDataGetValue2, // HAS_NULL + tColDataGetValue3, // HAS_NULL | HAS_NONE + tColDataGetValue4, // HAS_VALUE + tColDataGetValue5, // HAS_VALUE | HAS_NONE + tColDataGetValue6, // HAS_VALUE | HAS_NULL + tColDataGetValue7 // HAS_VALUE | HAS_NULL | HAS_NONE +}; +void tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { + ASSERT(iVal >= 0 && iVal < pColData->nVal && pColData->flag); + tColDataGetValueImpl[pColData->flag](pColData, iVal, pColVal); +} + +uint8_t tColDataGetBitValue(SColData *pColData, int32_t iVal) { + uint8_t v; + switch (pColData->flag) { + case HAS_NONE: + v = 0; + break; + case HAS_NULL: + v = 1; + break; + case (HAS_NULL | HAS_NONE): + v = GET_BIT1(pColData->pBitMap, iVal); + break; + case HAS_VALUE: + v = 2; + break; + case (HAS_VALUE | HAS_NONE): + v = GET_BIT1(pColData->pBitMap, iVal); + if (v) v = 2; + break; + case (HAS_VALUE | HAS_NULL): + v = GET_BIT1(pColData->pBitMap, iVal) + 1; + break; + case (HAS_VALUE | HAS_NULL | HAS_NONE): + v = GET_BIT2(pColData->pBitMap, iVal); + break; + default: + ASSERT(0); + break; + } + return v; +} + +int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { + int32_t code = 0; + int32_t size; + + ASSERT(pColDataSrc->nVal > 0); + ASSERT(pColDataDest->cid = pColDataSrc->cid); + ASSERT(pColDataDest->type = pColDataSrc->type); + + pColDataDest->smaOn = pColDataSrc->smaOn; + pColDataDest->nVal = pColDataSrc->nVal; + pColDataDest->flag = pColDataSrc->flag; + + // bitmap + if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { + size = BIT2_SIZE(pColDataSrc->nVal); + code = tRealloc(&pColDataDest->pBitMap, size); + if (code) goto _exit; + memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); + } + + // offset + if (IS_VAR_DATA_TYPE(pColDataDest->type)) { + size = sizeof(int32_t) * pColDataSrc->nVal; + + code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); + if (code) goto _exit; + + memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); + } + + // value + pColDataDest->nData = pColDataSrc->nData; + code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); + if (code) goto _exit; + memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); + +_exit: + return code; +} \ No newline at end of file diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index add19ae33718905952b62ae2d73bba8ef0963939..562364d5449e6a408d340773d51c0341cc11d15a 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -63,7 +63,7 @@ int32_t tsNumOfVnodeWriteThreads = 2; int32_t tsNumOfVnodeSyncThreads = 2; int32_t tsNumOfVnodeRsmaThreads = 2; int32_t tsNumOfQnodeQueryThreads = 4; -int32_t tsNumOfQnodeFetchThreads = 4; +int32_t tsNumOfQnodeFetchThreads = 1; int32_t tsNumOfSnodeSharedThreads = 2; int32_t tsNumOfSnodeUniqueThreads = 2; @@ -129,10 +129,6 @@ int32_t tsMinIntervalTime = 1; int32_t tsQueryBufferSize = -1; int64_t tsQueryBufferSizeBytes = -1; -// tsdb config -// For backward compatibility -bool tsdbForceKeepFile = false; - int32_t tsDiskCfgNum = 0; SDiskCfg tsDiskCfg[TFS_MAX_DISKS] = {0}; @@ -389,9 +385,9 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { tsNumOfQnodeQueryThreads = TMAX(tsNumOfQnodeQueryThreads, 4); if (cfgAddInt32(pCfg, "numOfQnodeQueryThreads", tsNumOfQnodeQueryThreads, 1, 1024, 0) != 0) return -1; - tsNumOfQnodeFetchThreads = tsNumOfCores / 2; - tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); - if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; +// tsNumOfQnodeFetchThreads = tsNumOfCores / 2; +// tsNumOfQnodeFetchThreads = TMAX(tsNumOfQnodeFetchThreads, 4); +// if (cfgAddInt32(pCfg, "numOfQnodeFetchThreads", tsNumOfQnodeFetchThreads, 1, 1024, 0) != 0) return -1; tsNumOfSnodeSharedThreads = tsNumOfCores / 4; tsNumOfSnodeSharedThreads = TRANGE(tsNumOfSnodeSharedThreads, 2, 4); @@ -531,6 +527,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->stype = stype; } +/* pItem = cfgGetItem(tsCfg, "numOfQnodeFetchThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { tsNumOfQnodeFetchThreads = numOfCores / 2; @@ -538,6 +535,7 @@ static int32_t taosUpdateServerCfg(SConfig *pCfg) { pItem->i32 = tsNumOfQnodeFetchThreads; pItem->stype = stype; } +*/ pItem = cfgGetItem(tsCfg, "numOfSnodeSharedThreads"); if (pItem != NULL && pItem->stype == CFG_STYPE_DEFAULT) { @@ -695,7 +693,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsNumOfVnodeSyncThreads = cfgGetItem(pCfg, "numOfVnodeSyncThreads")->i32; tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; - tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; +// tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; tsNumOfSnodeUniqueThreads = cfgGetItem(pCfg, "numOfSnodeUniqueThreads")->i32; tsRpcQueueMemoryAllowed = cfgGetItem(pCfg, "rpcQueueMemoryAllowed")->i64; @@ -943,8 +941,10 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { tsNumOfVnodeRsmaThreads = cfgGetItem(pCfg, "numOfVnodeRsmaThreads")->i32; } else if (strcasecmp("numOfQnodeQueryThreads", name) == 0) { tsNumOfQnodeQueryThreads = cfgGetItem(pCfg, "numOfQnodeQueryThreads")->i32; +/* } else if (strcasecmp("numOfQnodeFetchThreads", name) == 0) { tsNumOfQnodeFetchThreads = cfgGetItem(pCfg, "numOfQnodeFetchThreads")->i32; +*/ } else if (strcasecmp("numOfSnodeSharedThreads", name) == 0) { tsNumOfSnodeSharedThreads = cfgGetItem(pCfg, "numOfSnodeSharedThreads")->i32; } else if (strcasecmp("numOfSnodeUniqueThreads", name) == 0) { @@ -1133,7 +1133,7 @@ int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDi taosSetAllDebugFlag(cfgGetItem(pCfg, "debugFlag")->i32, false); - if (taosMulMkDir(tsLogDir) != 0) { + if (taosMulModeMkDir(tsLogDir, 0777) != 0) { uError("failed to create dir:%s since %s", tsLogDir, terrstr()); cfgCleanup(pCfg); return -1; diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index 3b29c17883f5949f7474f099649ae2273919f0ef..bae43ae5c460a76419b9567c7dc7aab648ce537b 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -994,6 +994,7 @@ int32_t tSerializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SVnodeLoad *pload = taosArrayGet(pReq->pVloads, i); if (tEncodeI32(&encoder, pload->vgId) < 0) return -1; if (tEncodeI32(&encoder, pload->syncState) < 0) return -1; + if (tEncodeI64(&encoder, pload->cacheUsage) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pload->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pload->totalStorage) < 0) return -1; @@ -1063,6 +1064,7 @@ int32_t tDeserializeSStatusReq(void *buf, int32_t bufLen, SStatusReq *pReq) { SVnodeLoad vload = {0}; if (tDecodeI32(&decoder, &vload.vgId) < 0) return -1; if (tDecodeI32(&decoder, &vload.syncState) < 0) return -1; + if (tDecodeI64(&decoder, &vload.cacheUsage) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &vload.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &vload.totalStorage) < 0) return -1; @@ -2024,6 +2026,9 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI64(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRollPeriod) < 0) return -1; if (tEncodeI64(&encoder, pReq->walSegmentSize) < 0) return -1; + if (tEncodeI32(&encoder, pReq->sstTrigger) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; if (tEncodeI8(&encoder, pReq->ignoreExist) < 0) return -1; if (tEncodeI32(&encoder, pReq->numOfRetensions) < 0) return -1; for (int32_t i = 0; i < pReq->numOfRetensions; ++i) { @@ -2033,6 +2038,7 @@ int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { if (tEncodeI8(&encoder, pRetension->freqUnit) < 0) return -1; if (tEncodeI8(&encoder, pRetension->keepUnit) < 0) return -1; } + if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2070,6 +2076,9 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) if (tDecodeI64(&decoder, &pReq->walRetentionSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->walRollPeriod) < 0) return -1; if (tDecodeI64(&decoder, &pReq->walSegmentSize) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->sstTrigger) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; if (tDecodeI8(&decoder, &pReq->ignoreExist) < 0) return -1; if (tDecodeI32(&decoder, &pReq->numOfRetensions) < 0) return -1; pReq->pRetensions = taosArrayInit(pReq->numOfRetensions, sizeof(SRetention)); @@ -2090,6 +2099,8 @@ int32_t tDeserializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) } } + if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1; + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2120,6 +2131,7 @@ int32_t tSerializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tEncodeI8(&encoder, pReq->strict) < 0) return -1; if (tEncodeI8(&encoder, pReq->cacheLast) < 0) return -1; if (tEncodeI8(&encoder, pReq->replications) < 0) return -1; + if (tEncodeI32(&encoder, pReq->sstTrigger) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -2146,6 +2158,7 @@ int32_t tDeserializeSAlterDbReq(void *buf, int32_t bufLen, SAlterDbReq *pReq) { if (tDecodeI8(&decoder, &pReq->strict) < 0) return -1; if (tDecodeI8(&decoder, &pReq->cacheLast) < 0) return -1; if (tDecodeI8(&decoder, &pReq->replications) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->sstTrigger) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -2453,6 +2466,8 @@ int32_t tSerializeSUseDbRspImp(SEncoder *pEncoder, const SUseDbRsp *pRsp) { if (tEncodeI64(pEncoder, pRsp->uid) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgVersion) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->vgNum) < 0) return -1; + if (tEncodeI16(pEncoder, pRsp->hashPrefix) < 0) return -1; + if (tEncodeI16(pEncoder, pRsp->hashSuffix) < 0) return -1; if (tEncodeI8(pEncoder, pRsp->hashMethod) < 0) return -1; for (int32_t i = 0; i < pRsp->vgNum; ++i) { @@ -2504,6 +2519,8 @@ int32_t tDeserializeSUseDbRspImp(SDecoder *pDecoder, SUseDbRsp *pRsp) { if (tDecodeI64(pDecoder, &pRsp->uid) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgVersion) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->vgNum) < 0) return -1; + if (tDecodeI16(pDecoder, &pRsp->hashPrefix) < 0) return -1; + if (tDecodeI16(pDecoder, &pRsp->hashSuffix) < 0) return -1; if (tDecodeI8(pDecoder, &pRsp->hashMethod) < 0) return -1; if (pRsp->vgNum <= 0) { @@ -3330,7 +3347,13 @@ int32_t tDeserializeSSTbHbRsp(void *buf, int32_t bufLen, SSTbHbRsp *pRsp) { return 0; } -void tFreeSTableMetaRsp(void *pRsp) { taosMemoryFreeClear(((STableMetaRsp *)pRsp)->pSchemas); } +void tFreeSTableMetaRsp(void *pRsp) { + if (NULL == pRsp) { + return; + } + + taosMemoryFreeClear(((STableMetaRsp *)pRsp)->pSchemas); +} void tFreeSTableIndexRsp(void *info) { if (NULL == info) { @@ -3762,6 +3785,10 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR if (tEncodeI64(&encoder, pReq->walRetentionSize) < 0) return -1; if (tEncodeI32(&encoder, pReq->walRollPeriod) < 0) return -1; if (tEncodeI64(&encoder, pReq->walSegmentSize) < 0) return -1; + if (tEncodeI16(&encoder, pReq->sstTrigger) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashPrefix) < 0) return -1; + if (tEncodeI16(&encoder, pReq->hashSuffix) < 0) return -1; + if (tEncodeI32(&encoder, pReq->tsdbPageSize) < 0) return -1; tEndEncode(&encoder); @@ -3834,6 +3861,10 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * if (tDecodeI64(&decoder, &pReq->walRetentionSize) < 0) return -1; if (tDecodeI32(&decoder, &pReq->walRollPeriod) < 0) return -1; if (tDecodeI64(&decoder, &pReq->walSegmentSize) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->sstTrigger) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashPrefix) < 0) return -1; + if (tDecodeI16(&decoder, &pReq->hashSuffix) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->tsdbPageSize) < 0) return -1; tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4711,9 +4742,8 @@ int32_t tSerializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tEncodeU64(&encoder, pReq->queryId) < 0) return -1; if (tEncodeU64(&encoder, pReq->taskId) < 0) return -1; if (tEncodeU32(&encoder, pReq->sqlLen) < 0) return -1; - if (tEncodeU32(&encoder, pReq->phyLen) < 0) return -1; if (tEncodeCStr(&encoder, pReq->sql) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->msg) < 0) return -1; + if (tEncodeBinary(&encoder, pReq->msg, pReq->phyLen) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4743,13 +4773,12 @@ int32_t tDeserializeSVDeleteReq(void *buf, int32_t bufLen, SVDeleteReq *pReq) { if (tDecodeU64(&decoder, &pReq->queryId) < 0) return -1; if (tDecodeU64(&decoder, &pReq->taskId) < 0) return -1; if (tDecodeU32(&decoder, &pReq->sqlLen) < 0) return -1; - if (tDecodeU32(&decoder, &pReq->phyLen) < 0) return -1; pReq->sql = taosMemoryCalloc(1, pReq->sqlLen + 1); if (NULL == pReq->sql) return -1; - pReq->msg = taosMemoryCalloc(1, pReq->phyLen + 1); - if (NULL == pReq->msg) return -1; if (tDecodeCStrTo(&decoder, pReq->sql) < 0) return -1; - if (tDecodeCStrTo(&decoder, pReq->msg) < 0) return -1; + uint64_t msgLen = 0; + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->msg, &msgLen) < 0) return -1; + pReq->phyLen = msgLen; tEndDecode(&decoder); @@ -5429,6 +5458,8 @@ void tFreeSSubmitRsp(SSubmitRsp *pRsp) { for (int32_t i = 0; i < pRsp->nBlocks; ++i) { SSubmitBlkRsp *sRsp = pRsp->pBlocks + i; taosMemoryFree(sRsp->tblFName); + tFreeSTableMetaRsp(sRsp->pMeta); + taosMemoryFree(sRsp->pMeta); } taosMemoryFree(pRsp->pBlocks); diff --git a/source/common/src/trow.c b/source/common/src/trow.c index 565498a47bb28bc5e76bb5a33911c86f8f42da5d..9880fe362eb55e909842cb0e9aec007667426c71 100644 --- a/source/common/src/trow.c +++ b/source/common/src/trow.c @@ -538,12 +538,12 @@ bool tdSTSRowIterGetTpVal(STSRowIter *pIter, col_type_t colType, int32_t offset, } else { pVal->val = POINTER_SHIFT(TD_ROW_DATA(pRow), offset); } - return TSDB_CODE_SUCCESS; + return true; } if (tdGetBitmapValType(pIter->pBitmap, pIter->colIdx - 1, &pVal->valType, 0) != TSDB_CODE_SUCCESS) { pVal->valType = TD_VTYPE_NONE; - return terrno; + return true; } if (pVal->valType == TD_VTYPE_NORM) { diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index e610b41a04dc7792638a251fa379bcacb37e0050..e3ad1b938979e3012a03ae7903450dd6630442d5 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -167,9 +167,13 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->walCfg.segSize = pCreate->walSegmentSize; pCfg->walCfg.level = pCreate->walLevel; + pCfg->sttTrigger = pCreate->sstTrigger; pCfg->hashBegin = pCreate->hashBegin; pCfg->hashEnd = pCreate->hashEnd; pCfg->hashMethod = pCreate->hashMethod; + pCfg->hashPrefix = pCreate->hashPrefix; + pCfg->hashSuffix = pCreate->hashSuffix; + pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->standby = pCfg->standby; pCfg->syncCfg.myIndex = pCreate->selfIndex; @@ -219,8 +223,13 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dDebug("vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d", createReq.vgId, - createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize); + dInfo( + "vgId:%d, start to create vnode, tsma:%d standby:%d cacheLast:%d cacheLastSize:%d sstTrigger:%d " + "tsdbPageSize:%d", + createReq.vgId, createReq.isTsma, createReq.standby, createReq.cacheLast, createReq.cacheLastSize, + createReq.sstTrigger, createReq.tsdbPageSize); + dInfo("vgId:%d, hashMethod:%d begin:%u end:%u prefix:%d surfix:%d", createReq.vgId, createReq.hashMethod, + createReq.hashBegin, createReq.hashEnd, createReq.hashPrefix, createReq.hashSuffix); vmGenerateVnodeCfg(&createReq, &vnodeCfg); if (vmTsmaAdjustDays(&vnodeCfg, &createReq) < 0) { diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index ea05215fe90d30708013fe4b1c8fc08d2be8d3d6..cd08ee4ad5210d9ce36e391c346a681a82abd373 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -305,11 +305,15 @@ typedef struct { int8_t hashMethod; // default is 1 int8_t cacheLast; int8_t schemaless; + int16_t hashPrefix; + int16_t hashSuffix; + int16_t sstTrigger; + int32_t tsdbPageSize; int32_t numOfRetensions; SArray* pRetensions; int32_t walRetentionPeriod; - int64_t walRetentionSize; int32_t walRollPeriod; + int64_t walRetentionSize; int64_t walSegmentSize; } SDbCfg; @@ -340,6 +344,7 @@ typedef struct { uint32_t hashEnd; char dbName[TSDB_DB_FNAME_LEN]; int64_t dbUid; + int64_t cacheUsage; int64_t numOfTables; int64_t numOfTimeSeries; int64_t totalStorage; diff --git a/source/dnode/mnode/impl/src/mndDb.c b/source/dnode/mnode/impl/src/mndDb.c index 8c1c3ba8735f21684b5f9577b9fad20beec110a7..d42ba1f7ffc45f0fb70457c01c26b35ef206454f 100644 --- a/source/dnode/mnode/impl/src/mndDb.c +++ b/source/dnode/mnode/impl/src/mndDb.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "mndDb.h" +#include "mndCluster.h" #include "mndDnode.h" #include "mndOffset.h" #include "mndPrivilege.h" @@ -30,7 +31,7 @@ #include "systable.h" #define DB_VER_NUMBER 1 -#define DB_RESERVE_SIZE 64 +#define DB_RESERVE_SIZE 54 static SSdbRaw *mndDbActionEncode(SDbObj *pDb); static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw); @@ -124,6 +125,10 @@ static SSdbRaw *mndDbActionEncode(SDbObj *pDb) { SDB_SET_INT64(pRaw, dataPos, pDb->cfg.walRetentionSize, _OVER) SDB_SET_INT32(pRaw, dataPos, pDb->cfg.walRollPeriod, _OVER) SDB_SET_INT64(pRaw, dataPos, pDb->cfg.walSegmentSize, _OVER) + SDB_SET_INT16(pRaw, dataPos, pDb->cfg.sstTrigger, _OVER) + SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashPrefix, _OVER) + SDB_SET_INT16(pRaw, dataPos, pDb->cfg.hashSuffix, _OVER) + SDB_SET_INT32(pRaw, dataPos, pDb->cfg.tsdbPageSize, _OVER) SDB_SET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -207,10 +212,23 @@ static SSdbRow *mndDbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pDb->cfg.walRetentionSize, _OVER) SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.walRollPeriod, _OVER) SDB_GET_INT64(pRaw, dataPos, &pDb->cfg.walSegmentSize, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.sstTrigger, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashPrefix, _OVER) + SDB_GET_INT16(pRaw, dataPos, &pDb->cfg.hashSuffix, _OVER) + SDB_GET_INT32(pRaw, dataPos, &pDb->cfg.tsdbPageSize, _OVER) SDB_GET_RESERVE(pRaw, dataPos, DB_RESERVE_SIZE, _OVER) taosInitRWLatch(&pDb->lock); + if (pDb->cfg.tsdbPageSize <= TSDB_MIN_TSDB_PAGESIZE) { + mInfo("db:%s, tsdbPageSize set from %d to default %d", pDb->name, pDb->cfg.tsdbPageSize, + TSDB_DEFAULT_TSDB_PAGESIZE); + } + + if (pDb->cfg.sstTrigger <= TSDB_MIN_STT_TRIGGER) { + mInfo("db:%s, sstTrigger set from %d to default %d", pDb->name, pDb->cfg.sstTrigger, TSDB_DEFAULT_SST_TRIGGER); + } + terrno = 0; _OVER: @@ -254,6 +272,8 @@ static int32_t mndDbActionUpdate(SSdb *pSdb, SDbObj *pOld, SDbObj *pNew) { pOld->cfg.strict = pNew->cfg.strict; pOld->cfg.cacheLast = pNew->cfg.cacheLast; pOld->cfg.replications = pNew->cfg.replications; + pOld->cfg.sstTrigger = pNew->cfg.sstTrigger; + pOld->cfg.tsdbPageSize = pNew->cfg.tsdbPageSize; taosWUnLockLatch(&pOld->lock); return 0; } @@ -330,6 +350,10 @@ static int32_t mndCheckDbCfg(SMnode *pMnode, SDbCfg *pCfg) { if (pCfg->walRetentionSize < TSDB_DB_MIN_WAL_RETENTION_SIZE) return -1; if (pCfg->walRollPeriod < TSDB_DB_MIN_WAL_ROLL_PERIOD) return -1; if (pCfg->walSegmentSize < TSDB_DB_MIN_WAL_SEGMENT_SIZE) return -1; + if (pCfg->sstTrigger < TSDB_MIN_STT_TRIGGER || pCfg->sstTrigger > TSDB_MAX_STT_TRIGGER) return -1; + if (pCfg->hashPrefix < TSDB_MIN_HASH_PREFIX || pCfg->hashPrefix > TSDB_MAX_HASH_PREFIX) return -1; + if (pCfg->hashSuffix < TSDB_MIN_HASH_SUFFIX || pCfg->hashSuffix > TSDB_MAX_HASH_SUFFIX) return -1; + if (pCfg->tsdbPageSize < TSDB_MIN_TSDB_PAGESIZE || pCfg->tsdbPageSize > TSDB_MAX_TSDB_PAGESIZE) return -1; terrno = 0; return terrno; @@ -358,11 +382,15 @@ static void mndSetDefaultDbCfg(SDbCfg *pCfg) { if (pCfg->numOfRetensions < 0) pCfg->numOfRetensions = 0; if (pCfg->schemaless < 0) pCfg->schemaless = TSDB_DB_SCHEMALESS_OFF; if (pCfg->walRetentionPeriod < 0 && pCfg->walRetentionPeriod != -1) - pCfg->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD; + pCfg->walRetentionPeriod = TSDB_REPS_DEF_DB_WAL_RET_PERIOD; if (pCfg->walRetentionSize < 0 && pCfg->walRetentionSize != -1) - pCfg->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE; - if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD; + pCfg->walRetentionSize = TSDB_REPS_DEF_DB_WAL_RET_SIZE; + if (pCfg->walRollPeriod < 0) pCfg->walRollPeriod = TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD; if (pCfg->walSegmentSize < 0) pCfg->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; + if (pCfg->sstTrigger <= 0) pCfg->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; + if (pCfg->hashPrefix < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; + if (pCfg->hashSuffix < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; + if (pCfg->tsdbPageSize <= 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; } static int32_t mndSetCreateDbRedoLogs(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroups) { @@ -479,6 +507,10 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, .walRetentionSize = pCreate->walRetentionSize, .walRollPeriod = pCreate->walRollPeriod, .walSegmentSize = pCreate->walSegmentSize, + .sstTrigger = pCreate->sstTrigger, + .hashPrefix = pCreate->hashPrefix, + .hashSuffix = pCreate->hashSuffix, + .tsdbPageSize = pCreate->tsdbPageSize, }; dbObj.cfg.numOfRetensions = pCreate->numOfRetensions; @@ -496,6 +528,12 @@ static int32_t mndCreateDb(SMnode *pMnode, SRpcMsg *pReq, SCreateDbReq *pCreate, return -1; } + if (dbObj.cfg.hashPrefix > 0) { + int32_t dbLen = strlen(dbObj.name) + 1; + mInfo("db:%s, hashPrefix adjust from %d to %d", dbObj.name, dbObj.cfg.hashPrefix, dbObj.cfg.hashPrefix + dbLen); + dbObj.cfg.hashPrefix += dbLen; + } + SVgObj *pVgroups = NULL; if (mndAllocVgroup(pMnode, &dbObj, &pVgroups) != 0) { mError("db:%s, failed to create since %s", pCreate->db, terrstr()); @@ -1155,6 +1193,8 @@ int32_t mndExtractDbInfo(SMnode *pMnode, SDbObj *pDb, SUseDbRsp *pRsp, const SUs pRsp->vgVersion = pDb->vgVersion; pRsp->vgNum = taosArrayGetSize(pRsp->pVgroupInfos); pRsp->hashMethod = pDb->cfg.hashMethod; + pRsp->hashPrefix = pDb->cfg.hashPrefix; + pRsp->hashSuffix = pDb->cfg.hashSuffix; return 0; } @@ -1287,6 +1327,8 @@ int32_t mndValidateDbInfo(SMnode *pMnode, SDbVgVersion *pDbs, int32_t numOfDbs, usedbRsp.vgVersion = pDb->vgVersion; usedbRsp.vgNum = (int32_t)taosArrayGetSize(usedbRsp.pVgroupInfos); usedbRsp.hashMethod = pDb->cfg.hashMethod; + usedbRsp.hashPrefix = pDb->cfg.hashPrefix; + usedbRsp.hashSuffix = pDb->cfg.hashSuffix; taosArrayPush(batchUseRsp.pArray, &usedbRsp); mndReleaseDb(pMnode, pDb); @@ -1536,6 +1578,24 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, STR_WITH_MAXSIZE_TO_VARSTR(buf, "NULL", bytes); } + const char *precStr = NULL; + switch (pDb->cfg.precision) { + case TSDB_TIME_PRECISION_MILLI: + precStr = TSDB_TIME_PRECISION_MILLI_STR; + break; + case TSDB_TIME_PRECISION_MICRO: + precStr = TSDB_TIME_PRECISION_MICRO_STR; + break; + case TSDB_TIME_PRECISION_NANO: + precStr = TSDB_TIME_PRECISION_NANO_STR; + break; + default: + precStr = "none"; + break; + } + char precVstr[10] = {0}; + STR_WITH_SIZE_TO_VARSTR(precVstr, precStr, 2); + char *statusStr = "ready"; if (objStatus == SDB_STATUS_CREATING) { statusStr = "creating"; @@ -1546,7 +1606,6 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, statusStr = "unsynced"; } } - char statusVstr[24] = {0}; STR_WITH_SIZE_TO_VARSTR(statusVstr, statusStr, strlen(statusStr)); @@ -1555,8 +1614,12 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); if (i == 0) { colDataAppend(pColInfo, rows, buf, false); + } else if (i == 1) { + colDataAppend(pColInfo, rows, (const char *)&pDb->createdTime, false); } else if (i == 3) { colDataAppend(pColInfo, rows, (const char *)&numOfTables, false); + } else if (i == 14) { + colDataAppend(pColInfo, rows, precVstr, false); } else if (i == 15) { colDataAppend(pColInfo, rows, statusVstr, false); } else { @@ -1621,23 +1684,6 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.compression, false); - const char *precStr = NULL; - switch (pDb->cfg.precision) { - case TSDB_TIME_PRECISION_MILLI: - precStr = TSDB_TIME_PRECISION_MILLI_STR; - break; - case TSDB_TIME_PRECISION_MICRO: - precStr = TSDB_TIME_PRECISION_MICRO_STR; - break; - case TSDB_TIME_PRECISION_NANO: - precStr = TSDB_TIME_PRECISION_NANO_STR; - break; - default: - precStr = "none"; - break; - } - char precVstr[10] = {0}; - STR_WITH_SIZE_TO_VARSTR(precVstr, precStr, 2); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)precVstr, false); @@ -1682,23 +1728,39 @@ static void mndDumpDbInfoData(SMnode *pMnode, SSDataBlock *pBlock, SDbObj *pDb, pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.walSegmentSize, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.sstTrigger, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + int16_t hashPrefix = pDb->cfg.hashPrefix; + if (hashPrefix > 0) { + hashPrefix = pDb->cfg.hashPrefix - strlen(pDb->name) - 1; + } + colDataAppend(pColInfo, rows, (const char *)&hashPrefix, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.hashSuffix, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, rows, (const char *)&pDb->cfg.tsdbPageSize, false); } taosMemoryFree(buf); } -static void setInformationSchemaDbCfg(SDbObj *pDbObj) { +static void setInformationSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) { tstrncpy(pDbObj->name, TSDB_INFORMATION_SCHEMA_DB, tListLen(pDbObj->name)); - pDbObj->createdTime = 0; + pDbObj->createdTime = mndGetClusterCreateTime(pMnode); pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; pDbObj->cfg.precision = TSDB_TIME_PRECISION_MILLI; } -static void setPerfSchemaDbCfg(SDbObj *pDbObj) { +static void setPerfSchemaDbCfg(SMnode *pMnode, SDbObj *pDbObj) { tstrncpy(pDbObj->name, TSDB_PERFORMANCE_SCHEMA_DB, tListLen(pDbObj->name)); - pDbObj->createdTime = 0; + pDbObj->createdTime = mndGetClusterCreateTime(pMnode); pDbObj->cfg.numOfVgroups = 0; pDbObj->cfg.strict = 1; pDbObj->cfg.replications = 1; @@ -1729,7 +1791,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc // Append the information_schema database into the result. if (!pShow->sysDbRsp) { SDbObj infoschemaDb = {0}; - setInformationSchemaDbCfg(&infoschemaDb); + setInformationSchemaDbCfg(pMnode, &infoschemaDb); size_t numOfTables = 0; getVisibleInfosTablesNum(sysinfo, &numOfTables); mndDumpDbInfoData(pMnode, pBlock, &infoschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); @@ -1737,7 +1799,7 @@ static int32_t mndRetrieveDbs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBloc numOfRows += 1; SDbObj perfschemaDb = {0}; - setPerfSchemaDbCfg(&perfschemaDb); + setPerfSchemaDbCfg(pMnode, &perfschemaDb); numOfTables = 0; getPerfDbMeta(NULL, &numOfTables); mndDumpDbInfoData(pMnode, pBlock, &perfschemaDb, pShow, numOfRows, numOfTables, true, 0, 1); diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index fc5e20ef288d733927499484675acddd042fb3ca..26b4080d14288bee23740394fa4e96755b90feb9 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -347,6 +347,7 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SVgObj *pVgroup = mndAcquireVgroup(pMnode, pVload->vgId); if (pVgroup != NULL) { if (pVload->syncState == TAOS_SYNC_STATE_LEADER) { + pVgroup->cacheUsage = pVload->cacheUsage; pVgroup->numOfTables = pVload->numOfTables; pVgroup->numOfTimeSeries = pVload->numOfTimeSeries; pVgroup->totalStorage = pVload->totalStorage; @@ -853,8 +854,8 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { } int32_t code = -1; - SSdb *pSdb = pMnode->pSdb; - void *pIter = NULL; + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; while (1) { SDnodeObj *pDnode = NULL; pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pDnode); @@ -877,7 +878,7 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { sdbRelease(pSdb, pDnode); } - + if (code == -1) { terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 3bfd7eb5964a446698556551bbe572f2dc568110..511a84290dd2a311f59cfc9467ff2619d92013cb 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -319,7 +319,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { bool multiTarget = pDbObj->cfg.numOfVgroups > 1; - if (planTotLevel == 2 || externalTargetDB || multiTarget) { + if (planTotLevel == 2 || externalTargetDB || multiTarget || pStream->fixedSinkVgId) { /*if (true) {*/ SArray* taskOneLevel = taosArrayInit(0, sizeof(void*)); taosArrayPush(pStream->tasks, &taskOneLevel); diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 1452c5ae2fd3e9cde7cb9052d22e10bfd31afb0f..8a968712e085b989edb88967405c278ce9349899 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -900,6 +900,7 @@ int32_t mndDropSubByTopic(SMnode *pMnode, STrans *pTrans, const char *topicName) // iter all vnode to delete handle if (taosHashGetSize(pSub->consumerHash) != 0) { sdbRelease(pSdb, pSub); + terrno = TSDB_CODE_MND_IN_REBALANCE; return -1; } int32_t sz = taosArrayGetSize(pSub->unassignedVgs); diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index eb072d013d0024e5b05a172c3c3d5d55ce41cd40..7b36966d6c03cb5c3c2d2fe174bffcad340977b5 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -713,7 +713,6 @@ static int32_t mndProcessDropTopicReq(SRpcMsg *pReq) { mndReleaseTopic(pMnode, pTopic); if (code != 0) { - terrno = code; mError("topic:%s, failed to drop since %s", dropReq.name, terrstr()); return -1; } diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 09eed7fb32e8831e6b6c863b44edd3e9e28110a3..a95bdef32302e04ff826d613e33a4fca7ede7f33 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -234,6 +234,10 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.walRetentionSize = pDb->cfg.walRetentionSize; createReq.walRollPeriod = pDb->cfg.walRollPeriod; createReq.walSegmentSize = pDb->cfg.walSegmentSize; + createReq.sstTrigger = pDb->cfg.sstTrigger; + createReq.hashPrefix = pDb->cfg.hashPrefix; + createReq.hashSuffix = pDb->cfg.hashSuffix; + createReq.tsdbPageSize = pDb->cfg.tsdbPageSize; for (int32_t v = 0; v < pVgroup->replica; ++v) { SReplica *pReplica = &createReq.replicas[v]; @@ -693,6 +697,9 @@ static int32_t mndRetrieveVgroups(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppendNULL(pColInfo, numOfRows); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->cacheUsage, false); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppendNULL(pColInfo, numOfRows); @@ -791,32 +798,43 @@ static int32_t mndRetrieveVnodes(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pB if (pShow->pIter == NULL) break; for (int32_t i = 0; i < pVgroup->replica && numOfRows < rows; ++i) { - SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SVnodeGid *pVgid = &pVgroup->vnodeGid[i]; + SColumnInfoData *pColInfo = NULL; cols = 0; - SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); - - SName name = {0}; - char db[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; - tNameFromString(&name, pVgroup->dbName, T_NAME_ACCT | T_NAME_DB); - tNameGetDbName(&name, varDataVal(db)); - varDataSetLen(db, strlen(varDataVal(db))); - pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)db, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->vgId, false); - uint32_t val = 0; pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&val, false); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); char buf[20] = {0}; STR_TO_VARSTR(buf, syncStr(pVgid->role)); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf, false); + const char *dbname = mndGetDbStr(pVgroup->dbName); + char b1[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + if (dbname != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(b1, dbname, TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(b1, "NULL", TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE); + } + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)b1, false); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataAppend(pColInfo, numOfRows, (const char *)&pVgid->dnodeId, false); + + SDnodeObj *pDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); + char b2[TSDB_EP_LEN + VARSTR_HEADER_SIZE] = {0}; + if (pDnode != NULL) { + STR_WITH_MAXSIZE_TO_VARSTR(b2, pDnode->ep, TSDB_EP_LEN + VARSTR_HEADER_SIZE); + } else { + STR_WITH_MAXSIZE_TO_VARSTR(b2, "NULL", TSDB_EP_LEN + VARSTR_HEADER_SIZE); + } pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataAppend(pColInfo, numOfRows, (const char *)&pVgroup->replica, false); // onlines + colDataAppend(pColInfo, numOfRows, (const char *)b2, false); numOfRows++; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index a3e17f53774c82ea9fca1ff0a88943c8e7971725..7a99d26683bf08b2595c51ff9c34b4aea588f800 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -29,6 +29,7 @@ target_sources( # sma "src/sma/smaEnv.c" "src/sma/smaUtil.c" + "src/sma/smaFS.c" "src/sma/smaOpen.c" "src/sma/smaCommit.c" "src/sma/smaRollup.c" @@ -49,6 +50,10 @@ target_sources( "src/tsdb/tsdbSnapshot.c" "src/tsdb/tsdbCacheRead.c" "src/tsdb/tsdbRetention.c" + "src/tsdb/tsdbDiskData.c" + "src/tsdb/tsdbCompress.c" + "src/tsdb/tsdbCompact.c" + "src/tsdb/tsdbMergeTree.c" # tq "src/tq/tq.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 5d4285b7c25e645dfccf18529cfd2173afa312cc..6ba10641f5e7054034f2008f1b317ec8121286c6 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -125,6 +125,9 @@ int32_t metaTbCursorNext(SMTbCursor *pTbCur); // typedef struct STsdb STsdb; typedef struct STsdbReader STsdbReader; +#define TSDB_DEFAULT_STT_FILE 8 +#define TSDB_DEFAULT_PAGE_SIZE 4096 + #define TIMEWINDOW_RANGE_CONTAINED 1 #define TIMEWINDOW_RANGE_EXTERNAL 2 @@ -155,6 +158,7 @@ int32_t tsdbGetTableSchema(SVnode *pVnode, int64_t uid, STSchema **pSchema, int6 void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity); size_t tsdbCacheGetCapacity(SVnode *pVnode); +size_t tsdbCacheGetUsage(SVnode *pVnode); // tq typedef struct SMetaTableInfo { @@ -287,6 +291,10 @@ struct SVnodeCfg { SVnodeStats vndStats; uint32_t hashBegin; uint32_t hashEnd; + int16_t sttTrigger; + int16_t hashPrefix; + int16_t hashSuffix; + int32_t tsdbPageSize; }; typedef struct { diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index c29c4cb6c4e84db96ba5d419b17da97b86169ebe..9931462e5fc94f0e2fde5f10f5e8a48a81899e48 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -38,9 +38,10 @@ typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; typedef struct STSmaStat STSmaStat; typedef struct SRSmaStat SRSmaStat; -typedef struct SSmaKey SSmaKey; +typedef struct SRSmaRef SRSmaRef; typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; +typedef struct SRSmaFS SRSmaFS; typedef struct SQTaskFile SQTaskFile; typedef struct SQTaskFReader SQTaskFReader; typedef struct SQTaskFWriter SQTaskFWriter; @@ -54,10 +55,21 @@ struct SSmaEnv { #define SMA_ENV_FLG_CLOSE ((int8_t)0x1) +struct SRSmaRef { + int64_t refId; // for SRSmaStat + int64_t suid; +}; + typedef struct { int8_t inited; int32_t rsetId; void *tmrHandle; // shared by all fetch tasks + /** + * @brief key: void* of SRSmaInfoItem, value: SRSmaRef + * N.B. Although there is a very small possibility that "void*" point to different objects while with the same + * address after release/renew, the functionality is not affected as it just used to fetch the rsma results. + */ + SHashObj *refHash; // shared by all vgroups } SSmaMgmt; #define SMA_ENV_LOCK(env) (&(env)->lock) @@ -73,20 +85,25 @@ struct STSmaStat { struct SQTaskFile { volatile int32_t nRef; - int64_t commitID; + int32_t padding; + int64_t version; int64_t size; }; struct SQTaskFReader { - SSma *pSma; - SQTaskFile fTask; - TdFilePtr pReadH; + SSma *pSma; + int64_t version; + TdFilePtr pReadH; }; struct SQTaskFWriter { - SSma *pSma; - SQTaskFile fTask; - TdFilePtr pWriteH; - char *fname; + SSma *pSma; + int64_t version; + TdFilePtr pWriteH; + char *fname; +}; + +struct SRSmaFS { + SArray *aQTaskInf; // array of SQTaskFile }; struct SRSmaStat { @@ -96,9 +113,10 @@ struct SRSmaStat { volatile int64_t nBufItems; // number of items in queue buffer SRWLatch lock; // r/w lock for rsma fs(e.g. qtaskinfo) volatile int32_t nFetchAll; // active number of fetch all - int8_t triggerStat; // shared by fetch tasks - int8_t commitStat; // 0 not in committing, 1 in committing - SArray *aTaskFile; // qTaskFiles committed recently(for recovery/snapshot r/w) + volatile int8_t triggerStat; // shared by fetch tasks + volatile int8_t commitStat; // 0 not in committing, 1 in committing + volatile int8_t delFlag; // 0 no deleted SRSmaInfo, 1 has deleted SRSmaInfo + SRSmaFS fs; // for recovery/snapshot r/w SHashObj *infoHash; // key: suid, value: SRSmaInfo tsem_t notEmpty; // has items in queue buffer }; @@ -118,21 +136,22 @@ struct SSmaStat { #define RSMA_TRIGGER_STAT(r) (&(r)->triggerStat) #define RSMA_COMMIT_STAT(r) (&(r)->commitStat) #define RSMA_REF_ID(r) ((r)->refId) +#define RSMA_FS(r) (&(r)->fs) #define RSMA_FS_LOCK(r) (&(r)->lock) struct SRSmaInfoItem { int8_t level : 4; int8_t fetchLevel : 4; int8_t triggerStat; - uint16_t nSkipped; + uint16_t nScanned; int32_t maxDelay; // ms tmr_h tmrId; }; struct SRSmaInfo { + SSma *pSma; STSchema *pTSchema; int64_t suid; - int64_t refId; // refId of SRSmaStat int64_t lastRecv; // ms int8_t assigned; // 0 idle, 1 assgined for exec int8_t delFlag; @@ -163,14 +182,6 @@ enum { TASK_TRIGGER_STAT_DROPPED = 5, }; -enum { - RSMA_ROLE_CREATE = 0, - RSMA_ROLE_DROP = 1, - RSMA_ROLE_SUBMIT = 2, - RSMA_ROLE_FETCH = 3, - RSMA_ROLE_ITERATE = 4, -}; - enum { RSMA_RESTORE_REBOOT = 1, RSMA_RESTORE_SYNC = 2, @@ -182,89 +193,49 @@ typedef enum { RSMA_EXEC_COMMIT = 3, // triggered by commit } ERsmaExecType; -void tdDestroySmaEnv(SSmaEnv *pSmaEnv); -void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); - -int32_t tdDropTSma(SSma *pSma, char *pMsg); -int32_t tdDropTSmaData(SSma *pSma, int64_t indexUid); -int32_t tdInsertRSmaData(SSma *pSma, char *msg); - -int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat); -int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat); -int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); -int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo); - -void *tdAcquireSmaRef(int32_t rsetId, int64_t refId); -int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId); - +// sma int32_t tdCheckAndInitSmaEnv(SSma *pSma, int8_t smaType); - +void tdDestroySmaEnv(SSmaEnv *pSmaEnv); +void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); int32_t tdLockSma(SSma *pSma); int32_t tdUnLockSma(SSma *pSma); +void *tdAcquireSmaRef(int32_t rsetId, int64_t refId); +int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId); -static FORCE_INLINE int8_t tdSmaStat(STSmaStat *pTStat) { - if (pTStat) { - return atomic_load_8(&pTStat->state); - } - return TSDB_SMA_STAT_UNKNOWN; -} - -static FORCE_INLINE bool tdSmaStatIsOK(STSmaStat *pTStat, int8_t *state) { - if (!pTStat) { - return false; - } - - if (state) { - *state = atomic_load_8(&pTStat->state); - return *state == TSDB_SMA_STAT_OK; - } - return atomic_load_8(&pTStat->state) == TSDB_SMA_STAT_OK; -} - -static FORCE_INLINE bool tdSmaStatIsExpired(STSmaStat *pTStat) { - return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_EXPIRED) : true; -} - -static FORCE_INLINE bool tdSmaStatIsDropped(STSmaStat *pTStat) { - return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_DROPPED) : true; +static FORCE_INLINE void tdRefSmaStat(SSma *pSma, SSmaStat *pStat) { + int32_t ref = T_REF_INC(pStat); + smaDebug("vgId:%d, ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); } - -static FORCE_INLINE void tdSmaStatSetOK(STSmaStat *pTStat) { - if (pTStat) { - atomic_store_8(&pTStat->state, TSDB_SMA_STAT_OK); - } +static FORCE_INLINE void tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { + int32_t ref = T_REF_DEC(pStat); + smaDebug("vgId:%d, unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); } -static FORCE_INLINE void tdSmaStatSetExpired(STSmaStat *pTStat) { - if (pTStat) { - atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_EXPIRED); - } +// rsma +void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version); +void tdRSmaFSClose(SRSmaFS *fs); +int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version); +void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version); +int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat); +int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile); +int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer); +int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); +int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); +int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); +int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); +void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); +void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); + +static FORCE_INLINE void tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { + int32_t ref = T_REF_INC(pRSmaInfo); + smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } - -static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { - if (pTStat) { - atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_DROPPED); - } +static FORCE_INLINE void tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { + int32_t ref = T_REF_DEC(pRSmaInfo); + smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); } -void tdRSmaQTaskInfoGetFileName(int32_t vid, int64_t version, char *outputName); -void tdRSmaQTaskInfoGetFullName(int32_t vid, int64_t version, const char *path, char *outputName); -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); -void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); -static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); -void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); -void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree); -int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash); -int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type); - -int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName); -int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer); -int32_t tdRsmaRestore(SSma *pSma, int8_t type, int64_t committedVer); - -int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); -int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); -int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); - // smaFileUtil ================ #define TD_FILE_HEAD_SIZE 512 diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index d1f5cfb122d6fdfee2cb8f54911a07a25cbb078c..063b33f3a915af2f370f56371698769271cf906d 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -42,15 +42,14 @@ typedef struct SMemTable SMemTable; typedef struct STbDataIter STbDataIter; typedef struct SMapData SMapData; typedef struct SBlockIdx SBlockIdx; -typedef struct SBlock SBlock; -typedef struct SBlockL SBlockL; -typedef struct SColData SColData; +typedef struct SDataBlk SDataBlk; +typedef struct SSttBlk SSttBlk; typedef struct SDiskDataHdr SDiskDataHdr; typedef struct SBlockData SBlockData; typedef struct SDelFile SDelFile; typedef struct SHeadFile SHeadFile; typedef struct SDataFile SDataFile; -typedef struct SLastFile SLastFile; +typedef struct SSttFile SSttFile; typedef struct SSmaFile SSmaFile; typedef struct SDFileSet SDFileSet; typedef struct SDataFWriter SDataFWriter; @@ -64,21 +63,40 @@ typedef struct STsdbReadSnap STsdbReadSnap; typedef struct SBlockInfo SBlockInfo; typedef struct SSmaInfo SSmaInfo; typedef struct SBlockCol SBlockCol; +typedef struct SVersionRange SVersionRange; +typedef struct SLDataIter SLDataIter; #define TSDB_FILE_DLMT ((uint32_t)0xF00AFA0F) #define TSDB_MAX_SUBBLOCKS 8 #define TSDB_FHDR_SIZE 512 -#define HAS_NONE ((int8_t)0x1) -#define HAS_NULL ((int8_t)0x2) -#define HAS_VALUE ((int8_t)0x4) - #define VERSION_MIN 0 #define VERSION_MAX INT64_MAX #define TSDBKEY_MIN ((TSDBKEY){.ts = TSKEY_MIN, .version = VERSION_MIN}) #define TSDBKEY_MAX ((TSDBKEY){.ts = TSKEY_MAX, .version = VERSION_MAX}) +#define TABLE_SAME_SCHEMA(SUID1, UID1, SUID2, UID2) ((SUID1) ? (SUID1) == (SUID2) : (UID1) == (UID2)) + +#define PAGE_CONTENT_SIZE(PAGE) ((PAGE) - sizeof(TSCKSUM)) +#define LOGIC_TO_FILE_OFFSET(LOFFSET, PAGE) \ + ((LOFFSET) / PAGE_CONTENT_SIZE(PAGE) * (PAGE) + (LOFFSET) % PAGE_CONTENT_SIZE(PAGE)) +#define FILE_TO_LOGIC_OFFSET(OFFSET, PAGE) ((OFFSET) / (PAGE)*PAGE_CONTENT_SIZE(PAGE) + (OFFSET) % (PAGE)) +#define PAGE_OFFSET(PGNO, PAGE) (((PGNO)-1) * (PAGE)) +#define OFFSET_PGNO(OFFSET, PAGE) ((OFFSET) / (PAGE) + 1) + +static FORCE_INLINE int64_t tsdbLogicToFileSize(int64_t lSize, int32_t szPage) { + int64_t fOffSet = LOGIC_TO_FILE_OFFSET(lSize, szPage); + int64_t pgno = OFFSET_PGNO(fOffSet, szPage); + int32_t szPageCont = PAGE_CONTENT_SIZE(szPage); + + if (fOffSet % szPageCont == 0) { + pgno--; + } + + return pgno * szPage; +} + // tsdbUtil.c ============================================================================================== // TSDBROW #define TSDBROW_TS(ROW) (((ROW)->type == 0) ? (ROW)->pTSRow->ts : (ROW)->pBlockData->aTSKEY[(ROW)->iRow]) @@ -111,29 +129,20 @@ int32_t tTABLEIDCmprFn(const void *p1, const void *p2); int32_t tPutBlockCol(uint8_t *p, void *ph); int32_t tGetBlockCol(uint8_t *p, void *ph); int32_t tBlockColCmprFn(const void *p1, const void *p2); -// SBlock -void tBlockReset(SBlock *pBlock); -int32_t tPutBlock(uint8_t *p, void *ph); -int32_t tGetBlock(uint8_t *p, void *ph); -int32_t tBlockCmprFn(const void *p1, const void *p2); -bool tBlockHasSma(SBlock *pBlock); -// SBlockL -int32_t tPutBlockL(uint8_t *p, void *ph); -int32_t tGetBlockL(uint8_t *p, void *ph); +// SDataBlk +void tDataBlkReset(SDataBlk *pBlock); +int32_t tPutDataBlk(uint8_t *p, void *ph); +int32_t tGetDataBlk(uint8_t *p, void *ph); +int32_t tDataBlkCmprFn(const void *p1, const void *p2); +bool tDataBlkHasSma(SDataBlk *pDataBlk); +// SSttBlk +int32_t tPutSttBlk(uint8_t *p, void *ph); +int32_t tGetSttBlk(uint8_t *p, void *ph); // SBlockIdx int32_t tPutBlockIdx(uint8_t *p, void *ph); int32_t tGetBlockIdx(uint8_t *p, void *ph); int32_t tCmprBlockIdx(void const *lhs, void const *rhs); int32_t tCmprBlockL(void const *lhs, void const *rhs); -// SColdata -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn); -void tColDataReset(SColData *pColData); -void tColDataClear(void *ph); -int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal); -int32_t tColDataGetValue(SColData *pColData, int32_t iRow, SColVal *pColVal); -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest); -int32_t tPutColData(uint8_t *p, SColData *pColData); -int32_t tGetColData(uint8_t *p, SColData *pColData); // SBlockData #define tBlockDataFirstRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, 0) #define tBlockDataLastRow(PBLOCKDATA) tsdbRowFromBlockData(PBLOCKDATA, (PBLOCKDATA)->nRow - 1) @@ -170,6 +179,7 @@ int32_t tGetDelData(uint8_t *p, void *ph); void tMapDataReset(SMapData *pMapData); void tMapDataClear(SMapData *pMapData); int32_t tMapDataPutItem(SMapData *pMapData, void *pItem, int32_t (*tPutItemFn)(uint8_t *, void *)); +int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo); void tMapDataGetItemByIdx(SMapData *pMapData, int32_t idx, void *pItem, int32_t (*tGetItemFn)(uint8_t *, void *)); int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t (*tItemCmprFn)(const void *, const void *), void *pItem); @@ -191,7 +201,6 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol uint8_t **ppBuf); int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, int32_t nVal, SColData *pColData, uint8_t **ppBuf); -int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck); // tsdbMemTable ============================================================================================== // SMemTable int32_t tsdbMemTableCreate(STsdb *pTsdb, SMemTable **ppMemTable); @@ -215,7 +224,7 @@ bool tsdbDelFileIsSame(SDelFile *pDelFile1, SDelFile *pDelFile2); int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype); int32_t tPutHeadFile(uint8_t *p, SHeadFile *pHeadFile); int32_t tPutDataFile(uint8_t *p, SDataFile *pDataFile); -int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile); +int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile); int32_t tPutSmaFile(uint8_t *p, SSmaFile *pSmaFile); int32_t tPutDelFile(uint8_t *p, SDelFile *pDelFile); int32_t tGetDelFile(uint8_t *p, SDelFile *pDelFile); @@ -224,7 +233,7 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet); void tsdbHeadFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SHeadFile *pHeadF, char fname[]); void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, char fname[]); -void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]); +void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]); void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]); // SDelFile void tsdbDelFileName(STsdb *pTsdb, SDelFile *pFile, char fname[]); @@ -249,8 +258,8 @@ int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pS int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync); int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter); int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx); -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *pMapData, SBlockIdx *pBlockIdx); -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL); +int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx); +int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk); int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, int8_t cmprAlg, int8_t toLast); @@ -259,11 +268,11 @@ int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo); int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet); int32_t tsdbDataFReaderClose(SDataFReader **ppReader); int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx); -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *pMapData); -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL); -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg); -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData); -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData); +int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk); +int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk); +int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pBlock, SArray *aColumnDataAgg); +int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pBlock, SBlockData *pBlockData); +int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData); // SDelFWriter int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb); int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync); @@ -278,6 +287,8 @@ int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx); // tsdbRead.c ============================================================================================== int32_t tsdbTakeReadSnap(STsdb *pTsdb, STsdbReadSnap **ppSnap); void tsdbUntakeReadSnap(STsdb *pTsdb, STsdbReadSnap *pSnap); +// tsdbMerge.c ============================================================================================== +int32_t tsdbMerge(STsdb *pTsdb); #define TSDB_CACHE_NO(c) ((c).cacheLast == 0) #define TSDB_CACHE_LAST_ROW(c) (((c).cacheLast & 1) > 0) @@ -324,6 +335,11 @@ struct TSDBKEY { TSKEY ts; }; +struct SVersionRange { + uint64_t minVer; + uint64_t maxVer; +}; + typedef struct SMemSkipListNode SMemSkipListNode; struct SMemSkipListNode { int8_t level; @@ -416,7 +432,7 @@ struct SSmaInfo { int32_t size; }; -struct SBlock { +struct SDataBlk { TSDBKEY minKey; TSDBKEY maxKey; int64_t minVer; @@ -428,7 +444,7 @@ struct SBlock { SSmaInfo smaInfo; }; -struct SBlockL { +struct SSttBlk { int64_t suid; int64_t minUid; int64_t maxUid; @@ -440,18 +456,6 @@ struct SBlockL { SBlockInfo bInfo; }; -struct SColData { - int16_t cid; - int8_t type; - int8_t smaOn; - int32_t nVal; - uint8_t flag; - uint8_t *pBitMap; - int32_t *aOffset; - int32_t nData; - uint8_t *pData; -}; - // (SBlockData){.suid = 0, .uid = 0}: block data not initialized // (SBlockData){.suid = suid, .uid = uid}: block data for ONE child table int .data file // (SBlockData){.suid = suid, .uid = 0}: block data for N child tables int .last file @@ -467,12 +471,6 @@ struct SBlockData { SArray *aColData; // SArray }; -// ================== TSDB global config -extern bool tsdbForceKeepFile; - -#define TSDB_FS_ITER_FORWARD TSDB_ORDER_ASC -#define TSDB_FS_ITER_BACKWARD TSDB_ORDER_DESC - struct TABLEID { tb_uid_t suid; tb_uid_t uid; @@ -536,7 +534,7 @@ struct SDataFile { int64_t size; }; -struct SLastFile { +struct SSttFile { volatile int32_t nRef; int64_t commitID; @@ -556,8 +554,9 @@ struct SDFileSet { int32_t fid; SHeadFile *pHeadF; SDataFile *pDataF; - SLastFile *pLastF; SSmaFile *pSmaF; + uint8_t nSttF; + SSttFile *aSttF[TSDB_MAX_STT_TRIGGER]; }; struct SRowIter { @@ -572,37 +571,99 @@ struct SRowMerger { SArray *pArray; // SArray }; -struct SDelFWriter { - STsdb *pTsdb; - SDelFile fDel; - TdFilePtr pWriteH; +typedef struct { + char *path; + int32_t szPage; + int32_t flag; + TdFilePtr pFD; + int64_t pgno; + uint8_t *pBuf; + int64_t szFile; +} STsdbFD; +struct SDelFWriter { + STsdb *pTsdb; + SDelFile fDel; + STsdbFD *pWriteH; uint8_t *aBuf[1]; }; +struct STsdbReadSnap { + SMemTable *pMem; + SMemTable *pIMem; + STsdbFS fs; +}; + struct SDataFWriter { STsdb *pTsdb; SDFileSet wSet; - TdFilePtr pHeadFD; - TdFilePtr pDataFD; - TdFilePtr pLastFD; - TdFilePtr pSmaFD; + STsdbFD *pHeadFD; + STsdbFD *pDataFD; + STsdbFD *pSmaFD; + STsdbFD *pSttFD; SHeadFile fHead; SDataFile fData; - SLastFile fLast; SSmaFile fSma; + SSttFile fStt[TSDB_MAX_STT_TRIGGER]; uint8_t *aBuf[4]; }; -struct STsdbReadSnap { - SMemTable *pMem; - SMemTable *pIMem; - STsdbFS fs; +struct SDataFReader { + STsdb *pTsdb; + SDFileSet *pSet; + STsdbFD *pHeadFD; + STsdbFD *pDataFD; + STsdbFD *pSmaFD; + STsdbFD *aSttFD[TSDB_MAX_STT_TRIGGER]; + uint8_t *aBuf[3]; }; +typedef struct { + int64_t suid; + int64_t uid; + TSDBROW row; +} SRowInfo; + +typedef struct SSttBlockLoadInfo { + SBlockData blockData[2]; + SArray *aSttBlk; + int32_t blockIndex[2]; // to denote the loaded block in the corresponding position. + int32_t currentLoadBlockIndex; + int32_t loadBlocks; + double elapsedTime; +} SSttBlockLoadInfo; + +typedef struct SMergeTree { + int8_t backward; + SRBTree rbt; + SArray *pIterList; + SLDataIter *pIter; + bool destroyLoadInfo; + SSttBlockLoadInfo *pLoadInfo; + const char *idStr; +} SMergeTree; + +typedef struct { + int64_t suid; + int64_t uid; + STSchema *pTSchema; +} SSkmInfo; + +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pVerRange, void *pLoadInfo, const char *idStr); +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter); +bool tMergeTreeNext(SMergeTree *pMTree); +TSDBROW tMergeTreeGetRow(SMergeTree *pMTree); +void tMergeTreeClose(SMergeTree *pMTree); + +SSttBlockLoadInfo *tCreateLastBlockLoadInfo(); +void resetLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); +void getLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo, int64_t *blocks, double *el); +void *destroyLastBlockLoadInfo(SSttBlockLoadInfo *pLoadInfo); + // ========== inline functions ========== static FORCE_INLINE int32_t tsdbKeyCmprFn(const void *p1, const void *p2) { TSDBKEY *pKey1 = (TSDBKEY *)p1; diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 9b252df58b2a87c4baf8453ad597d62e50b61a33..0e85e7bfb60313b106d8838986fa685eedf2c409 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -36,6 +36,7 @@ #include "tlosertree.h" #include "tlrucache.h" #include "tmsgcb.h" +#include "trbtree.h" #include "tref.h" #include "tskiplist.h" #include "tstream.h" @@ -70,8 +71,8 @@ typedef struct SStreamTaskReader SStreamTaskReader; typedef struct SStreamTaskWriter SStreamTaskWriter; typedef struct SStreamStateReader SStreamStateReader; typedef struct SStreamStateWriter SStreamStateWriter; -typedef struct SRsmaSnapReader SRsmaSnapReader; -typedef struct SRsmaSnapWriter SRsmaSnapWriter; +typedef struct SRSmaSnapReader SRSmaSnapReader; +typedef struct SRSmaSnapWriter SRSmaSnapWriter; typedef struct SSnapDataHdr SSnapDataHdr; #define VNODE_META_DIR "meta" @@ -102,8 +103,8 @@ int metaCommit(SMeta* pMeta); int metaCreateSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaAlterSTable(SMeta* pMeta, int64_t version, SVCreateStbReq* pReq); int metaDropSTable(SMeta* pMeta, int64_t verison, SVDropStbReq* pReq, SArray* tbUidList); -int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp **pMetaRsp); -int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids); +int metaCreateTable(SMeta* pMeta, int64_t version, SVCreateTbReq* pReq, STableMetaRsp** pMetaRsp); +int metaDropTable(SMeta* pMeta, int64_t version, SVDropTbReq* pReq, SArray* tbUids, int64_t* tbUid); int metaTtlDropTable(SMeta* pMeta, int64_t ttl, SArray* tbUids); int metaAlterTable(SMeta* pMeta, int64_t version, SVAlterTbReq* pReq, STableMetaRsp* pMetaRsp); SSchemaWrapper* metaGetTableSchema(SMeta* pMeta, tb_uid_t uid, int32_t sver, bool isinline); @@ -173,7 +174,8 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); -int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* data, int64_t ver); +int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver); int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec); int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); @@ -206,7 +208,7 @@ int32_t tdProcessRSmaCreate(SSma* pSma, SVCreateStbReq* pReq); int32_t tdProcessRSmaSubmit(SSma* pSma, void* pMsg, int32_t inputType); int32_t tdProcessRSmaDrop(SSma* pSma, SVDropStbReq* pReq); int32_t tdFetchTbUidList(SSma* pSma, STbUidStore** ppStore, tb_uid_t suid, tb_uid_t uid); -int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore); +int32_t tdUpdateTbUidList(SSma* pSma, STbUidStore* pUidStore, bool isAdd); void tdUidStoreDestory(STbUidStore* pStore); void* tdUidStoreFree(STbUidStore* pStore); @@ -246,14 +248,14 @@ int32_t tqOffsetSnapWrite(STqOffsetWriter* pWriter, uint8_t* pData, uint32_t nDa // SStreamTaskReader ====================================== // SStreamStateWriter ===================================== // SStreamStateReader ===================================== -// SRsmaSnapReader ======================================== -int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapReader** ppReader); -int32_t rsmaSnapReaderClose(SRsmaSnapReader** ppReader); -int32_t rsmaSnapRead(SRsmaSnapReader* pReader, uint8_t** ppData); -// SRsmaSnapWriter ======================================== -int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWriter** ppWriter); -int32_t rsmaSnapWrite(SRsmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); -int32_t rsmaSnapWriterClose(SRsmaSnapWriter** ppWriter, int8_t rollback); +// SRSmaSnapReader ======================================== +int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader); +int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader); +int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData); +// SRSmaSnapWriter ======================================== +int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter); +int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback); typedef struct { int8_t streamType; // sma or other diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index 7df355a59b1a1099faf19daf13c73db3cc9ea095..a34569b08ecdd87483dbf31e8c6d7e406e8ae766 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -129,10 +129,16 @@ _err: bool metaIsTableExist(SMeta *pMeta, tb_uid_t uid) { // query uid.idx + metaRLock(pMeta); + if (tdbTbGet(pMeta->pUidIdx, &uid, sizeof(uid), NULL, NULL) < 0) { + metaULock(pMeta); + return false; } + metaULock(pMeta); + return true; } @@ -182,9 +188,14 @@ tb_uid_t metaGetTableEntryUidByName(SMeta *pMeta, const char *name) { } int metaGetTableNameByUid(void *meta, uint64_t uid, char *tbName) { + int code = 0; SMetaReader mr = {0}; metaReaderInit(&mr, (SMeta *)meta, 0); - metaGetTableEntryByUid(&mr, uid); + code = metaGetTableEntryByUid(&mr, uid); + if (code < 0) { + metaReaderClear(&mr); + return -1; + } STR_TO_VARSTR(tbName, mr.me.name); metaReaderClear(&mr); @@ -280,6 +291,38 @@ _query: tDecoderClear(&dc); goto _exit; } + { // Traverse to find the previous qualified data + TBC *pCur; + tdbTbcOpen(pMeta->pTbDb, &pCur, NULL); + STbDbKey key = {.version = sver, .uid = INT64_MAX}; + int c = 0; + tdbTbcMoveTo(pCur, &key, sizeof(key), &c); + if(c < 0){ + tdbTbcMoveToPrev(pCur); + } + + void *pKey = NULL; + void *pVal = NULL; + int vLen = 0, kLen = 0; + while(1){ + int32_t ret = tdbTbcPrev(pCur, &pKey, &kLen, &pVal, &vLen); + if (ret < 0) break; + + STbDbKey *tmp = (STbDbKey*)pKey; + if(tmp->uid != uid){ + continue; + } + SDecoder dcNew = {0}; + SMetaEntry meNew = {0}; + tDecoderInit(&dcNew, pVal, vLen); + metaDecodeEntry(&dcNew, &meNew); + pSchema = tCloneSSchemaWrapper(&meNew.stbEntry.schemaRow); + tDecoderClear(&dcNew); + tdbTbcClose(pCur); + goto _exit; + } + tdbTbcClose(pCur); + } } else if (me.type == TSDB_CHILD_TABLE) { uid = me.ctbEntry.suid; tDecoderClear(&dc); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index 583a2e098f8a54ac61f21d696c6e65c62cd5c4ab..22ec8118a2911f738b2aa7ce27f8b32b7fd4d461 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -244,6 +244,7 @@ int metaDropSTable(SMeta *pMeta, int64_t verison, SVDropStbReq *pReq, SArray *tb // check if super table exists rc = tdbTbGet(pMeta->pNameIdx, pReq->name, strlen(pReq->name) + 1, &pData, &nData); if (rc < 0 || *(tb_uid_t *)pData != pReq->suid) { + tdbFree(pData); terrno = TSDB_CODE_TDB_STB_NOT_EXIST; return -1; } @@ -309,7 +310,7 @@ int metaAlterSTable(SMeta *pMeta, int64_t version, SVCreateStbReq *pReq) { int64_t oversion; SDecoder dc = {0}; int32_t ret; - int32_t c; + int32_t c = -2; tdbTbcOpen(pMeta->pUidIdx, &pUidIdxc, &pMeta->txn); ret = tdbTbcMoveTo(pUidIdxc, &pReq->suid, sizeof(tb_uid_t), &c); @@ -416,20 +417,22 @@ int metaCreateTable(SMeta *pMeta, int64_t version, SVCreateTbReq *pReq, STableMe me.ctbEntry.pTags = pReq->ctb.pTag; #ifdef TAG_FILTER_DEBUG - SArray* pTagVals = NULL; - int32_t code = tTagToValArray((STag*)pReq->ctb.pTag, &pTagVals); + SArray *pTagVals = NULL; + int32_t code = tTagToValArray((STag *)pReq->ctb.pTag, &pTagVals); for (int i = 0; i < taosArrayGetSize(pTagVals); i++) { - STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, i); + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, i); if (IS_VAR_DATA_TYPE(pTagVal->type)) { - char* buf = taosMemoryCalloc(pTagVal->nData + 1, 1); + char *buf = taosMemoryCalloc(pTagVal->nData + 1, 1); memcpy(buf, pTagVal->pData, pTagVal->nData); - metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, pTagVal->type, buf); + metaDebug("metaTag table:%s varchar index:%d cid:%d type:%d value:%s", pReq->name, i, pTagVal->cid, + pTagVal->type, buf); taosMemoryFree(buf); } else { double val = 0; GET_TYPED_DATA(val, double, pTagVal->type, &pTagVal->i64); - metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, pTagVal->type, val); + metaDebug("metaTag table:%s number index:%d cid:%d type:%d value:%f", pReq->name, i, pTagVal->cid, + pTagVal->type, val); } } #endif @@ -474,7 +477,7 @@ _err: return -1; } -int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids) { +int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUids, tb_uid_t *tbUid) { void *pData = NULL; int nData = 0; int rc = 0; @@ -496,6 +499,10 @@ int metaDropTable(SMeta *pMeta, int64_t version, SVDropTbReq *pReq, SArray *tbUi taosArrayPush(tbUids, &uid); } + if ((type == TSDB_CHILD_TABLE) && tbUid) { + *tbUid = uid; + } + tdbFree(pData); return 0; } diff --git a/source/dnode/vnode/src/sma/smaCommit.c b/source/dnode/vnode/src/sma/smaCommit.c index 3cf50a035a720f7bf9e106c69a8a88e1117a8954..07ec7d06947b569b0452b283120430bb44e919ca 100644 --- a/source/dnode/vnode/src/sma/smaCommit.c +++ b/source/dnode/vnode/src/sma/smaCommit.c @@ -15,13 +15,15 @@ #include "sma.h" +extern SSmaMgmt smaMgmt; + static int32_t tdProcessRSmaSyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncCommitImpl(SSma *pSma); static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma); -static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); +static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat); /** * @brief Only applicable to Rollup SMA @@ -166,114 +168,65 @@ static int32_t tdProcessRSmaSyncCommitImpl(SSma *pSma) { return TSDB_CODE_SUCCESS; } -static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { - SVnode *pVnode = pSma->pVnode; - int64_t committed = pRSmaStat->commitAppliedVer; - TdDirPtr pDir = NULL; - TdDirEntryPtr pDirEntry = NULL; - char dir[TSDB_FILENAME_LEN]; - const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$"; - regex_t regex; - int code = 0; - - tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); - - // Resource allocation and init - if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) { - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, rsma post commit, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf); - return TSDB_CODE_FAILED; - } - - if ((pDir = taosOpenDir(dir)) == NULL) { - regfree(®ex); - terrno = TAOS_SYSTEM_ERROR(errno); - smaDebug("vgId:%d, rsma post commit, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); - return TSDB_CODE_FAILED; - } - - int32_t dirLen = strlen(dir); - char *dirEnd = POINTER_SHIFT(dir, dirLen); - regmatch_t regMatch[2]; - while ((pDirEntry = taosReadDir(pDir)) != NULL) { - char *entryName = taosGetDirEntryName(pDirEntry); - if (!entryName) { - continue; - } - - code = regexec(®ex, entryName, 2, regMatch, 0); - - if (code == 0) { - // match - int64_t version = -1; - sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &version); - if ((version < committed) && (version > -1)) { - strncpy(dirEnd, entryName, TSDB_FILENAME_LEN - dirLen); - if (taosRemoveFile(dir) != 0) { - terrno = TAOS_SYSTEM_ERROR(errno); - smaWarn("vgId:%d, committed version:%" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed, - dir, terrstr()); - } else { - smaDebug("vgId:%d, committed version:%" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, dir); - } - } - } else if (code == REG_NOMATCH) { - // not match - smaTrace("vgId:%d, rsma post commit, not match %s", TD_VID(pVnode), entryName); - continue; - } else { - // has other error - char errbuf[128]; - regerror(code, ®ex, errbuf, sizeof(errbuf)); - smaWarn("vgId:%d, rsma post commit, regexec failed since %s", TD_VID(pVnode), errbuf); - - taosCloseDir(&pDir); - regfree(®ex); - return TSDB_CODE_FAILED; - } - } - - taosCloseDir(&pDir); - regfree(®ex); - - return TSDB_CODE_SUCCESS; -} - // SQTaskFile ====================================================== -// int32_t tCmprQTaskFile(void const *lhs, void const *rhs) { -// int64_t *lCommitted = *(int64_t *)lhs; -// SQTaskFile *rQTaskF = (SQTaskFile *)rhs; - -// if (lCommitted < rQTaskF->commitID) { -// return -1; -// } else if (lCommitted > rQTaskF->commitID) { -// return 1; -// } - -// return 0; -// } -#if 0 /** * @brief At most time, there is only one qtaskinfo file committed latest in aTaskFile. Sometimes, there would be * multiple qtaskinfo files supporting snapshot replication. * * @param pSma - * @param pRSmaStat + * @param pStat * @return int32_t */ -static int32_t tdCleanupQTaskInfoFiles(SSma *pSma, SRSmaStat *pRSmaStat) { - SVnode *pVnode = pSma->pVnode; - int64_t committed = pRSmaStat->commitAppliedVer; - SArray *aTaskFile = pRSmaStat->aTaskFile; +static int32_t tdUpdateQTaskInfoFiles(SSma *pSma, SRSmaStat *pStat) { + SVnode *pVnode = pSma->pVnode; + SRSmaFS *pFS = RSMA_FS(pStat); + int64_t committed = pStat->commitAppliedVer; + int64_t fsMaxVer = -1; + char qTaskInfoFullName[TSDB_FILENAME_LEN]; + + taosWLockLatch(RSMA_FS_LOCK(pStat)); + + for (int32_t i = 0; i < taosArrayGetSize(pFS->aQTaskInf);) { + SQTaskFile *pTaskF = taosArrayGet(pFS->aQTaskInf, i); + int32_t oldVal = atomic_fetch_sub_32(&pTaskF->nRef, 1); + if ((oldVal <= 1) && (pTaskF->version < committed)) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); + if (taosRemoveFile(qTaskInfoFullName) < 0) { + smaWarn("vgId:%d, cleanup qinf, committed %" PRIi64 ", failed to remove %s since %s", TD_VID(pVnode), committed, + qTaskInfoFullName, tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + smaDebug("vgId:%d, cleanup qinf, committed %" PRIi64 ", success to remove %s", TD_VID(pVnode), committed, + qTaskInfoFullName); + } + taosArrayRemove(pFS->aQTaskInf, i); + continue; + } + ++i; + } + + if (taosArrayGetSize(pFS->aQTaskInf) > 0) { + fsMaxVer = ((SQTaskFile *)taosArrayGetLast(pFS->aQTaskInf))->version; + } - void *qTaskFile = taosArraySearch(aTaskFile, committed, tCmprQTaskFile, TD_LE); - + if (fsMaxVer < committed) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), committed, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); + if (taosCheckExistFile(qTaskInfoFullName)) { + SQTaskFile qFile = {.nRef = 1, .padding = 0, .version = committed, .size = 0}; + if (taosArrayPush(pFS->aQTaskInf, &qFile) < 0) { + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + } + } else { + smaDebug("vgId:%d, update qinf, no need as committed %" PRIi64 " not larger than fsMaxVer %" PRIi64, TD_VID(pVnode), + committed, fsMaxVer); + } + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); return TSDB_CODE_SUCCESS; } -#endif /** * @brief post-commit for rollup sma @@ -290,8 +243,7 @@ static int32_t tdProcessRSmaSyncPostCommitImpl(SSma *pSma) { SRSmaStat *pRSmaStat = SMA_RSMA_STAT(pSma); - // cleanup outdated qtaskinfo files - tdCleanupQTaskInfoFiles(pSma, pRSmaStat); + tdUpdateQTaskInfoFiles(pSma, pRSmaStat); return TSDB_CODE_SUCCESS; } @@ -365,12 +317,12 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { if (tdRSmaPersistExecImpl(pRSmaStat, RSMA_INFO_HASH(pRSmaStat)) < 0) { return TSDB_CODE_FAILED; } - smaInfo("vgId:%d, rsma commit, operator state commited, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); + smaInfo("vgId:%d, rsma commit, operator state committed, TID:%p", SMA_VID(pSma), (void *)taosGetSelfPthreadId()); #if 0 // consuming task of qTaskInfo clone // step 4: swap queue/qall and iQueue/iQall // lock - // taosWLockLatch(SMA_ENV_LOCK(pEnv)); + taosWLockLatch(SMA_ENV_LOCK(pEnv)); ASSERT(RSMA_INFO_HASH(pRSmaStat)); @@ -386,7 +338,7 @@ static int32_t tdProcessRSmaAsyncPreCommitImpl(SSma *pSma) { } // unlock - // taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); #endif return TSDB_CODE_SUCCESS; @@ -428,33 +380,29 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { } SRSmaStat *pRSmaStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); - SArray *rsmaDeleted = NULL; // step 1: merge qTaskInfo and iQTaskInfo // lock - // taosWLockLatch(SMA_ENV_LOCK(pEnv)); - - void *pIter = NULL; - while ((pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter))) { - tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); - SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - int32_t refVal = T_REF_VAL_GET(pRSmaInfo); - if (refVal == 0) { - if (!rsmaDeleted) { - if ((rsmaDeleted = taosArrayInit(1, sizeof(tb_uid_t)))) { - taosArrayPush(rsmaDeleted, pSuid); - } + if (1 == atomic_val_compare_exchange_8(&pRSmaStat->delFlag, 1, 0)) { + taosWLockLatch(SMA_ENV_LOCK(pEnv)); + + void *pIter = NULL; + while ((pIter = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), pIter))) { + tb_uid_t *pSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)pIter; + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + int32_t refVal = T_REF_VAL_GET(pRSmaInfo); + if (refVal == 0) { + taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(*pSuid)); + } else { + smaDebug( + "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for " + "table:%" PRIi64, + SMA_VID(pSma), refVal, *pSuid); } - } else { - smaDebug( - "vgId:%d, rsma async post commit, not free rsma info since ref is %d although already deleted for " - "table:%" PRIi64, - SMA_VID(pSma), refVal, *pSuid); - } - continue; - } + continue; + } #if 0 if (pRSmaInfo->taskInfo[0]) { if (pRSmaInfo->iTaskInfo[0]) { @@ -469,27 +417,13 @@ static int32_t tdProcessRSmaAsyncPostCommitImpl(SSma *pSma) { taosHashPut(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t), pIter, sizeof(pIter)); smaDebug("vgId:%d, rsma async post commit, migrated from iRsmaInfoHash for table:%" PRIi64, SMA_VID(pSma), *pSuid); #endif - } - - for (int32_t i = 0; i < taosArrayGetSize(rsmaDeleted); ++i) { - tb_uid_t *pSuid = taosArrayGet(rsmaDeleted, i); - void *pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); - if ((pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { - tdFreeRSmaInfo(pSma, pRSmaInfo, true); - smaDebug( - "vgId:%d, rsma async post commit, free rsma info since already deleted and ref is 0 for " - "table:%" PRIi64, - SMA_VID(pSma), *pSuid); } - taosHashRemove(RSMA_INFO_HASH(pRSmaStat), pSuid, sizeof(tb_uid_t)); - } - taosArrayDestroy(rsmaDeleted); - // unlock - // taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + // unlock + taosWUnLockLatch(SMA_ENV_LOCK(pEnv)); + } - // step 2: cleanup outdated qtaskinfo files - tdCleanupQTaskInfoFiles(pSma, pRSmaStat); + tdUpdateQTaskInfoFiles(pSma, pRSmaStat); atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index 32a419022a312f9ab21681b9bc6f819c7792f51e..b870ea1b6223ce0d2fd1cf527cd5b0c20c27a5b7 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -28,6 +28,8 @@ static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, SSmaEnv **ppEnv); static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma); static int32_t tdRsmaStartExecutor(const SSma *pSma); static int32_t tdRsmaStopExecutor(const SSma *pSma); +static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); +static void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); static void *tdFreeTSmaStat(STSmaStat *pStat); static void tdDestroyRSmaStat(void *pRSmaStat); @@ -59,12 +61,23 @@ int32_t smaInit() { return TSDB_CODE_FAILED; } + int32_t type = (8 == POINTER_BYTES) ? TSDB_DATA_TYPE_UBIGINT : TSDB_DATA_TYPE_UINT; + smaMgmt.refHash = taosHashInit(64, taosGetDefaultHashFunction(type), true, HASH_ENTRY_LOCK); + if (!smaMgmt.refHash) { + taosCloseRef(smaMgmt.rsetId); + atomic_store_8(&smaMgmt.inited, 0); + smaError("failed to init sma tmr hanle since %s", terrstr()); + return TSDB_CODE_FAILED; + } + // init fetch timer handle smaMgmt.tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); if (!smaMgmt.tmrHandle) { taosCloseRef(smaMgmt.rsetId); + taosHashCleanup(smaMgmt.refHash); + smaMgmt.refHash = NULL; atomic_store_8(&smaMgmt.inited, 0); - smaError("failed to init sma tmr hanle since %s", terrstr()); + smaError("failed to init sma tmr handle since %s", terrstr()); return TSDB_CODE_FAILED; } @@ -93,6 +106,8 @@ void smaCleanUp() { if (old == 1) { taosCloseRef(smaMgmt.rsetId); + taosHashCleanup(smaMgmt.refHash); + smaMgmt.refHash = NULL; taosTmrCleanUp(smaMgmt.tmrHandle); smaInfo("sma mgmt env is cleaned up, rsetId:%d, tmrHandle:%p", smaMgmt.rsetId, smaMgmt.tmrHandle); atomic_store_8(&smaMgmt.inited, 0); @@ -162,37 +177,19 @@ void *tdFreeSmaEnv(SSmaEnv *pSmaEnv) { return NULL; } -int32_t tdRefSmaStat(SSma *pSma, SSmaStat *pStat) { - if (!pStat) return 0; - - int ref = T_REF_INC(pStat); - smaDebug("vgId:%d, ref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); - return 0; -} - -int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { - if (!pStat) return 0; - - int ref = T_REF_DEC(pStat); - smaDebug("vgId:%d, unref sma stat:%p, val:%d", SMA_VID(pSma), pStat, ref); - return 0; -} - -int32_t tdRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { - if (!pRSmaInfo) return 0; +static void tRSmaInfoHashFreeNode(void *data) { + SRSmaInfo *pRSmaInfo = NULL; + SRSmaInfoItem *pItem = NULL; - int ref = T_REF_INC(pRSmaInfo); - smaDebug("vgId:%d, ref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); - return 0; -} - -int32_t tdUnRefRSmaInfo(SSma *pSma, SRSmaInfo *pRSmaInfo) { - if (!pRSmaInfo) return 0; - - int ref = T_REF_DEC(pRSmaInfo); - smaDebug("vgId:%d, unref rsma info:%p, val:%d", SMA_VID(pSma), pRSmaInfo, ref); - - return 0; + if ((pRSmaInfo = *(SRSmaInfo **)data)) { + if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 0)) && pItem->level) { + taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES); + } + if ((pItem = RSMA_INFO_ITEM((SRSmaInfo *)pRSmaInfo, 1)) && pItem->level) { + taosHashRemove(smaMgmt.refHash, &pItem, POINTER_BYTES); + } + tdFreeRSmaInfo(pRSmaInfo->pSma, pRSmaInfo, true); + } } static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { @@ -240,10 +237,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pS if (!RSMA_INFO_HASH(pRSmaStat)) { return TSDB_CODE_FAILED; } + taosHashSetFreeFp(RSMA_INFO_HASH(pRSmaStat), tRSmaInfoHashFreeNode); if (tdRsmaStartExecutor(pSma) < 0) { return TSDB_CODE_FAILED; } + + if (!(RSMA_FS(pRSmaStat)->aQTaskInf = taosArrayInit(1, sizeof(SQTaskFile)))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } } else if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { // TODO } else { @@ -278,14 +281,6 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { tsem_destroy(&(pStat->notEmpty)); // step 2: destroy the rsma info and associated fetch tasks - if (taosHashGetSize(RSMA_INFO_HASH(pStat)) > 0) { - void *infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), NULL); - while (infoHash) { - SRSmaInfo *pSmaInfo = *(SRSmaInfo **)infoHash; - tdFreeRSmaInfo(pSma, pSmaInfo, true); - infoHash = taosHashIterate(RSMA_INFO_HASH(pStat), infoHash); - } - } taosHashCleanup(RSMA_INFO_HASH(pStat)); // step 3: wait for all triggered fetch tasks to finish @@ -307,12 +302,15 @@ static void tdDestroyRSmaStat(void *pRSmaStat) { // step 4: tdRsmaStopExecutor(pSma); - // step 5: free pStat + // step 5: + tdRSmaFSClose(RSMA_FS(pStat)); + + // step 6: free pStat taosMemoryFreeClear(pStat); } } -void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { +static void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { tdDestroySmaState(pSmaStat, smaType); if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { taosMemoryFreeClear(pSmaStat); @@ -329,7 +327,7 @@ void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { * @return int32_t */ -int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { +static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { tdDestroyTSmaStat(SMA_STAT_TSMA(pSmaStat)); @@ -337,7 +335,7 @@ int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { SRSmaStat *pRSmaStat = &pSmaStat->rsmaStat; int32_t vid = SMA_VID(pRSmaStat->pSma); int64_t refId = RSMA_REF_ID(pRSmaStat); - if (taosRemoveRef(smaMgmt.rsetId, RSMA_REF_ID(pRSmaStat)) < 0) { + if (taosRemoveRef(smaMgmt.rsetId, refId) < 0) { smaError("vgId:%d, remove refId:%" PRIi64 " from rsmaRef:%" PRIi32 " failed since %s", vid, refId, smaMgmt.rsetId, terrstr()); } else { @@ -461,6 +459,8 @@ static int32_t tdRsmaStopExecutor(const SSma *pSma) { taosThreadJoin(pthread[i], NULL); } } + + smaInfo("vgId:%d, rsma executor stopped, number:%d", SMA_VID(pSma), tsNumOfVnodeRsmaThreads); } return 0; } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaFS.c b/source/dnode/vnode/src/sma/smaFS.c new file mode 100644 index 0000000000000000000000000000000000000000..8e8611f0e869d9e3d29c7456af142f05dba15b91 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaFS.c @@ -0,0 +1,264 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +// ================================================================================================= + +static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output); +static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2); +static int32_t tdQTaskInfCmprFn2(const void *p1, const void *p2); +/** + * @brief Open RSma FS from qTaskInfo files + * + * @param pSma + * @param version + * @return int32_t + */ +int32_t tdRSmaFSOpen(SSma *pSma, int64_t version) { + SVnode *pVnode = pSma->pVnode; + int64_t commitID = pVnode->state.commitID; + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = NULL; + SArray *output = NULL; + + terrno = TSDB_CODE_SUCCESS; + + if (!pEnv) { + return TSDB_CODE_SUCCESS; + } + + if (tdFetchQTaskInfoFiles(pSma, version, &output) < 0) { + goto _end; + } + + pStat = (SRSmaStat *)SMA_ENV_STAT(pEnv); + + for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { + int32_t vid = 0; + int64_t version = -1; + sscanf((const char *)taosArrayGetP(output, i), "v%dqinf.v%" PRIi64, &vid, &version); + SQTaskFile qTaskFile = {.version = version, .nRef = 1}; + if ((terrno = tdRSmaFSUpsertQTaskFile(RSMA_FS(pStat), &qTaskFile)) < 0) { + goto _end; + } + smaInfo("vgId:%d, open fs, version:%" PRIi64 ", ref:%" PRIi64, TD_VID(pVnode), qTaskFile.version, qTaskFile.nRef); + } + +_end: + for (int32_t i = 0; i < taosArrayGetSize(output); ++i) { + void *ptr = taosArrayGetP(output, i); + taosMemoryFreeClear(ptr); + } + taosArrayDestroy(output); + + if (terrno != TSDB_CODE_SUCCESS) { + smaError("vgId:%d, open rsma fs failed since %s", TD_VID(pVnode), terrstr()); + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +void tdRSmaFSClose(SRSmaFS *fs) { taosArrayDestroy(fs->aQTaskInf); } + +static int32_t tdQTaskInfCmprFn1(const void *p1, const void *p2) { + if (*(int64_t *)p1 < ((SQTaskFile *)p2)->version) { + return -1; + } else if (*(int64_t *)p1 > ((SQTaskFile *)p2)->version) { + return 1; + } + return 0; +} + +int32_t tdRSmaFSRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { + SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; + SQTaskFile *pTaskF = NULL; + int32_t oldVal = 0; + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + if ((pTaskF = taosArraySearch(aQTaskInf, &version, tdQTaskInfCmprFn1, TD_EQ))) { + oldVal = atomic_fetch_add_32(&pTaskF->nRef, 1); + ASSERT(oldVal > 0); + } + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + return oldVal; +} + +int64_t tdRSmaFSMaxVer(SSma *pSma, SRSmaStat *pStat) { + SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; + int64_t version = -1; + + taosRLockLatch(RSMA_FS_LOCK(pStat)); + if (taosArrayGetSize(aQTaskInf) > 0) { + version = ((SQTaskFile *)taosArrayGetLast(aQTaskInf))->version; + } + taosRUnLockLatch(RSMA_FS_LOCK(pStat)); + return version; +} + +void tdRSmaFSUnRef(SSma *pSma, SRSmaStat *pStat, int64_t version) { + SVnode *pVnode = pSma->pVnode; + SArray *aQTaskInf = RSMA_FS(pStat)->aQTaskInf; + char qTaskFullName[TSDB_FILENAME_LEN]; + SQTaskFile *pTaskF = NULL; + int32_t idx = -1; + + taosWLockLatch(RSMA_FS_LOCK(pStat)); + if ((idx = taosArraySearchIdx(aQTaskInf, &version, tdQTaskInfCmprFn1, TD_EQ)) >= 0) { + ASSERT(idx < taosArrayGetSize(aQTaskInf)); + pTaskF = taosArrayGet(aQTaskInf, idx); + if (atomic_sub_fetch_32(&pTaskF->nRef, 1) <= 0) { + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), pTaskF->version, tfsGetPrimaryPath(pVnode->pTfs), qTaskFullName); + if (taosRemoveFile(qTaskFullName) < 0) { + smaWarn("vgId:%d, failed to remove %s since %s", TD_VID(pVnode), qTaskFullName, + tstrerror(TAOS_SYSTEM_ERROR(errno))); + } else { + smaDebug("vgId:%d, success to remove %s", TD_VID(pVnode), qTaskFullName); + } + taosArrayRemove(aQTaskInf, idx); + } + } + taosWUnLockLatch(RSMA_FS_LOCK(pStat)); +} + +/** + * @brief Fetch qtaskfiles LE than version + * + * @param pSma + * @param version + * @param output + * @return int32_t + */ +static int32_t tdFetchQTaskInfoFiles(SSma *pSma, int64_t version, SArray **output) { + SVnode *pVnode = pSma->pVnode; + TdDirPtr pDir = NULL; + TdDirEntryPtr pDirEntry = NULL; + char dir[TSDB_FILENAME_LEN]; + const char *pattern = "v[0-9]+qinf\\.v([0-9]+)?$"; + regex_t regex; + int code = 0; + + tdGetVndDirName(TD_VID(pVnode), tfsGetPrimaryPath(pVnode->pTfs), VNODE_RSMA_DIR, true, dir); + + if (!taosCheckExistFile(dir)) { + smaDebug("vgId:%d, fetch qtask files, no need as dir %s not exist", TD_VID(pVnode), dir); + return TSDB_CODE_SUCCESS; + } + + // Resource allocation and init + if ((code = regcomp(®ex, pattern, REG_EXTENDED)) != 0) { + terrno = TSDB_CODE_RSMA_REGEX_MATCH; + char errbuf[128]; + regerror(code, ®ex, errbuf, sizeof(errbuf)); + smaWarn("vgId:%d, fetch qtask files, regcomp for %s failed since %s", TD_VID(pVnode), dir, errbuf); + return TSDB_CODE_FAILED; + } + + if (!(pDir = taosOpenDir(dir))) { + regfree(®ex); + terrno = TAOS_SYSTEM_ERROR(errno); + smaError("vgId:%d, fetch qtask files, open dir %s failed since %s", TD_VID(pVnode), dir, terrstr()); + return TSDB_CODE_FAILED; + } + + int32_t dirLen = strlen(dir); + char *dirEnd = POINTER_SHIFT(dir, dirLen); + regmatch_t regMatch[2]; + while ((pDirEntry = taosReadDir(pDir))) { + char *entryName = taosGetDirEntryName(pDirEntry); + if (!entryName) { + continue; + } + + code = regexec(®ex, entryName, 2, regMatch, 0); + + if (code == 0) { + // match + smaInfo("vgId:%d, fetch qtask files, max ver:%" PRIi64 ", %s found", TD_VID(pVnode), version, entryName); + + int64_t ver = -1; + sscanf((const char *)POINTER_SHIFT(entryName, regMatch[1].rm_so), "%" PRIi64, &ver); + if ((ver <= version) && (ver > -1)) { + if (!(*output)) { + if (!(*output = taosArrayInit(1, POINTER_BYTES))) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + } + char *entryDup = strdup(entryName); + if (!entryDup) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + if (!taosArrayPush(*output, &entryDup)) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _end; + } + } else { + } + } else if (code == REG_NOMATCH) { + // not match + smaTrace("vgId:%d, fetch qtask files, not match %s", TD_VID(pVnode), entryName); + continue; + } else { + // has other error + char errbuf[128]; + regerror(code, ®ex, errbuf, sizeof(errbuf)); + smaWarn("vgId:%d, fetch qtask files, regexec failed since %s", TD_VID(pVnode), errbuf); + terrno = TSDB_CODE_RSMA_REGEX_MATCH; + goto _end; + } + } +_end: + taosCloseDir(&pDir); + regfree(®ex); + return terrno == 0 ? TSDB_CODE_SUCCESS : TSDB_CODE_FAILED; +} + +static int32_t tdQTaskFileCmprFn2(const void *p1, const void *p2) { + if (((SQTaskFile *)p1)->version < ((SQTaskFile *)p2)->version) { + return -1; + } else if (((SQTaskFile *)p1)->version > ((SQTaskFile *)p2)->version) { + return 1; + } + + return 0; +} + +int32_t tdRSmaFSUpsertQTaskFile(SRSmaFS *pFS, SQTaskFile *qTaskFile) { + int32_t code = 0; + int32_t idx = taosArraySearchIdx(pFS->aQTaskInf, qTaskFile, tdQTaskFileCmprFn2, TD_GE); + + if (idx < 0) { + idx = taosArrayGetSize(pFS->aQTaskInf); + } else { + SQTaskFile *pTaskF = (SQTaskFile *)taosArrayGet(pFS->aQTaskInf, idx); + int32_t c = tdQTaskFileCmprFn2(pTaskF, qTaskFile); + if (c == 0) { + pTaskF->nRef = qTaskFile->nRef; + pTaskF->version = qTaskFile->version; + pTaskF->size = qTaskFile->size; + goto _exit; + } + } + + if (taosArrayInsert(pFS->aQTaskInf, idx, qTaskFile) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + +_exit: + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index 3c3097bb2fa7e7717a61d11894490ddc9f571790..d9ffda279f16501ac8f39fde7fe14728640db676 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -150,7 +150,7 @@ int32_t smaOpen(SVnode *pVnode) { } // restore the rsma - if (tdRsmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed) < 0) { + if (tdRSmaRestore(pSma, RSMA_RESTORE_REBOOT, pVnode->state.committed) < 0) { goto _err; } } @@ -181,8 +181,8 @@ int32_t smaClose(SSma *pSma) { * @param committedVer * @return int32_t */ -int32_t tdRsmaRestore(SSma *pSma, int8_t type, int64_t committedVer) { +int32_t tdRSmaRestore(SSma *pSma, int8_t type, int64_t committedVer) { ASSERT(VND_IS_RSMA(pSma->pVnode)); - return tdProcessRSmaRestoreImpl(pSma, type, committedVer); + return tdRSmaProcessRestoreImpl(pSma, type, committedVer); } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index d184926817ddfe5517ca675a9e203e5a349ccf47..468ce5e2d12b6cbae7db8f26740dc078e07a8d6b 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -19,8 +19,8 @@ #define RSMA_QTASKINFO_HEAD_LEN (sizeof(int32_t) + sizeof(int8_t) + sizeof(int64_t)) // len + type + suid #define RSMA_QTASKEXEC_SMOOTH_SIZE (100) // cnt #define RSMA_SUBMIT_BATCH_SIZE (1024) // cnt -#define RSMA_FETCH_DELAY_MAX (900000) // ms -#define RSMA_FETCH_ACTIVE_MAX (1800) // ms +#define RSMA_FETCH_DELAY_MAX (120000) // ms +#define RSMA_FETCH_ACTIVE_MAX (1000) // ms #define RSMA_FETCH_INTERVAL (5000) // ms SSmaMgmt smaMgmt = { @@ -34,7 +34,7 @@ typedef struct SRSmaQTaskInfoItem SRSmaQTaskInfoItem; typedef struct SRSmaQTaskInfoIter SRSmaQTaskInfoIter; static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); -static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd); static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat *pStat, SRSmaInfo *pRSmaInfo, int8_t idx); static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, int32_t inputType, SRSmaInfo *pInfo, @@ -42,10 +42,12 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSiz static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid); static void tdReleaseRSmaInfo(SSma *pSma, SRSmaInfo *pInfo); static void tdFreeRSmaSubmitItems(SArray *pItems); -static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr); +static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo); static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSmaInfoItem *pItem, STSchema *pTSchema, int64_t suid); static void tdRSmaFetchTrigger(void *param, void *tmrId); +static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo); +static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level); static int32_t tdRSmaQTaskInfoIterInit(SRSmaQTaskInfoIter *pIter, STFile *pTFile); static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isFinish); static int32_t tdRSmaQTaskInfoRestore(SSma *pSma, int8_t type, SRSmaQTaskInfoIter *pIter); @@ -96,7 +98,7 @@ static FORCE_INLINE int32_t tdRSmaQTaskInfoContLen(int32_t lenWithHead) { static FORCE_INLINE void tdRSmaQTaskInfoIterDestroy(SRSmaQTaskInfoIter *pIter) { taosMemoryFreeClear(pIter->pBuf); } -void tdFreeQTaskInfo(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { +static void tdRSmaQTaskInfoFree(qTaskInfo_t *taskHandle, int32_t vgId, int32_t level) { // Note: free/kill may in RC if (!taskHandle || !(*taskHandle)) return; qTaskInfo_t otaskHandle = atomic_load_ptr(taskHandle); @@ -123,20 +125,20 @@ void *tdFreeRSmaInfo(SSma *pSma, SRSmaInfo *pInfo, bool isDeepFree) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (isDeepFree && pItem->tmrId) { - smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pInfo->suid, - pItem->tmrId, i + 1); + smaDebug("vgId:%d, stop fetch timer %p for table %" PRIi64 " level %d", SMA_VID(pSma), pItem->tmrId, + pInfo->suid, i + 1); taosTmrStopA(&pItem->tmrId); } if (isDeepFree && pInfo->taskInfo[i]) { - tdFreeQTaskInfo(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); + tdRSmaQTaskInfoFree(&pInfo->taskInfo[i], SMA_VID(pSma), i + 1); } else { smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty taskInfo", SMA_VID(pSma), pInfo->suid, i + 1); } if (pInfo->iTaskInfo[i]) { - tdFreeQTaskInfo(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); + tdRSmaQTaskInfoFree(&pInfo->iTaskInfo[i], SMA_VID(pSma), i + 1); } else { smaDebug("vgId:%d, table %" PRIi64 " no need to destroy rsma info level %d since empty iTaskInfo", SMA_VID(pSma), pInfo->suid, i + 1); @@ -173,7 +175,7 @@ static FORCE_INLINE int32_t tdUidStoreInit(STbUidStore **pStore) { return TSDB_CODE_SUCCESS; } -static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) { +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, bool isAdd) { SRSmaInfo *pRSmaInfo = NULL; if (!suid || !tbUids) { @@ -197,7 +199,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, true)) < 0)) { + if (((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0)) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); @@ -213,12 +215,12 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids) return TSDB_CODE_SUCCESS; } -int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) { +int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore, bool isAdd) { if (!pStore || (taosArrayGetSize(pStore->tbUids) == 0)) { return TSDB_CODE_SUCCESS; } - if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids) != TSDB_CODE_SUCCESS) { + if (tdUpdateTbUidListImpl(pSma, &pStore->suid, pStore->tbUids, isAdd) != TSDB_CODE_SUCCESS) { return TSDB_CODE_FAILED; } @@ -227,7 +229,7 @@ int32_t tdUpdateTbUidList(SSma *pSma, STbUidStore *pStore) { tb_uid_t *pTbSuid = (tb_uid_t *)taosHashGetKey(pIter, NULL); SArray *pTbUids = *(SArray **)pIter; - if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids) != TSDB_CODE_SUCCESS) { + if (tdUpdateTbUidListImpl(pSma, pTbSuid, pTbUids, isAdd) != TSDB_CODE_SUCCESS) { taosHashCancelIterate(pStore->uidHash, pIter); return TSDB_CODE_FAILED; } @@ -300,7 +302,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat return TSDB_CODE_FAILED; } SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); - pItem->triggerStat = TASK_TRIGGER_STAT_INACTIVE; + pItem->triggerStat = TASK_TRIGGER_STAT_ACTIVE; // fetch the data when reboot if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { int64_t msInterval = convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); @@ -313,10 +315,19 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat } pItem->level = idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2; - taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - smaInfo("vgId:%d, table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 + ASSERT(pItem->level > 0); + + SRSmaRef rsmaRef = {.refId = pStat->refId, .suid = pRSmaInfo->suid}; + taosHashPut(smaMgmt.refHash, &pItem, POINTER_BYTES, &rsmaRef, sizeof(rsmaRef)); + + pItem->fetchLevel = pItem->level; + taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); + + + smaInfo("vgId:%d, item:%p table:%" PRIi64 " level:%" PRIi8 " maxdelay:%" PRIi64 " watermark:%" PRIi64 ", finally maxdelay:%" PRIi32, - TD_VID(pVnode), pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], pItem->maxDelay); + TD_VID(pVnode), pItem, pRSmaInfo->suid, idx + 1, param->maxdelay[idx], param->watermark[idx], + pItem->maxDelay); } return TSDB_CODE_SUCCESS; } @@ -330,7 +341,7 @@ static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaStat * @param tbName * @return int32_t */ -int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { +int32_t tdRSmaProcessCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, const char *tbName) { if ((param->qmsgLen[0] == 0) && (param->qmsgLen[1] == 0)) { smaDebug("vgId:%d, no qmsg1/qmsg2 for rollup table %s %" PRIi64, SMA_VID(pSma), tbName, suid); return TSDB_CODE_SUCCESS; @@ -368,7 +379,10 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con terrno = TSDB_CODE_TDB_IVD_TB_SCHEMA_VERSION; goto _err; } + pRSmaInfo->pSma = pSma; pRSmaInfo->pTSchema = pTSchema; + pRSmaInfo->suid = suid; + T_REF_INIT_VAL(pRSmaInfo, 1); if (!(pRSmaInfo->queue = taosOpenQueue())) { goto _err; } @@ -382,9 +396,6 @@ int32_t tdProcessRSmaCreateImpl(SSma *pSma, SRSmaParam *param, int64_t suid, con if (!(pRSmaInfo->iQall = taosAllocateQall())) { goto _err; } - pRSmaInfo->suid = suid; - pRSmaInfo->refId = RSMA_REF_ID(pStat); - T_REF_INIT_VAL(pRSmaInfo, 1); if (tdSetRSmaInfoItemParams(pSma, param, pStat, pRSmaInfo, 0) < 0) { goto _err; @@ -427,7 +438,7 @@ int32_t tdProcessRSmaCreate(SSma *pSma, SVCreateStbReq *pReq) { return TSDB_CODE_SUCCESS; } - return tdProcessRSmaCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name); + return tdRSmaProcessCreateImpl(pSma, &pReq->rsmaParam, pReq->suid, pReq->name); } /** @@ -461,6 +472,7 @@ int32_t tdProcessRSmaDrop(SSma *pSma, SVDropStbReq *pReq) { } // set del flag for data in mem + atomic_store_8(&pRSmaStat->delFlag, 1); RSMA_INFO_SET_DEL(pRSmaInfo); tdUnRefRSmaInfo(pSma, pRSmaInfo); @@ -654,15 +666,15 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma if (taosArrayGetSize(pResList) == 0) { if (terrno == 0) { - // smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); + // smaDebug("vgId:%d, no rsma level %" PRIi8 " data fetched yet", SMA_VID(pSma), pItem->level); } else { - smaDebug("vgId:%d, no rsma %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); + smaDebug("vgId:%d, no rsma level %" PRIi8 " data fetched since %s", SMA_VID(pSma), pItem->level, terrstr()); goto _err; } break; } else { - smaDebug("vgId:%d, rsma %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); + smaDebug("vgId:%d, rsma level %" PRIi8 " data fetched", SMA_VID(pSma), pItem->level); } #if 0 char flag[10] = {0}; @@ -676,21 +688,22 @@ static int32_t tdRSmaExecAndSubmitResult(SSma *pSma, qTaskInfo_t taskInfo, SRSma // TODO: the schema update should be handled later(TD-17965) if (buildSubmitReqFromDataBlock(&pReq, output, pTSchema, SMA_VID(pSma), suid) < 0) { - smaError("vgId:%d, build submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", - SMA_VID(pSma), suid, pItem->level, terrstr()); + smaError("vgId:%d, build submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), + suid, pItem->level, terrstr()); goto _err; } if (pReq && tdProcessSubmitReq(sinkTsdb, output->info.version, pReq) < 0) { taosMemoryFreeClear(pReq); - smaError("vgId:%d, process submit req for rsma stable %" PRIi64 " level %" PRIi8 " failed since %s", + smaError("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " failed since %s", SMA_VID(pSma), suid, pItem->level, terrstr()); goto _err; } - taosMemoryFreeClear(pReq); - smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " version:%" PRIi64, - SMA_VID(pSma), suid, pItem->level, output->info.version); + smaDebug("vgId:%d, process submit req for rsma table %" PRIi64 " level %" PRIi8 " ver %" PRIi64 " len %" PRIu32, + SMA_VID(pSma), suid, pItem->level, output->info.version, htonl(pReq->header.contLen)); + + taosMemoryFreeClear(pReq); } } @@ -817,6 +830,95 @@ static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t msgSize, return TSDB_CODE_SUCCESS; } +static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, + tb_uid_t suid, int8_t idx) { + SVnode *pVnode = pSma->pVnode; + char *pOutput = NULL; + int32_t len = 0; + + if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { + smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + SReadHandle handle = { + .meta = pVnode->pMeta, + .vnode = pVnode, + .initTqReader = 1, + }; + ASSERT(!dstTaskInfo); + dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); + if (!dstTaskInfo) { + terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; + goto _err; + } + + if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + goto _err; + } + + smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); + + taosMemoryFreeClear(pOutput); + return TSDB_CODE_SUCCESS; +_err: + taosMemoryFreeClear(pOutput); + tdRSmaQTaskInfoFree(dstTaskInfo, TD_VID(pVnode), idx + 1); + smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, + terrstr()); + return TSDB_CODE_FAILED; +} + +/** + * @brief Clone qTaskInfo of SRSmaInfo + * + * @param pSma + * @param pInfo + * @return int32_t + */ +static int32_t tdRSmaInfoClone(SSma *pSma, SRSmaInfo *pInfo) { + SRSmaParam *param = NULL; + if (!pInfo) { + return TSDB_CODE_SUCCESS; + } + + SMetaReader mr = {0}; + metaReaderInit(&mr, SMA_META(pSma), 0); + smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); + if (metaGetTableEntryByUid(&mr, pInfo->suid) < 0) { + smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, + terrstr()); + goto _err; + } + ASSERT(mr.me.type == TSDB_SUPER_TABLE); + ASSERT(mr.me.uid == pInfo->suid); + if (TABLE_IS_ROLLUP(mr.me.flags)) { + param = &mr.me.stbEntry.rsmaParam; + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { + if (!pInfo->iTaskInfo[i]) { + continue; + } + if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { + goto _err; + } + } + smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); + } else { + terrno = TSDB_CODE_RSMA_INVALID_SCHEMA; + goto _err; + } + + metaReaderClear(&mr); + return TSDB_CODE_SUCCESS; +_err: + metaReaderClear(&mr); + smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); + return TSDB_CODE_FAILED; +} + /** * @brief During async commit, the SRSmaInfo object would be COW from iRSmaInfoHash and write lock should be applied. * @@ -840,25 +942,25 @@ static SRSmaInfo *tdAcquireRSmaInfoBySuid(SSma *pSma, int64_t suid) { return NULL; } - // taosRLockLatch(SMA_ENV_LOCK(pEnv)); + taosRLockLatch(SMA_ENV_LOCK(pEnv)); pRSmaInfo = taosHashGet(RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (pRSmaInfo && (pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } if (!pRSmaInfo->taskInfo[0]) { - if (tdCloneRSmaInfo(pSma, pRSmaInfo) < 0) { - // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + if (tdRSmaInfoClone(pSma, pRSmaInfo) < 0) { + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } } tdRefRSmaInfo(pSma, pRSmaInfo); - // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); ASSERT(pRSmaInfo->suid == suid); return pRSmaInfo; } - // taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); + taosRUnLockLatch(SMA_ENV_LOCK(pEnv)); return NULL; } @@ -1006,7 +1108,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { " qmsgLen:%" PRIi32, TD_VID(pVnode), suid, i, param->maxdelay[i], param->watermark[i], param->qmsgLen[i]); } - if (tdProcessRSmaCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { + if (tdRSmaProcessCreateImpl(pSma, &mr.me.stbEntry.rsmaParam, suid, mr.me.name) < 0) { smaError("vgId:%d, rsma restore env failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); goto _err; } @@ -1019,7 +1121,7 @@ static int32_t tdRSmaRestoreQTaskInfoInit(SSma *pSma, int64_t *nTables) { goto _err; } - if (tdUpdateTbUidList(pVnode->pSma, &uidStore) < 0) { + if (tdUpdateTbUidList(pVnode->pSma, &uidStore, true) < 0) { smaError("vgId:%d, rsma restore, update tb uid list failed for %" PRIi64 " since %s", TD_VID(pVnode), suid, terrstr()); goto _err; @@ -1118,7 +1220,7 @@ static int32_t tdRSmaRestoreTSDataReload(SSma *pSma) { return TSDB_CODE_SUCCESS; } -int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) { +int32_t tdRSmaProcessRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) { // step 1: iterate all stables to restore the rsma env int64_t nTables = 0; if (tdRSmaRestoreQTaskInfoInit(pSma, &nTables) < 0) { @@ -1139,6 +1241,12 @@ int32_t tdProcessRSmaRestoreImpl(SSma *pSma, int8_t type, int64_t qtaskFileVer) if (tdRSmaRestoreTSDataReload(pSma) < 0) { goto _err; } + + // step 4: open SRSmaFS for qTaskFiles + if (tdRSmaFSOpen(pSma, qtaskFileVer) < 0) { + goto _err; + } + smaInfo("vgId:%d, restore rsma task %" PRIi8 " from qtaskf %" PRIi64 " succeed", SMA_VID(pSma), type, qtaskFileVer); return TSDB_CODE_SUCCESS; _err: @@ -1237,29 +1345,31 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF return TSDB_CODE_FAILED; } - if (tdReadTFile(pTFile, pIter->qBuf, nBytes) != nBytes) { + if (tdReadTFile(pTFile, pIter->pBuf, nBytes) != nBytes) { return TSDB_CODE_FAILED; } int32_t infoLen = 0; - taosDecodeFixedI32(pIter->qBuf, &infoLen); + taosDecodeFixedI32(pIter->pBuf, &infoLen); if (infoLen > nBytes) { if (infoLen <= RSMA_QTASKINFO_BUFSIZE) { terrno = TSDB_CODE_RSMA_FILE_CORRUPTED; smaError("iterate rsma qtaskinfo file %s failed since %s", TD_TFILE_FULL_NAME(pIter->pTFile), terrstr()); return TSDB_CODE_FAILED; } - pIter->nAlloc = infoLen; - void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); - if (!pBuf) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return TSDB_CODE_FAILED; + if (pIter->nAlloc < infoLen) { + pIter->nAlloc = infoLen; + void *pBuf = taosMemoryRealloc(pIter->pBuf, infoLen); + if (!pBuf) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return TSDB_CODE_FAILED; + } + pIter->pBuf = pBuf; } - pIter->pBuf = pBuf; - pIter->qBuf = pIter->pBuf; + nBytes = infoLen; - if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET)) { + if (tdSeekTFile(pTFile, pIter->offset, SEEK_SET) < 0) { return TSDB_CODE_FAILED; } @@ -1268,6 +1378,7 @@ static int32_t tdRSmaQTaskInfoIterNextBlock(SRSmaQTaskInfoIter *pIter, bool *isF } } + pIter->qBuf = pIter->pBuf; pIter->offset += nBytes; pIter->nBytes = nBytes; pIter->nBufPos = 0; @@ -1345,17 +1456,24 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { return TSDB_CODE_SUCCESS; } + int64_t fsMaxVer = tdRSmaFSMaxVer(pSma, pRSmaStat); + if (pRSmaStat->commitAppliedVer <= fsMaxVer) { + smaDebug("vgId:%d, rsma persist, no need as applied %" PRIi64 " not larger than fsMaxVer %" PRIi64, vid, + pRSmaStat->commitAppliedVer, fsMaxVer); + return TSDB_CODE_SUCCESS; + } + STFile tFile = {0}; #if 0 if (pRSmaStat->commitAppliedVer > 0) { char qTaskInfoFName[TSDB_FILENAME_LEN]; tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); + smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; } if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); + smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); goto _err; } smaDebug("vgId:%d, rsma, serialize qTaskInfo, file %s created", vid, TD_TFILE_FULL_NAME(&tFile)); @@ -1405,11 +1523,11 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { char qTaskInfoFName[TSDB_FILENAME_LEN]; tdRSmaQTaskInfoGetFileName(vid, pRSmaStat->commitAppliedVer, qTaskInfoFName); if (tdInitTFile(&tFile, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFName) < 0) { - smaError("vgId:%d, rsma persit, init %s failed since %s", vid, qTaskInfoFName, terrstr()); + smaError("vgId:%d, rsma persist, init %s failed since %s", vid, qTaskInfoFName, terrstr()); goto _err; } if (tdCreateTFile(&tFile, true, TD_FTYPE_RSMA_QTASKINFO) < 0) { - smaError("vgId:%d, rsma persit, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); + smaError("vgId:%d, rsma persist, create %s failed since %s", vid, TD_TFILE_FULL_NAME(&tFile), terrstr()); goto _err; } smaDebug("vgId:%d, rsma, table %" PRIi64 " serialize qTaskInfo, file %s created", vid, pRSmaInfo->suid, @@ -1453,7 +1571,7 @@ int32_t tdRSmaPersistExecImpl(SRSmaStat *pRSmaStat, SHashObj *pInfoHash) { } return TSDB_CODE_SUCCESS; _err: - smaError("vgId:%d, rsma persit failed since %s", vid, terrstr()); + smaError("vgId:%d, rsma persist failed since %s", vid, terrstr()); if (isFileCreated) { tdRemoveTFile(&tFile); tdDestroyTFile(&tFile); @@ -1468,26 +1586,46 @@ _err: * @param tmrId */ static void tdRSmaFetchTrigger(void *param, void *tmrId) { - SRSmaInfoItem *pItem = param; + SRSmaRef *pRSmaRef = NULL; SSma *pSma = NULL; - SRSmaInfo *pRSmaInfo = tdGetRSmaInfoByItem(pItem); + SRSmaStat *pStat = NULL; + SRSmaInfo *pRSmaInfo = NULL; + SRSmaInfoItem *pItem = NULL; - if (RSMA_INFO_IS_DEL(pRSmaInfo)) { - smaDebug("rsma fetch task not start since rsma info already deleted, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, - pRSmaInfo->refId); + if (!(pRSmaRef = taosHashGet(smaMgmt.refHash, ¶m, POINTER_BYTES))) { + smaDebug("rsma fetch task not start since rsma info item:%p not exist in refHash:%p, rsetId:%" PRIi64, param, + *(int64_t *)¶m, smaMgmt.refHash, smaMgmt.rsetId); return; } - SRSmaStat *pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); - - if (!pStat) { + if (!(pStat = (SRSmaStat *)tdAcquireSmaRef(smaMgmt.rsetId, pRSmaRef->refId))) { smaDebug("rsma fetch task not start since rsma stat already destroyed, rsetId:%" PRIi64 " refId:%d)", - smaMgmt.rsetId, pRSmaInfo->refId); + smaMgmt.rsetId, pRSmaRef->refId); // pRSmaRef freed in taosHashRemove + taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); return; } pSma = pStat->pSma; + if (!(pRSmaInfo = tdAcquireRSmaInfoBySuid(pSma, pRSmaRef->suid))) { + smaDebug("rsma fetch task not start since rsma info not exist, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + pRSmaRef->refId); // pRSmaRef freed in taosHashRemove + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); + taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); + return; + } + + if (RSMA_INFO_IS_DEL(pRSmaInfo)) { + smaDebug("rsma fetch task not start since rsma info already deleted, rsetId:%" PRIi64 " refId:%d)", smaMgmt.rsetId, + pRSmaRef->refId); // pRSmaRef freed in taosHashRemove + tdReleaseRSmaInfo(pSma, pRSmaInfo); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); + taosHashRemove(smaMgmt.refHash, ¶m, POINTER_BYTES); + return; + } + + pItem = *(SRSmaInfoItem **)¶m; + // if rsma trigger stat in paused, cancelled or finished, not start fetch task int8_t rsmaTriggerStat = atomic_load_8(RSMA_TRIGGER_STAT(pStat)); switch (rsmaTriggerStat) { @@ -1495,11 +1633,12 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { case TASK_TRIGGER_STAT_CANCELLED: { smaDebug("vgId:%d, rsma fetch task not start for level %" PRIi8 " since stat is %" PRIi8 ", rsetId rsetId:%" PRIi64 " refId:%d", - SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaInfo->refId); + SMA_VID(pSma), pItem->level, rsmaTriggerStat, smaMgmt.rsetId, pRSmaRef->refId); if (rsmaTriggerStat == TASK_TRIGGER_STAT_PAUSED) { taosTmrReset(tdRSmaFetchTrigger, RSMA_FETCH_INTERVAL, pItem, smaMgmt.tmrHandle, &pItem->tmrId); } - tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); + tdReleaseRSmaInfo(pSma, pRSmaInfo); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); return; } default: @@ -1544,11 +1683,11 @@ static void tdRSmaFetchTrigger(void *param, void *tmrId) { _end: taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, smaMgmt.tmrHandle, &pItem->tmrId); - tdReleaseSmaRef(smaMgmt.rsetId, pRSmaInfo->refId); + tdReleaseRSmaInfo(pSma, pRSmaInfo); + tdReleaseSmaRef(smaMgmt.rsetId, pRSmaRef->refId); } static void tdFreeRSmaSubmitItems(SArray *pItems) { - ASSERT(taosArrayGetSize(pItems) > 0); for (int32_t i = 0; i < taosArrayGetSize(pItems); ++i) { taosFreeQitem(*(void **)taosArrayGet(pItems, i)); } @@ -1560,10 +1699,9 @@ static void tdFreeRSmaSubmitItems(SArray *pItems) { * * @param pSma * @param pInfo - * @param pSubmitArr * @return int32_t */ -static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmitArr) { +static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo) { SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; for (int8_t i = 1; i <= TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = RSMA_INFO_ITEM(pInfo, i - 1); @@ -1574,21 +1712,23 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmi continue; } - int64_t curMs = taosGetTimestampMs(); - if ((pItem->nSkipped * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { - smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch executed", - SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); - } else if (((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX)) { - ++pItem->nSkipped; - smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", - SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); - continue; + if ((++pItem->nScanned * pItem->maxDelay) > RSMA_FETCH_DELAY_MAX) { + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi8 " maxDelay:%d, fetch executed", + SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay); } else { - smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", - SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + int64_t curMs = taosGetTimestampMs(); + if ((curMs - pInfo->lastRecv) < RSMA_FETCH_ACTIVE_MAX) { + smaTrace("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch skipped ", + SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + atomic_store_8(&pItem->triggerStat, TASK_TRIGGER_STAT_ACTIVE); // restore the active stat + continue; + } else { + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " curMs:%" PRIi64 " lastRecv:%" PRIi64 ", fetch executed ", + SMA_VID(pSma), pInfo->suid, i, curMs, pInfo->lastRecv); + } } - pItem->nSkipped = 0; + pItem->nScanned = 0; if ((terrno = qSetMultiStreamInput(taskInfo, &dataBlock, 1, STREAM_INPUT__DATA_BLOCK)) < 0) { goto _err; @@ -1599,20 +1739,18 @@ static int32_t tdRSmaFetchAllResult(SSma *pSma, SRSmaInfo *pInfo, SArray *pSubmi } tdCleanupStreamInputDataBlock(taskInfo); - smaInfo("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 " maxDelay:%d, fetch finished", - SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay); + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi8 " maxDelay:%d, fetch finished", + SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay); } else { - smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nSkipped:%" PRIi8 + smaDebug("vgId:%d, suid:%" PRIi64 " level:%" PRIi8 " nScanned:%" PRIi8 " maxDelay:%d, fetch not executed as fetch level is %" PRIi8, - SMA_VID(pSma), pInfo->suid, i, pItem->nSkipped, pItem->maxDelay, pItem->fetchLevel); + SMA_VID(pSma), pInfo->suid, i, pItem->nScanned, pItem->maxDelay, pItem->fetchLevel); } } _end: - tdReleaseRSmaInfo(pSma, pInfo); return TSDB_CODE_SUCCESS; _err: - tdReleaseRSmaInfo(pSma, pInfo); return TSDB_CODE_FAILED; } @@ -1696,6 +1834,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { bool occupied = (batchMax <= 1); if (batchMax > 1) { batchMax = 100 / batchMax; + batchMax = TMAX(batchMax, 4); } while (occupied || (++batchCnt < batchMax)) { // greedy mode taosReadAllQitems(pInfo->queue, pInfo->qall); // queue has mutex lock @@ -1705,13 +1844,15 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { smaDebug("vgId:%d, batchSize:%d, execType:%" PRIi8, SMA_VID(pSma), qallItemSize, type); } - int8_t oldStat = atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 2); - if (oldStat == 0 || - ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) { - atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1); - tdRSmaFetchAllResult(pSma, pInfo, pSubmitArr); - if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) { - atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); + if (RSMA_INFO_ITEM(pInfo, 0)->fetchLevel || RSMA_INFO_ITEM(pInfo, 1)->fetchLevel) { + int8_t oldStat = atomic_val_compare_exchange_8(RSMA_COMMIT_STAT(pRSmaStat), 0, 2); + if (oldStat == 0 || + ((oldStat == 2) && atomic_load_8(RSMA_TRIGGER_STAT(pRSmaStat)) < TASK_TRIGGER_STAT_PAUSED)) { + atomic_fetch_add_32(&pRSmaStat->nFetchAll, 1); + tdRSmaFetchAllResult(pSma, pInfo); + if (0 == atomic_sub_fetch_32(&pRSmaStat->nFetchAll, 1)) { + atomic_store_8(RSMA_COMMIT_STAT(pRSmaStat), 0); + } } } @@ -1784,7 +1925,7 @@ int32_t tdRSmaProcessExecImpl(SSma *pSma, ERsmaExecType type) { tsem_wait(&pRSmaStat->notEmpty); if ((pEnv->flag & SMA_ENV_FLG_CLOSE) && (atomic_load_64(&pRSmaStat->nBufItems) <= 0)) { - smaInfo("vgId:%d, exec task end, flag:%" PRIi8 ", nBufItems:%" PRIi64, SMA_VID(pSma), pEnv->flag, + smaDebug("vgId:%d, exec task end, flag:%" PRIi8 ", nBufItems:%" PRIi64, SMA_VID(pSma), pEnv->flag, atomic_load_64(&pRSmaStat->nBufItems)); break; } diff --git a/source/dnode/vnode/src/sma/smaSnapshot.c b/source/dnode/vnode/src/sma/smaSnapshot.c index 335c15a539ef31d66d83377f90da225e45ffd893..5a0167a75fc799366396015f8323e770a217a1cb 100644 --- a/source/dnode/vnode/src/sma/smaSnapshot.c +++ b/source/dnode/vnode/src/sma/smaSnapshot.c @@ -15,11 +15,13 @@ #include "sma.h" -static int32_t rsmaSnapReadQTaskInfo(SRsmaSnapReader* pReader, uint8_t** ppData); -static int32_t rsmaSnapWriteQTaskInfo(SRsmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppData); +static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData); +static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t version); +static int32_t rsmaQTaskInfSnapReaderClose(SQTaskFReader** ppReader); -// SRsmaSnapReader ======================================== -struct SRsmaSnapReader { +// SRSmaSnapReader ======================================== +struct SRSmaSnapReader { SSma* pSma; int64_t sver; int64_t ever; @@ -33,13 +35,13 @@ struct SRsmaSnapReader { SQTaskFReader* pQTaskFReader; }; -int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapReader** ppReader) { +int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapReader** ppReader) { int32_t code = 0; SVnode* pVnode = pSma->pVnode; - SRsmaSnapReader* pReader = NULL; + SRSmaSnapReader* pReader = NULL; // alloc - pReader = (SRsmaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); + pReader = (SRSmaSnapReader*)taosMemoryCalloc(1, sizeof(*pReader)); if (pReader == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -48,7 +50,7 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapRead pReader->sver = sver; pReader->ever = ever; - // rsma1/rsma2 + // open rsma1/rsma2 for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pSma->pRSmaTsdb[i]) { code = tsdbSnapReaderOpen(pSma->pRSmaTsdb[i], sver, ever, i == 0 ? SNAP_DATA_RSMA1 : SNAP_DATA_RSMA2, @@ -59,51 +61,112 @@ int32_t rsmaSnapReaderOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapRead } } - // qtaskinfo - // 1. add ref to qtaskinfo.v${ever} if exists and then start to replicate + // open qtaskinfo + if ((code = rsmaQTaskInfSnapReaderOpen(pReader, ever)) < 0) { + goto _err; + } + + *ppReader = pReader; + + return TSDB_CODE_SUCCESS; +_err: + smaError("vgId:%d, vnode snapshot rsma reader open failed since %s", TD_VID(pVnode), tstrerror(code)); + return TSDB_CODE_FAILED; +} + +static int32_t rsmaQTaskInfSnapReaderOpen(SRSmaSnapReader* pReader, int64_t version) { + int32_t code = 0; + SSma* pSma = pReader->pSma; + SVnode* pVnode = pSma->pVnode; + SSmaEnv* pEnv = NULL; + SRSmaStat* pStat = NULL; + + if (!(pEnv = SMA_RSMA_ENV(pSma))) { + smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as env is NULL", + TD_VID(pVnode), version); + return TSDB_CODE_SUCCESS; + } + + pStat = (SRSmaStat*)SMA_ENV_STAT(pEnv); + + int32_t ref = tdRSmaFSRef(pReader->pSma, pStat, version); + if (ref < 1) { + smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as ref is %d", + TD_VID(pVnode), version, ref); + return TSDB_CODE_SUCCESS; + } + char qTaskInfoFullName[TSDB_FILENAME_LEN]; - tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), ever, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); + tdRSmaQTaskInfoGetFullName(TD_VID(pVnode), version, tfsGetPrimaryPath(pVnode->pTfs), qTaskInfoFullName); if (!taosCheckExistFile(qTaskInfoFullName)) { - smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo not need as %s not exists", TD_VID(pVnode), - qTaskInfoFullName); - } else { - pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); - if (!pReader->pQTaskFReader) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + tdRSmaFSUnRef(pSma, pStat, version); + smaInfo("vgId:%d, vnode snapshot rsma reader for qtaskinfo version %" PRIi64 " not need as %s not exists", + TD_VID(pVnode), qTaskInfoFullName); + return TSDB_CODE_SUCCESS; + } - TdFilePtr qTaskF = taosOpenFile(qTaskInfoFullName, TD_FILE_READ); - if (!qTaskF) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - pReader->pQTaskFReader->pReadH = qTaskF; -#if 0 - SQTaskFile* pQTaskF = &pReader->pQTaskFReader->fTask; - pQTaskF->nRef = 1; -#endif + pReader->pQTaskFReader = taosMemoryCalloc(1, sizeof(SQTaskFReader)); + if (!pReader->pQTaskFReader) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _end; } - *ppReader = pReader; - smaInfo("vgId:%d, vnode snapshot rsma reader opened %s succeed", TD_VID(pVnode), qTaskInfoFullName); + TdFilePtr fp = taosOpenFile(qTaskInfoFullName, TD_FILE_READ); + if (!fp) { + code = TAOS_SYSTEM_ERROR(errno); + taosMemoryFreeClear(pReader->pQTaskFReader); + goto _end; + } + + pReader->pQTaskFReader->pReadH = fp; + pReader->pQTaskFReader->pSma = pSma; + pReader->pQTaskFReader->version = pReader->ever; + +_end: + if (code < 0) { + tdRSmaFSUnRef(pSma, pStat, version); + smaError("vgId:%d, vnode snapshot rsma reader open %s succeed", TD_VID(pVnode), qTaskInfoFullName); + return TSDB_CODE_FAILED; + } + + smaInfo("vgId:%d, vnode snapshot rsma reader open %s succeed", TD_VID(pVnode), qTaskInfoFullName); + return TSDB_CODE_SUCCESS; +} + +static int32_t rsmaQTaskInfSnapReaderClose(SQTaskFReader** ppReader) { + if (!(*ppReader)) { + return TSDB_CODE_SUCCESS; + } + + SSma* pSma = (*ppReader)->pSma; + SRSmaStat* pStat = SMA_RSMA_STAT(pSma); + int64_t version = (*ppReader)->version; + + taosCloseFile(&(*ppReader)->pReadH); + tdRSmaFSUnRef(pSma, pStat, version); + taosMemoryFreeClear(*ppReader); + smaInfo("vgId:%d, vnode snapshot rsma reader closed for qTaskInfo version %" PRIi64, SMA_VID(pSma), version); + return TSDB_CODE_SUCCESS; -_err: - smaError("vgId:%d, vnode snapshot rsma reader opened failed since %s", TD_VID(pVnode), tstrerror(code)); - return TSDB_CODE_FAILED; } -static int32_t rsmaSnapReadQTaskInfo(SRsmaSnapReader* pReader, uint8_t** ppBuf) { +static int32_t rsmaSnapReadQTaskInfo(SRSmaSnapReader* pReader, uint8_t** ppBuf) { int32_t code = 0; SSma* pSma = pReader->pSma; int64_t n = 0; uint8_t* pBuf = NULL; SQTaskFReader* qReader = pReader->pQTaskFReader; + if (!qReader) { + *ppBuf = NULL; + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, qTaskReader is NULL", SMA_VID(pSma)); + return 0; + } + if (!qReader->pReadH) { *ppBuf = NULL; - smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, readh is empty", SMA_VID(pSma)); + smaInfo("vgId:%d, vnode snapshot rsma reader qtaskinfo, readh is NULL", SMA_VID(pSma)); return 0; } @@ -153,7 +216,7 @@ _err: return code; } -int32_t rsmaSnapRead(SRsmaSnapReader* pReader, uint8_t** ppData) { +int32_t rsmaSnapRead(SRSmaSnapReader* pReader, uint8_t** ppData) { int32_t code = 0; *ppData = NULL; @@ -205,9 +268,9 @@ _err: return code; } -int32_t rsmaSnapReaderClose(SRsmaSnapReader** ppReader) { +int32_t rsmaSnapReaderClose(SRSmaSnapReader** ppReader) { int32_t code = 0; - SRsmaSnapReader* pReader = *ppReader; + SRSmaSnapReader* pReader = *ppReader; for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pReader->pDataReader[i]) { @@ -215,11 +278,7 @@ int32_t rsmaSnapReaderClose(SRsmaSnapReader** ppReader) { } } - if (pReader->pQTaskFReader) { - taosCloseFile(&pReader->pQTaskFReader->pReadH); - taosMemoryFreeClear(pReader->pQTaskFReader); - smaInfo("vgId:%d, vnode snapshot rsma reader closed for qTaskInfo", SMA_VID(pReader->pSma)); - } + rsmaQTaskInfSnapReaderClose(&pReader->pQTaskFReader); smaInfo("vgId:%d, vnode snapshot rsma reader closed", SMA_VID(pReader->pSma)); @@ -227,8 +286,8 @@ int32_t rsmaSnapReaderClose(SRsmaSnapReader** ppReader) { return code; } -// SRsmaSnapWriter ======================================== -struct SRsmaSnapWriter { +// SRSmaSnapWriter ======================================== +struct SRSmaSnapWriter { SSma* pSma; int64_t sver; int64_t ever; @@ -244,13 +303,13 @@ struct SRsmaSnapWriter { SQTaskFWriter* pQTaskFWriter; }; -int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRsmaSnapWriter** ppWriter) { +int32_t rsmaSnapWriterOpen(SSma* pSma, int64_t sver, int64_t ever, SRSmaSnapWriter** ppWriter) { int32_t code = 0; - SRsmaSnapWriter* pWriter = NULL; + SRSmaSnapWriter* pWriter = NULL; SVnode* pVnode = pSma->pVnode; // alloc - pWriter = (SRsmaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); + pWriter = (SRSmaSnapWriter*)taosMemoryCalloc(1, sizeof(*pWriter)); if (pWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -301,9 +360,9 @@ _err: return code; } -int32_t rsmaSnapWriterClose(SRsmaSnapWriter** ppWriter, int8_t rollback) { +int32_t rsmaSnapWriterClose(SRSmaSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; - SRsmaSnapWriter* pWriter = *ppWriter; + SRSmaSnapWriter* pWriter = *ppWriter; SVnode* pVnode = pWriter->pSma->pVnode; if (rollback) { @@ -332,7 +391,7 @@ int32_t rsmaSnapWriterClose(SRsmaSnapWriter** ppWriter, int8_t rollback) { pWriter->pQTaskFWriter->fname, qTaskInfoFullName); // rsma restore - if ((code = tdRsmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever)) < 0) { + if ((code = tdRSmaRestore(pWriter->pSma, RSMA_RESTORE_SYNC, pWriter->ever)) < 0) { goto _err; } smaInfo("vgId:%d, vnode snapshot rsma writer restore from %s succeed", SMA_VID(pWriter->pSma), qTaskInfoFullName); @@ -349,7 +408,7 @@ _err: return code; } -int32_t rsmaSnapWrite(SRsmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { +int32_t rsmaSnapWrite(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; @@ -377,7 +436,7 @@ _err: return code; } -static int32_t rsmaSnapWriteQTaskInfo(SRsmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { +static int32_t rsmaSnapWriteQTaskInfo(SRSmaSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; SQTaskFWriter* qWriter = pWriter->pQTaskFWriter; diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 1687cd46a07a7b0a70107eb825fb06b6f9314441..e2cb51f586b2ec0306d15a39ed3048a321ff1179 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -20,6 +20,10 @@ #define SMA_STORAGE_MINUTES_DAY 1440 #define SMA_STORAGE_SPLIT_FACTOR 14400 // least records in tsma file +static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); +static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); +static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); + // TODO: Who is responsible for resource allocate and release? int32_t tdProcessTSmaInsert(SSma *pSma, int64_t indexUid, const char *msg) { int32_t code = TSDB_CODE_SUCCESS; @@ -59,7 +63,7 @@ int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t * * @param days unit is minute * @return int32_t */ -int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { +static int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days) { SDecoder coder = {0}; tDecoderInit(&coder, pCont, contLen); @@ -106,7 +110,7 @@ _err: * @param pMsg * @return int32_t */ -int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { +static int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { SSmaCfg *pCfg = (SSmaCfg *)pMsg; if (TD_VID(pSma->pVnode) == pCfg->dstVgId) { @@ -145,7 +149,7 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg) { * @param msg * @return int32_t */ -int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { +static int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { const SArray *pDataBlocks = (const SArray *)msg; // TODO: destroy SSDataBlocks(msg) if (!pDataBlocks) { @@ -174,7 +178,6 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { return TSDB_CODE_FAILED; } - tdRefSmaStat(pSma, pStat); pTsmaStat = SMA_STAT_TSMA(pStat); if (!pTsmaStat->pTSma) { @@ -226,9 +229,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { goto _err; } - tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_SUCCESS; _err: - tdUnRefSmaStat(pSma, pStat); return TSDB_CODE_FAILED; } diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c index d771797963a5cd9d242fea1f4d65a5634f12b5e8..a4ba0a61a57b781ff062328fe18920f3ce15dacc 100644 --- a/source/dnode/vnode/src/sma/smaUtil.c +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -305,93 +305,4 @@ int32_t tdReleaseSmaRef(int32_t rsetId, int64_t refId) { smaDebug("rsma release ref for rsetId:%" PRIi64 " refId:%d success", rsetId, refId); return TSDB_CODE_SUCCESS; -} - -static int32_t tdCloneQTaskInfo(SSma *pSma, qTaskInfo_t dstTaskInfo, qTaskInfo_t srcTaskInfo, SRSmaParam *param, - tb_uid_t suid, int8_t idx) { - SVnode *pVnode = pSma->pVnode; - char *pOutput = NULL; - int32_t len = 0; - - if ((terrno = qSerializeTaskStatus(srcTaskInfo, &pOutput, &len)) < 0) { - smaError("vgId:%d, rsma clone, table %" PRIi64 " serialize qTaskInfo failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; - } - - SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTqReader = 1, - }; - ASSERT(!dstTaskInfo); - dstTaskInfo = qCreateStreamExecTaskInfo(param->qmsg[idx], &handle); - if (!dstTaskInfo) { - terrno = TSDB_CODE_RSMA_QTASKINFO_CREATE; - goto _err; - } - - if (qDeserializeTaskStatus(dstTaskInfo, pOutput, len) < 0) { - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - goto _err; - } - - smaDebug("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " succeed", TD_VID(pVnode), suid); - - taosMemoryFreeClear(pOutput); - return TSDB_CODE_SUCCESS; -_err: - taosMemoryFreeClear(pOutput); - tdFreeQTaskInfo(dstTaskInfo, TD_VID(pVnode), idx + 1); - smaError("vgId:%d, rsma clone, restore rsma task for table:%" PRIi64 " failed since %s", TD_VID(pVnode), suid, - terrstr()); - return TSDB_CODE_FAILED; -} - -/** - * @brief Clone qTaskInfo of SRSmaInfo - * - * @param pSma - * @param pInfo - * @return int32_t - */ -int32_t tdCloneRSmaInfo(SSma *pSma, SRSmaInfo *pInfo) { - SRSmaParam *param = NULL; - if (!pInfo) { - return TSDB_CODE_SUCCESS; - } - - SMetaReader mr = {0}; - metaReaderInit(&mr, SMA_META(pSma), 0); - smaDebug("vgId:%d, rsma clone qTaskInfo for suid:%" PRIi64, SMA_VID(pSma), pInfo->suid); - if (metaGetTableEntryByUid(&mr, pInfo->suid) < 0) { - smaError("vgId:%d, rsma clone, failed to get table meta for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, - terrstr()); - goto _err; - } - ASSERT(mr.me.type == TSDB_SUPER_TABLE); - ASSERT(mr.me.uid == pInfo->suid); - if (TABLE_IS_ROLLUP(mr.me.flags)) { - param = &mr.me.stbEntry.rsmaParam; - for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { - if (!pInfo->iTaskInfo[i]) { - continue; - } - if (tdCloneQTaskInfo(pSma, pInfo->taskInfo[i], pInfo->iTaskInfo[i], param, pInfo->suid, i) < 0) { - goto _err; - } - } - smaDebug("vgId:%d, rsma clone env success for %" PRIi64, SMA_VID(pSma), pInfo->suid); - } else { - terrno = TSDB_CODE_RSMA_INVALID_SCHEMA; - goto _err; - } - - metaReaderClear(&mr); - return TSDB_CODE_SUCCESS; -_err: - metaReaderClear(&mr); - smaError("vgId:%d, rsma clone env failed for %" PRIi64 " since %s", SMA_VID(pSma), pInfo->suid, terrstr()); - return TSDB_CODE_FAILED; } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 3f42d8360ee8648ec66068c31773caea6faea41f..c8841e5e169c7cdf249894348ab949a225f58acc 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -816,7 +816,86 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msg return streamMetaAddSerializedTask(pTq->pStreamMeta, version, msg, msgLen); } -int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq, int64_t ver) { +int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { + bool failed = false; + SDecoder* pCoder = &(SDecoder){0}; + SDeleteRes* pRes = &(SDeleteRes){0}; + + pRes->uidList = taosArrayInit(0, sizeof(tb_uid_t)); + if (pRes->uidList == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + failed = true; + } + + tDecoderInit(pCoder, pReq, len); + tDecodeDeleteRes(pCoder, pRes); + tDecoderClear(pCoder); + + int32_t sz = taosArrayGetSize(pRes->uidList); + if (sz == 0 || pRes->affectedRows == 0) { + taosArrayDestroy(pRes->uidList); + return 0; + } + SSDataBlock* pDelBlock = createSpecialDataBlock(STREAM_DELETE_DATA); + blockDataEnsureCapacity(pDelBlock, sz); + pDelBlock->info.rows = sz; + pDelBlock->info.version = ver; + + for (int32_t i = 0; i < sz; i++) { + // start key column + SColumnInfoData* pStartCol = taosArrayGet(pDelBlock->pDataBlock, START_TS_COLUMN_INDEX); + colDataAppend(pStartCol, i, (const char*)&pRes->skey, false); // end key column + SColumnInfoData* pEndCol = taosArrayGet(pDelBlock->pDataBlock, END_TS_COLUMN_INDEX); + colDataAppend(pEndCol, i, (const char*)&pRes->ekey, false); + // uid column + SColumnInfoData* pUidCol = taosArrayGet(pDelBlock->pDataBlock, UID_COLUMN_INDEX); + int64_t* pUid = taosArrayGet(pRes->uidList, i); + colDataAppend(pUidCol, i, (const char*)pUid, false); + + colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, GROUPID_COLUMN_INDEX), i); + colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX), i); + colDataAppendNULL(taosArrayGet(pDelBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX), i); + } + + taosArrayDestroy(pRes->uidList); + + void* pIter = NULL; + while (1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) break; + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; + + qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + + SStreamDataBlock* pStreamBlock = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + pStreamBlock->type = STREAM_INPUT__DATA_BLOCK; + pStreamBlock->blocks = taosArrayInit(0, sizeof(SSDataBlock)); + SSDataBlock block = {0}; + assignOneDataBlock(&block, pDelBlock); + block.info.type = STREAM_DELETE_DATA; + taosArrayPush(pStreamBlock->blocks, &block); + + if (!failed) { + if (streamTaskInput(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->taskId); + continue; + } + + if (streamSchedExec(pTask) < 0) { + qError("stream task launch failed, task id %d", pTask->taskId); + continue; + } + } else { + streamTaskInputFail(pTask); + } + } + blockDataDestroy(pDelBlock); + + return 0; +} + +int32_t tqProcessSubmitReq(STQ* pTq, SSubmitReq* pReq, int64_t ver) { void* pIter = NULL; bool failed = false; SStreamDataSubmit* pSubmit = NULL; diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 8c3fa254461e2f31a71787606f6378c2f50868b5..d00907f6778d176d1eabe77f92a243a1c5dd3cbd 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -241,6 +241,20 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp continue; } } + if (pHandle->fetchMeta) { + SSubmitBlk* pBlk = pReader->pBlock; + if (pBlk->schemaLen > 0) { + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); + } + void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); + memcpy(createReq, pBlk->data, pBlk->schemaLen); + taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen); + taosArrayPush(pRsp->createTableReq, &createReq); + pRsp->createTableNum++; + } + } tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); blockDataFreeRes(&block); tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); @@ -261,34 +275,25 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SSubmitReq* pReq, STaosxRsp continue; } } - tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); - blockDataFreeRes(&block); - tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); - pRsp->blockNum++; - } -#if 1 - if (pHandle->fetchMeta && pRsp->blockNum) { - SSubmitMsgIter iter = {0}; - tInitSubmitMsgIter(pReq, &iter); - STaosxRsp* pXrsp = (STaosxRsp*)pRsp; - while (1) { - SSubmitBlk* pBlk = NULL; - if (tGetSubmitMsgNext(&iter, &pBlk) < 0) break; - if (pBlk == NULL) break; + if (pHandle->fetchMeta) { + SSubmitBlk* pBlk = pReader->pBlock; if (pBlk->schemaLen > 0) { - if (pXrsp->createTableNum == 0) { - pXrsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); - pXrsp->createTableReq = taosArrayInit(0, sizeof(void*)); + if (pRsp->createTableNum == 0) { + pRsp->createTableLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->createTableReq = taosArrayInit(0, sizeof(void*)); } void* createReq = taosMemoryCalloc(1, pBlk->schemaLen); memcpy(createReq, pBlk->data, pBlk->schemaLen); - taosArrayPush(pXrsp->createTableLen, &pBlk->schemaLen); - taosArrayPush(pXrsp->createTableReq, &createReq); - pXrsp->createTableNum++; + taosArrayPush(pRsp->createTableLen, &pBlk->schemaLen); + taosArrayPush(pRsp->createTableReq, &createReq); + pRsp->createTableNum++; } } + tqAddBlockDataToRsp(&block, (SMqDataRsp*)pRsp, taosArrayGetSize(block.pDataBlock)); + blockDataFreeRes(&block); + tqAddBlockSchemaToRsp(pExec, (SMqDataRsp*)pRsp); + pRsp->blockNum++; } -#endif } if (pRsp->blockNum == 0) { diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index ed7fa80c476fff2d6436232b0e610f0b6f61f1cd..a57e8174fee9f82fd35c425e9214e48fba91f709 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -213,20 +213,25 @@ int32_t tqPushMsgNew(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_ #endif int tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver) { - if (vnodeIsRoleLeader(pTq->pVnode) && msgType == TDMT_VND_SUBMIT) { - if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; - - void* data = taosMemoryMalloc(msgLen); - if (data == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); - return -1; + if (vnodeIsRoleLeader(pTq->pVnode)) { + if (msgType == TDMT_VND_SUBMIT) { + if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) return 0; + + void* data = taosMemoryMalloc(msgLen); + if (data == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("failed to copy data for stream since out of memory"); + return -1; + } + memcpy(data, msg, msgLen); + SSubmitReq* pReq = (SSubmitReq*)data; + pReq->version = ver; + + tqProcessSubmitReq(pTq, data, ver); + } + if (msgType == TDMT_VND_DELETE) { + tqProcessDelReq(pTq, POINTER_SHIFT(msg, sizeof(SMsgHead)), msgLen - sizeof(SMsgHead), ver); } - memcpy(data, msg, msgLen); - SSubmitReq* pReq = (SSubmitReq*)data; - pReq->version = ver; - - tqProcessStreamTrigger(pTq, data, ver); } return 0; diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 61c68775559ebcbaca1853fdadecaaa1456c170c..8da783a5bd82ccd23fe051e76396489cc6516058 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -418,31 +418,17 @@ typedef enum { } SFSLASTNEXTROWSTATES; typedef struct { - SFSLASTNEXTROWSTATES state; // [input] - STsdb *pTsdb; // [input] - SBlockIdx *pBlockIdxExp; // [input] - STSchema *pTSchema; // [input] + SFSLASTNEXTROWSTATES state; // [input] + STsdb *pTsdb; // [input] tb_uid_t suid; tb_uid_t uid; int32_t nFileSet; int32_t iFileSet; SArray *aDFileSet; SDataFReader *pDataFReader; - SArray *aBlockL; - SBlockL *pBlockL; - SBlockData *pBlockDataL; - SBlockData blockDataL; - int32_t nRow; - int32_t iRow; TSDBROW row; - /* - SArray *aBlockIdx; - SBlockIdx *pBlockIdx; - SMapData blockMap; - int32_t nBlock; - int32_t iBlock; - SBlock block; - */ + + SMergeTree mergeTree; } SFSLastNextRowIter; static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { @@ -451,22 +437,16 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { switch (state->state) { case SFSLASTNEXTROW_FS: - // state->aDFileSet = state->pTsdb->pFS->cState->aDFileSet; state->nFileSet = taosArrayGetSize(state->aDFileSet); state->iFileSet = state->nFileSet; - state->pBlockDataL = NULL; - case SFSLASTNEXTROW_FILESET: { SDFileSet *pFileSet = NULL; _next_fileset: if (--state->iFileSet >= 0) { pFileSet = (SDFileSet *)taosArrayGet(state->aDFileSet, state->iFileSet); } else { - if (state->pBlockDataL) { - tBlockDataDestroy(state->pBlockDataL, 1); - state->pBlockDataL = NULL; - } + // tMergeTreeClose(&state->mergeTree); *ppRow = NULL; return code; @@ -475,68 +455,24 @@ static int32_t getNextRowFromFSLast(void *iter, TSDBROW **ppRow) { code = tsdbDataFReaderOpen(&state->pDataFReader, state->pTsdb, pFileSet); if (code) goto _err; - if (!state->aBlockL) { - state->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - } else { - taosArrayClear(state->aBlockL); - } - - code = tsdbReadBlockL(state->pDataFReader, state->aBlockL); - if (code) goto _err; - - // SBlockL *pBlockL = (SBlockL *)taosArrayGet(state->aBlockL, state->iBlockL); - - state->pBlockL = taosArraySearch(state->aBlockL, state->pBlockIdxExp, tCmprBlockL, TD_EQ); - if (!state->pBlockL) { + tMergeTreeOpen(&state->mergeTree, 1, state->pDataFReader, state->suid, state->uid, + &(STimeWindow){.skey = TSKEY_MIN, .ekey = TSKEY_MAX}, + &(SVersionRange){.minVer = 0, .maxVer = UINT64_MAX}, NULL, NULL); + bool hasVal = tMergeTreeNext(&state->mergeTree); + if (!hasVal) { + state->state = SFSLASTNEXTROW_FILESET; + // tMergeTreeClose(&state->mergeTree); goto _next_fileset; } - - int64_t suid = state->pBlockL->suid; - int64_t uid = state->pBlockL->maxUid; - - if (!state->pBlockDataL) { - state->pBlockDataL = &state->blockDataL; - - tBlockDataCreate(state->pBlockDataL); - } - code = tBlockDataInit(state->pBlockDataL, suid, suid ? 0 : uid, state->pTSchema); - if (code) goto _err; - } - case SFSLASTNEXTROW_BLOCKDATA: - code = tsdbReadLastBlock(state->pDataFReader, state->pBlockL, state->pBlockDataL); - if (code) goto _err; - - state->nRow = state->blockDataL.nRow; - state->iRow = state->nRow - 1; - - if (!state->pBlockDataL->uid) { - while (state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { - --state->iRow; - } - } - state->state = SFSLASTNEXTROW_BLOCKROW; + } case SFSLASTNEXTROW_BLOCKROW: - if (state->pBlockDataL->uid) { - if (state->iRow >= 0) { - state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); - *ppRow = &state->row; - - if (--state->iRow < 0) { - state->state = SFSLASTNEXTROW_FILESET; - } - } - } else { - if (state->iRow >= 0 && state->pBlockIdxExp->uid == state->pBlockDataL->aUid[state->iRow]) { - state->row = tsdbRowFromBlockData(state->pBlockDataL, state->iRow); - *ppRow = &state->row; - - if (--state->iRow < 0 || state->pBlockIdxExp->uid != state->pBlockDataL->aUid[state->iRow]) { - state->state = SFSLASTNEXTROW_FILESET; - } - } + state->row = tMergeTreeGetRow(&state->mergeTree); + *ppRow = &state->row; + bool hasVal = tMergeTreeNext(&state->mergeTree); + if (!hasVal) { + state->state = SFSLASTNEXTROW_FILESET; } - return code; default: ASSERT(0); @@ -548,15 +484,6 @@ _err: tsdbDataFReaderClose(&state->pDataFReader); state->pDataFReader = NULL; } - if (state->aBlockL) { - taosArrayDestroy(state->aBlockL); - state->aBlockL = NULL; - } - if (state->pBlockDataL) { - tBlockDataDestroy(state->pBlockDataL, 1); - state->pBlockDataL = NULL; - } - *ppRow = NULL; return code; @@ -574,14 +501,6 @@ int32_t clearNextRowFromFSLast(void *iter) { tsdbDataFReaderClose(&state->pDataFReader); state->pDataFReader = NULL; } - if (state->aBlockL) { - taosArrayDestroy(state->aBlockL); - state->aBlockL = NULL; - } - if (state->pBlockDataL) { - tBlockDataDestroy(state->pBlockDataL, 1); - state->pBlockDataL = NULL; - } return code; } @@ -609,7 +528,7 @@ typedef struct SFSNextRowIter { SMapData blockMap; int32_t nBlock; int32_t iBlock; - SBlock block; + SDataBlk block; SBlockData blockData; SBlockData *pBlockData; int32_t nRow; @@ -670,7 +589,7 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } tMapDataReset(&state->blockMap); - code = tsdbReadBlock(state->pDataFReader, state->pBlockIdx, &state->blockMap); + code = tsdbReadDataBlk(state->pDataFReader, state->pBlockIdx, &state->blockMap); if (code) goto _err; state->nBlock = state->blockMap.nItem; @@ -684,13 +603,13 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow) { } case SFSNEXTROW_BLOCKDATA: if (state->iBlock >= 0) { - SBlock block = {0}; + SDataBlk block = {0}; - tBlockReset(&block); + tDataBlkReset(&block); // tBlockDataReset(&state->blockData); tBlockDataReset(state->pBlockData); - tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetBlock); + tMapDataGetItemByIdx(&state->blockMap, state->iBlock, &block, tGetDataBlk); /* code = tsdbReadBlockData(state->pDataFReader, &state->blockIdx, &block, &state->blockData, NULL, NULL); */ tBlockDataReset(state->pBlockData); code = tBlockDataInit(state->pBlockData, state->suid, state->uid, state->pTSchema); @@ -878,7 +797,7 @@ static bool tsdbKeyDeleted(TSDBKEY *key, SArray *pSkyline, int64_t *iSkyline) { if (key->ts > pItemBack->ts) { return false; } else if (key->ts >= pItemFront->ts && key->ts <= pItemBack->ts) { - if ((key->version <= pItemFront->version || key->ts == pItemBack->ts && key->version <= pItemBack->version)) { + if (key->version <= pItemFront->version || (key->ts == pItemBack->ts && key->version <= pItemBack->version)) { return true; } else { return false; @@ -972,8 +891,6 @@ static int32_t nextRowIterOpen(CacheNextRowIter *pIter, tb_uid_t uid, STsdb *pTs pIter->fsLastState.state = (SFSLASTNEXTROWSTATES)SFSNEXTROW_FS; pIter->fsLastState.pTsdb = pTsdb; pIter->fsLastState.aDFileSet = pIter->pReadSnap->fs.aDFileSet; - pIter->fsLastState.pBlockIdxExp = &pIter->idx; - pIter->fsLastState.pTSchema = pTSchema; pIter->fsLastState.suid = suid; pIter->fsLastState.uid = uid; @@ -1372,25 +1289,33 @@ int32_t tsdbCacheGetLastH(SLRUCache *pCache, tb_uid_t uid, STsdb *pTsdb, LRUHand // getTableCacheKeyS(uid, "l", key, &keyLen); getTableCacheKey(uid, 1, key, &keyLen); LRUHandle *h = taosLRUCacheLookup(pCache, key, keyLen); - if (h) { - } else { - SArray *pLastArray = NULL; - code = mergeLast(uid, pTsdb, &pLastArray); - // if table's empty or error, return code of -1 - // if (code < 0 || pRow == NULL) { - if (code < 0 || pLastArray == NULL) { - *handle = NULL; - return 0; - } - - _taos_lru_deleter_t deleter = deleteTableCacheLast; - LRUStatus status = - taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, TAOS_LRU_PRIORITY_LOW); - if (status != TAOS_LRU_STATUS_OK) { - code = -1; - } + if (!h) { + taosThreadMutexLock(&pTsdb->lruMutex); h = taosLRUCacheLookup(pCache, key, keyLen); + if (!h) { + SArray *pLastArray = NULL; + code = mergeLast(uid, pTsdb, &pLastArray); + // if table's empty or error, return code of -1 + // if (code < 0 || pRow == NULL) { + if (code < 0 || pLastArray == NULL) { + *handle = NULL; + return 0; + } + + _taos_lru_deleter_t deleter = deleteTableCacheLast; + LRUStatus status = taosLRUCacheInsert(pCache, key, keyLen, pLastArray, pLastArray->capacity, deleter, NULL, + TAOS_LRU_PRIORITY_LOW); + if (status != TAOS_LRU_STATUS_OK) { + code = -1; + } + + taosThreadMutexUnlock(&pTsdb->lruMutex); + + h = taosLRUCacheLookup(pCache, key, keyLen); + } else { + taosThreadMutexUnlock(&pTsdb->lruMutex); + } } *handle = h; @@ -1411,3 +1336,5 @@ void tsdbCacheSetCapacity(SVnode *pVnode, size_t capacity) { } size_t tsdbCacheGetCapacity(SVnode *pVnode) { return taosLRUCacheGetCapacity(pVnode->pTsdb->lruCache); } + +size_t tsdbCacheGetUsage(SVnode *pVnode) { return taosLRUCacheGetUsage(pVnode->pTsdb->lruCache); } diff --git a/source/dnode/vnode/src/tsdb/tsdbCommit.c b/source/dnode/vnode/src/tsdb/tsdbCommit.c index 04a6de8472ade43b3e7e2b420f8e2d0c0656a2c8..a619b9f2e4f827d72f2aad5fd752ae002ac2fc74 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCommit.c +++ b/source/dnode/vnode/src/tsdb/tsdbCommit.c @@ -14,17 +14,27 @@ */ #include "tsdb.h" -typedef struct { - int64_t suid; - int64_t uid; - STSchema *pTSchema; -} SSkmInfo; + +typedef enum { MEMORY_DATA_ITER = 0, STT_DATA_ITER } EDataIterT; typedef struct { - int64_t suid; - int64_t uid; - TSDBROW row; -} SRowInfo; + SRBTreeNode n; + SRowInfo r; + EDataIterT type; + union { + struct { + int32_t iTbDataP; + STbDataIter iter; + }; // memory data iter + struct { + int32_t iStt; + SArray *aSttBlk; + int32_t iSttBlk; + SBlockData bData; + int32_t iRow; + }; // stt file data iter + }; +} SDataIter; typedef struct { STsdb *pTsdb; @@ -35,8 +45,9 @@ typedef struct { int32_t minRow; int32_t maxRow; int8_t cmprAlg; - SArray *aTbDataP; - STsdbFS fs; + int8_t sttTrigger; + SArray *aTbDataP; // memory + STsdbFS fs; // disk // -------------- TSKEY nextKey; // reset by each table commit int32_t commitFid; @@ -45,25 +56,24 @@ typedef struct { // commit file data struct { SDataFReader *pReader; - // data - SArray *aBlockIdx; // SArray - int32_t iBlockIdx; - SBlockIdx *pBlockIdx; - SMapData mBlock; // SMapData - SBlockData bData; - // last - SArray *aBlockL; // SArray - int32_t iBlockL; - SBlockData bDatal; - int32_t iRow; - SRowInfo *pRowInfo; - SRowInfo rowInfo; + SArray *aBlockIdx; // SArray + int32_t iBlockIdx; + SBlockIdx *pBlockIdx; + SMapData mBlock; // SMapData + SBlockData bData; } dReader; + struct { + SDataIter *pIter; + SRBTree rbt; + SDataIter dataIter; + SDataIter aDataIter[TSDB_MAX_STT_TRIGGER]; + int8_t toLastOnly; + }; struct { SDataFWriter *pWriter; SArray *aBlockIdx; // SArray - SArray *aBlockL; // SArray - SMapData mBlock; // SMapData + SArray *aSttBlk; // SArray + SMapData mBlock; // SMapData SBlockData bData; SBlockData bDatal; } dWriter; @@ -82,6 +92,26 @@ static int32_t tsdbCommitData(SCommitter *pCommitter); static int32_t tsdbCommitDel(SCommitter *pCommitter); static int32_t tsdbCommitCache(SCommitter *pCommitter); static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno); +static int32_t tsdbNextCommitRow(SCommitter *pCommitter); + +int32_t tRowInfoCmprFn(const void *p1, const void *p2) { + SRowInfo *pInfo1 = (SRowInfo *)p1; + SRowInfo *pInfo2 = (SRowInfo *)p2; + + if (pInfo1->suid < pInfo2->suid) { + return -1; + } else if (pInfo1->suid > pInfo2->suid) { + return 1; + } + + if (pInfo1->uid < pInfo2->uid) { + return -1; + } else if (pInfo1->uid > pInfo2->uid) { + return 1; + } + + return tsdbRowCmprFn(&pInfo1->row, &pInfo2->row); +} int32_t tsdbBegin(STsdb *pTsdb) { int32_t code = 0; @@ -290,19 +320,22 @@ _err: return code; } -static int32_t tsdbCommitterUpdateTableSchema(SCommitter *pCommitter, int64_t suid, int64_t uid) { +int32_t tsdbUpdateTableSchema(SMeta *pMeta, int64_t suid, int64_t uid, SSkmInfo *pSkmInfo) { int32_t code = 0; if (suid) { - if (pCommitter->skmTable.suid == suid) goto _exit; + if (pSkmInfo->suid == suid) { + pSkmInfo->uid = uid; + goto _exit; + } } else { - if (pCommitter->skmTable.uid == uid) goto _exit; + if (pSkmInfo->uid == uid) goto _exit; } - pCommitter->skmTable.suid = suid; - pCommitter->skmTable.uid = uid; - tTSchemaDestroy(pCommitter->skmTable.pTSchema); - code = metaGetTbTSchemaEx(pCommitter->pTsdb->pVnode->pMeta, suid, uid, -1, &pCommitter->skmTable.pTSchema); + pSkmInfo->suid = suid; + pSkmInfo->uid = uid; + tTSchemaDestroy(pSkmInfo->pTSchema); + code = metaGetTbTSchemaEx(pMeta, suid, uid, -1, &pSkmInfo->pTSchema); if (code) goto _exit; _exit: @@ -334,54 +367,6 @@ _exit: return code; } -static int32_t tsdbCommitterNextLastRow(SCommitter *pCommitter) { - int32_t code = 0; - - ASSERT(pCommitter->dReader.pReader); - ASSERT(pCommitter->dReader.pRowInfo); - - SBlockData *pBlockDatal = &pCommitter->dReader.bDatal; - pCommitter->dReader.iRow++; - if (pCommitter->dReader.iRow < pBlockDatal->nRow) { - if (pBlockDatal->uid) { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; - } else { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[pCommitter->dReader.iRow]; - } - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.iBlockL++; - if (pCommitter->dReader.iBlockL < taosArrayGetSize(pCommitter->dReader.aBlockL)) { - SBlockL *pBlockL = (SBlockL *)taosArrayGet(pCommitter->dReader.aBlockL, pCommitter->dReader.iBlockL); - int64_t suid = pBlockL->suid; - int64_t uid = pBlockL->maxUid; - - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); - if (code) goto _exit; - - code = tBlockDataInit(pBlockDatal, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); - if (code) goto _exit; - - code = tsdbReadLastBlock(pCommitter->dReader.pReader, pBlockL, pBlockDatal); - if (code) goto _exit; - - pCommitter->dReader.iRow = 0; - pCommitter->dReader.pRowInfo->suid = pBlockDatal->suid; - if (pBlockDatal->uid) { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->uid; - } else { - pCommitter->dReader.pRowInfo->uid = pBlockDatal->aUid[0]; - } - pCommitter->dReader.pRowInfo->row = tsdbRowFromBlockData(pBlockDatal, pCommitter->dReader.iRow); - } else { - pCommitter->dReader.pRowInfo = NULL; - } - } - -_exit: - return code; -} - static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { int32_t code = 0; @@ -392,7 +377,7 @@ static int32_t tsdbCommitterNextTableData(SCommitter *pCommitter) { pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); + code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _exit; ASSERT(pCommitter->dReader.mBlock.nItem > 0); @@ -404,6 +389,85 @@ _exit: return code; } +static int32_t tsdbOpenCommitIter(SCommitter *pCommitter) { + int32_t code = 0; + + pCommitter->pIter = NULL; + tRBTreeCreate(&pCommitter->rbt, tRowInfoCmprFn); + + // memory + TSDBKEY tKey = {.ts = pCommitter->minKey, .version = VERSION_MIN}; + SDataIter *pIter = &pCommitter->dataIter; + pIter->type = MEMORY_DATA_ITER; + pIter->iTbDataP = 0; + for (; pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP); pIter->iTbDataP++) { + STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); + tsdbTbDataIterOpen(pTbData, &tKey, 0, &pIter->iter); + TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); + pRow = NULL; + } + + if (pRow == NULL) continue; + + pIter->r.suid = pTbData->suid; + pIter->r.uid = pTbData->uid; + pIter->r.row = *pRow; + break; + } + ASSERT(pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)); + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); + + // disk + pCommitter->toLastOnly = 0; + SDataFReader *pReader = pCommitter->dReader.pReader; + if (pReader) { + if (pReader->pSet->nSttF >= pCommitter->sttTrigger) { + int8_t iIter = 0; + for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { + pIter = &pCommitter->aDataIter[iIter]; + pIter->type = STT_DATA_ITER; + pIter->iStt = iStt; + + code = tsdbReadSttBlk(pCommitter->dReader.pReader, iStt, pIter->aSttBlk); + if (code) goto _err; + + if (taosArrayGetSize(pIter->aSttBlk) == 0) continue; + + pIter->iSttBlk = 0; + SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, 0); + code = tsdbReadSttBlock(pCommitter->dReader.pReader, iStt, pSttBlk, &pIter->bData); + if (code) goto _err; + + pIter->iRow = 0; + pIter->r.suid = pIter->bData.suid; + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); + + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pIter); + iIter++; + } + } else { + for (int32_t iStt = 0; iStt < pReader->pSet->nSttF; iStt++) { + SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; + if (pSttFile->size > pSttFile->offset) { + pCommitter->toLastOnly = 1; + break; + } + } + } + } + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + return code; + +_err: + return code; +} + static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { int32_t code = 0; STsdb *pTsdb = pCommitter->pTsdb; @@ -416,8 +480,8 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { pCommitter->nextKey = TSKEY_MAX; // Reader - pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &(SDFileSet){.fid = pCommitter->commitFid}, - tDFileSetCmprFn, TD_EQ); + SDFileSet tDFileSet = {.fid = pCommitter->commitFid}; + pRSet = (SDFileSet *)taosArraySearch(pCommitter->fs.aDFileSet, &tDFileSet, tDFileSetCmprFn, TD_EQ); if (pRSet) { code = tsdbDataFReaderOpen(&pCommitter->dReader.pReader, pTsdb, pRSet); if (code) goto _err; @@ -427,68 +491,58 @@ static int32_t tsdbCommitFileDataStart(SCommitter *pCommitter) { if (code) goto _err; pCommitter->dReader.iBlockIdx = 0; - if (pCommitter->dReader.iBlockIdx < taosArrayGetSize(pCommitter->dReader.aBlockIdx)) { - pCommitter->dReader.pBlockIdx = - (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, pCommitter->dReader.iBlockIdx); - - code = tsdbReadBlock(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); + if (taosArrayGetSize(pCommitter->dReader.aBlockIdx) > 0) { + pCommitter->dReader.pBlockIdx = (SBlockIdx *)taosArrayGet(pCommitter->dReader.aBlockIdx, 0); + code = tsdbReadDataBlk(pCommitter->dReader.pReader, pCommitter->dReader.pBlockIdx, &pCommitter->dReader.mBlock); if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; } tBlockDataReset(&pCommitter->dReader.bData); - - // last - code = tsdbReadBlockL(pCommitter->dReader.pReader, pCommitter->dReader.aBlockL); - if (code) goto _err; - - pCommitter->dReader.iBlockL = -1; - pCommitter->dReader.iRow = -1; - pCommitter->dReader.pRowInfo = &pCommitter->dReader.rowInfo; - tBlockDataReset(&pCommitter->dReader.bDatal); - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; } else { pCommitter->dReader.pBlockIdx = NULL; - pCommitter->dReader.pRowInfo = NULL; } // Writer - SHeadFile fHead; - SDataFile fData; - SLastFile fLast; - SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; + SHeadFile fHead = {.commitID = pCommitter->commitID}; + SDataFile fData = {.commitID = pCommitter->commitID}; + SSmaFile fSma = {.commitID = pCommitter->commitID}; + SSttFile fStt = {.commitID = pCommitter->commitID}; + SDFileSet wSet = {.fid = pCommitter->commitFid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; if (pRSet) { - wSet.diskId = pRSet->diskId; - wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; + ASSERT(pRSet->nSttF <= pCommitter->sttTrigger); fData = *pRSet->pDataF; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; fSma = *pRSet->pSmaF; + wSet.diskId = pRSet->diskId; + if (pRSet->nSttF < pCommitter->sttTrigger) { + for (int32_t iStt = 0; iStt < pRSet->nSttF; iStt++) { + wSet.aSttF[iStt] = pRSet->aSttF[iStt]; + } + wSet.nSttF = pRSet->nSttF + 1; + } else { + wSet.nSttF = 1; + } } else { SDiskID did = {0}; - tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); - tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); - wSet.diskId = did; - wSet.fid = pCommitter->commitFid; - fHead = (SHeadFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - fData = (SDataFile){.commitID = pCommitter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pCommitter->commitID, .size = 0, .offset = 0}; - fSma = (SSmaFile){.commitID = pCommitter->commitID, .size = 0}; + wSet.nSttF = 1; } + wSet.aSttF[wSet.nSttF - 1] = &fStt; code = tsdbDataFWriterOpen(&pCommitter->dWriter.pWriter, pTsdb, &wSet); if (code) goto _err; taosArrayClear(pCommitter->dWriter.aBlockIdx); - taosArrayClear(pCommitter->dWriter.aBlockL); + taosArrayClear(pCommitter->dWriter.aSttBlk); tMapDataReset(&pCommitter->dWriter.mBlock); tBlockDataReset(&pCommitter->dWriter.bData); tBlockDataReset(&pCommitter->dWriter.bDatal); + // open iter + code = tsdbOpenCommitIter(pCommitter); + if (code) goto _err; + _exit: return code; @@ -497,50 +551,45 @@ _err: return code; } -static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { - int32_t code = 0; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - SBlock block; +int32_t tsdbWriteDataBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SMapData *mDataBlk, int8_t cmprAlg) { + int32_t code = 0; - ASSERT(pBlockData->nRow > 0); + if (pBlockData->nRow == 0) return code; - if (pBlock) { - block = *pBlock; // as a subblock - } else { - tBlockReset(&block); // as a new block - } + SDataBlk dataBlk; + tDataBlkReset(&dataBlk); // info - block.nRow += pBlockData->nRow; + dataBlk.nRow += pBlockData->nRow; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; if (iRow == 0) { - if (tsdbKeyCmprFn(&block.minKey, &key) > 0) { - block.minKey = key; + if (tsdbKeyCmprFn(&dataBlk.minKey, &key) > 0) { + dataBlk.minKey = key; } } else { if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - block.hasDup = 1; + dataBlk.hasDup = 1; } } - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&block.maxKey, &key) < 0) { - block.maxKey = key; + if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&dataBlk.maxKey, &key) < 0) { + dataBlk.maxKey = key; } - block.minVer = TMIN(block.minVer, key.version); - block.maxVer = TMAX(block.maxVer, key.version); + dataBlk.minVer = TMIN(dataBlk.minVer, key.version); + dataBlk.maxVer = TMAX(dataBlk.maxVer, key.version); } // write - block.nSubBlock++; - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &block.aSubBlock[block.nSubBlock - 1], - ((block.nSubBlock == 1) && !block.hasDup) ? &block.smaInfo : NULL, pCommitter->cmprAlg, 0); + dataBlk.nSubBlock++; + code = tsdbWriteBlockData(pWriter, pBlockData, &dataBlk.aSubBlock[dataBlk.nSubBlock - 1], + ((dataBlk.nSubBlock == 1) && !dataBlk.hasDup) ? &dataBlk.smaInfo : NULL, cmprAlg, 0); if (code) goto _err; - // put SBlock - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, &block, tPutBlock); + // put SDataBlk + code = tMapDataPutItem(mDataBlk, &dataBlk, tPutDataBlk); if (code) goto _err; // clear @@ -549,39 +598,38 @@ static int32_t tsdbCommitDataBlock(SCommitter *pCommitter, SBlock *pBlock) { return code; _err: - tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb commit data block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { - int32_t code = 0; - SBlockL blockL; - SBlockData *pBlockData = &pCommitter->dWriter.bDatal; +int32_t tsdbWriteSttBlock(SDataFWriter *pWriter, SBlockData *pBlockData, SArray *aSttBlk, int8_t cmprAlg) { + int32_t code = 0; + SSttBlk sstBlk; - ASSERT(pBlockData->nRow > 0); + if (pBlockData->nRow == 0) return code; // info - blockL.suid = pBlockData->suid; - blockL.nRow = pBlockData->nRow; - blockL.minKey = TSKEY_MAX; - blockL.maxKey = TSKEY_MIN; - blockL.minVer = VERSION_MAX; - blockL.maxVer = VERSION_MIN; + sstBlk.suid = pBlockData->suid; + sstBlk.nRow = pBlockData->nRow; + sstBlk.minKey = TSKEY_MAX; + sstBlk.maxKey = TSKEY_MIN; + sstBlk.minVer = VERSION_MAX; + sstBlk.maxVer = VERSION_MIN; for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - blockL.minKey = TMIN(blockL.minKey, pBlockData->aTSKEY[iRow]); - blockL.maxKey = TMAX(blockL.maxKey, pBlockData->aTSKEY[iRow]); - blockL.minVer = TMIN(blockL.minVer, pBlockData->aVersion[iRow]); - blockL.maxVer = TMAX(blockL.maxVer, pBlockData->aVersion[iRow]); + sstBlk.minKey = TMIN(sstBlk.minKey, pBlockData->aTSKEY[iRow]); + sstBlk.maxKey = TMAX(sstBlk.maxKey, pBlockData->aTSKEY[iRow]); + sstBlk.minVer = TMIN(sstBlk.minVer, pBlockData->aVersion[iRow]); + sstBlk.maxVer = TMAX(sstBlk.maxVer, pBlockData->aVersion[iRow]); } - blockL.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; - blockL.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; + sstBlk.minUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[0]; + sstBlk.maxUid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[pBlockData->nRow - 1]; // write - code = tsdbWriteBlockData(pCommitter->dWriter.pWriter, pBlockData, &blockL.bInfo, NULL, pCommitter->cmprAlg, 1); + code = tsdbWriteBlockData(pWriter, pBlockData, &sstBlk.bInfo, NULL, cmprAlg, 1); if (code) goto _err; - // push SBlockL - if (taosArrayPush(pCommitter->dWriter.aBlockL, &blockL) == NULL) { + // push SSttBlk + if (taosArrayPush(aSttBlk, &sstBlk) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } @@ -592,773 +640,197 @@ static int32_t tsdbCommitLastBlock(SCommitter *pCommitter) { return code; _err: - tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb commit last block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbMergeCommitData(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockDataR = &pCommitter->dReader.bData; - SBlockData *pBlockDataW = &pCommitter->dWriter.bData; +static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { + int32_t code = 0; - code = tsdbReadDataBlock(pCommitter->dReader.pReader, pBlock, pBlockDataR); + // write aBlockIdx + code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); if (code) goto _err; - tBlockDataClear(pBlockDataW); - int32_t iRow = 0; - TSDBROW row; - TSDBROW *pRow1 = tsdbTbDataIterGet(pIter); - TSDBROW *pRow2 = &row; - *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); - while (pRow1 && pRow2) { - int32_t c = tsdbRowCmprFn(pRow1, pRow2); - - if (c < 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow1)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockDataW, pRow1, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - // next - tsdbTbDataIterNext(pIter); - pRow1 = tsdbTbDataIterGet(pIter); - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); - if (code) goto _err; - - iRow++; - if (iRow < pBlockDataR->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); - } else { - pRow2 = NULL; - } - } else { - ASSERT(0); - } - - // check - if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - } + // write aSttBlk + code = tsdbWriteSttBlk(pCommitter->dWriter.pWriter, pCommitter->dWriter.aSttBlk); + if (code) goto _err; - while (pRow2) { - code = tBlockDataAppendRow(pBlockDataW, pRow2, NULL, pTbData->uid); - if (code) goto _err; + // update file header + code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); + if (code) goto _err; - iRow++; - if (iRow < pBlockDataR->nRow) { - *pRow2 = tsdbRowFromBlockData(pBlockDataR, iRow); - } else { - pRow2 = NULL; - } + // upsert SDFileSet + code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); + if (code) goto _err; - // check - if (pBlockDataW->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } - } + // close and sync + code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); + if (code) goto _err; - // check - if (pBlockDataW->nRow > 0) { - code = tsdbCommitDataBlock(pCommitter, NULL); + if (pCommitter->dReader.pReader) { + code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); if (code) goto _err; } +_exit: return code; _err: - tsdbError("vgId:%d, tsdb merge commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbCommitTableMemData(SCommitter *pCommitter, STbDataIter *pIter, TSDBKEY toKey) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockData = &pCommitter->dWriter.bData; - - tBlockDataClear(pBlockData); - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - while (true) { - if (pRow == NULL) { - if (pBlockData->nRow > 0) { - goto _write_block; - } else { - break; - } - } - - // update schema - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; +static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { + int32_t code = 0; - // append - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); + while (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) < 0) { + SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; + code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); if (code) goto _err; - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow) { - TSDBKEY rowKey = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&rowKey, &toKey) >= 0) { - pRow = NULL; - } + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; } - if (pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - _write_block: - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - } + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; } return code; _err: - tsdbError("vgId:%d, tsdb commit table mem data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbGetNumOfRowsLessThan(STbDataIter *pIter, TSDBKEY key) { - int32_t nRow = 0; - - STbDataIter iter = *pIter; - while (true) { - TSDBROW *pRow = tsdbTbDataIterGet(&iter); - if (pRow == NULL) break; +static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter); +static int32_t tsdbCommitFileData(SCommitter *pCommitter) { + int32_t code = 0; + STsdb *pTsdb = pCommitter->pTsdb; + SMemTable *pMemTable = pTsdb->imem; - int32_t c = tsdbKeyCmprFn(&TSDBROW_KEY(pRow), &key); - if (c < 0) { - nRow++; - tsdbTbDataIterNext(&iter); - } else if (c > 0) { - break; - } else { - ASSERT(0); - } - } + // commit file data start + code = tsdbCommitFileDataStart(pCommitter); + if (code) goto _err; - return nRow; -} + // impl + code = tsdbCommitFileDataImpl(pCommitter); + if (code) goto _err; -static int32_t tsdbMergeAsSubBlock(SCommitter *pCommitter, STbDataIter *pIter, SBlock *pBlock) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - SBlockData *pBlockData = &pCommitter->dWriter.bData; + // commit file data end + code = tsdbCommitFileDataEnd(pCommitter); + if (code) goto _err; - tBlockDataClear(pBlockData); - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - while (true) { - if (pRow == NULL) break; + return code; - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; +_err: + tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbDataFReaderClose(&pCommitter->dReader.pReader); + tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); + return code; +} - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; +// ---------------------------------------------------------------------------- +static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { + int32_t code = 0; - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow) { - TSDBKEY rowKey = TSDBROW_KEY(pRow); - if (tsdbKeyCmprFn(&rowKey, &pBlock->maxKey) > 0) { - pRow = NULL; - } - } - } + memset(pCommitter, 0, sizeof(*pCommitter)); + ASSERT(pTsdb->mem && pTsdb->imem == NULL); - ASSERT(pBlockData->nRow > 0 && pBlock->nRow + pBlockData->nRow <= pCommitter->maxRow); + taosThreadRwlockWrlock(&pTsdb->rwLock); + pTsdb->imem = pTsdb->mem; + pTsdb->mem = NULL; + taosThreadRwlockUnlock(&pTsdb->rwLock); - code = tsdbCommitDataBlock(pCommitter, pBlock); + pCommitter->pTsdb = pTsdb; + pCommitter->commitID = pTsdb->pVnode->state.commitID; + pCommitter->minutes = pTsdb->keepCfg.days; + pCommitter->precision = pTsdb->keepCfg.precision; + pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; + pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; + pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; + pCommitter->sttTrigger = pTsdb->pVnode->config.sttTrigger; + pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); + if (pCommitter->aTbDataP == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + code = tsdbFSCopy(pTsdb, &pCommitter->fs); if (code) goto _err; return code; _err: - tsdbError("vgId:%d, tsdb merge as subblock failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbMergeCommitLast(SCommitter *pCommitter, STbDataIter *pIter) { - int32_t code = 0; - STbData *pTbData = pIter->pTbData; - int32_t nRow = tsdbGetNumOfRowsLessThan(pIter, (TSDBKEY){.ts = pCommitter->maxKey + 1, .version = VERSION_MIN}); +static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { + int32_t code = 0; - if (pCommitter->dReader.pRowInfo && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pRowInfo) == 0) { - if (pCommitter->dReader.pRowInfo->suid) { // super table - for (int32_t iRow = pCommitter->dReader.iRow; iRow < pCommitter->dReader.bDatal.nRow; iRow++) { - if (pTbData->uid != pCommitter->dReader.bDatal.aUid[iRow]) break; - nRow++; - } - } else { // normal table - ASSERT(pCommitter->dReader.iRow == 0); - nRow += pCommitter->dReader.bDatal.nRow; - } + // reader + pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pCommitter->dReader.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - if (nRow == 0) goto _exit; + code = tBlockDataCreate(&pCommitter->dReader.bData); + if (code) goto _exit; + + // merger + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { + SDataIter *pIter = &pCommitter->aDataIter[iStt]; + pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pIter->aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - TSDBROW *pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; + code = tBlockDataCreate(&pIter->bData); + if (code) goto _exit; } - SRowInfo *pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && pRowInfo->uid != pTbData->uid) { - pRowInfo = NULL; + // writer + pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pCommitter->dWriter.aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } - while (nRow) { - SBlockData *pBlockData; - int8_t toData; + pCommitter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pCommitter->dWriter.aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } - if (nRow < pCommitter->minRow) { // to .last - toData = 0; - pBlockData = &pCommitter->dWriter.bDatal; + code = tBlockDataCreate(&pCommitter->dWriter.bData); + if (code) goto _exit; - // commit and reset block data schema if need - // QUESTION: Is there a case that pBlockData->nRow == 0 but need to change schema ? - if (pBlockData->suid || pBlockData->uid) { - if (pBlockData->suid != pTbData->suid || pBlockData->suid == 0) { - if (pBlockData->nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } + code = tBlockDataCreate(&pCommitter->dWriter.bDatal); + if (code) goto _exit; - tBlockDataReset(pBlockData); - } - } +_exit: + return code; +} - // set block data schema if need - if (pBlockData->suid == 0 && pBlockData->uid == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); - if (code) goto _err; +static void tsdbCommitDataEnd(SCommitter *pCommitter) { + // reader + taosArrayDestroy(pCommitter->dReader.aBlockIdx); + tMapDataClear(&pCommitter->dReader.mBlock); + tBlockDataDestroy(&pCommitter->dReader.bData, 1); - code = - tBlockDataInit(pBlockData, pTbData->suid, pTbData->suid ? 0 : pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - } - - if (pBlockData->nRow + nRow > pCommitter->maxRow) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - } else { // to .data - toData = 1; - pBlockData = &pCommitter->dWriter.bData; - ASSERT(pBlockData->nRow == 0); - } - - while (pRow && pRowInfo) { - int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); - if (c < 0) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - } else if (c > 0) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); - if (code) goto _err; - - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; - - pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && pRowInfo->uid != pTbData->uid) { - pRowInfo = NULL; - } - } else { - ASSERT(0); - } - - nRow--; - if (toData) { - if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - goto _outer_break; - } - } - } - - while (pRow) { - code = tsdbCommitterUpdateRowSchema(pCommitter, pTbData->suid, pTbData->uid, TSDBROW_SVERSION(pRow)); - if (code) goto _err; - - code = tBlockDataAppendRow(pBlockData, pRow, pCommitter->skmRow.pTSchema, pTbData->uid); - if (code) goto _err; - - tsdbTbDataIterNext(pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - nRow--; - if (toData) { - if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - goto _outer_break; - } - } - } - - while (pRowInfo) { - code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pTbData->uid); - if (code) goto _err; - - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; - - pRowInfo = pCommitter->dReader.pRowInfo; - if (pRowInfo && pRowInfo->uid != pTbData->uid) { - pRowInfo = NULL; - } - - nRow--; - if (toData) { - if (nRow == 0 || pBlockData->nRow >= pCommitter->maxRow * 4 / 5) { - code = tsdbCommitDataBlock(pCommitter, NULL); - if (code) goto _err; - goto _outer_break; - } - } - } - - _outer_break: - ASSERT(nRow >= 0); - } - -_exit: - return code; - -_err: - tsdbError("vgId:%d tsdb merge commit last failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbCommitTableData(SCommitter *pCommitter, STbData *pTbData) { - int32_t code = 0; - - ASSERT(pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) >= 0); - ASSERT(pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, pTbData) >= 0); - - // merge commit table data - STbDataIter iter = {0}; - STbDataIter *pIter = &iter; - TSDBROW *pRow; - - tsdbTbDataIterOpen(pTbData, &(TSDBKEY){.ts = pCommitter->minKey, .version = VERSION_MIN}, 0, pIter); - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - - if (pRow == NULL) { - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, pTbData) == 0) { - SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - - goto _exit; - } - - int32_t iBlock = 0; - SBlock block; - SBlock *pBlock = █ - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - - code = tsdbCommitterUpdateTableSchema(pCommitter, pTbData->suid, pTbData->uid); - if (code) goto _err; - - tMapDataReset(&pCommitter->dWriter.mBlock); - code = tBlockDataInit(&pCommitter->dReader.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - code = tBlockDataInit(&pCommitter->dWriter.bData, pTbData->suid, pTbData->uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - - // .data merge - while (pBlock && pRow) { - int32_t c = tBlockCmprFn(pBlock, &(SBlock){.minKey = TSDBROW_KEY(pRow), .maxKey = TSDBROW_KEY(pRow)}); - if (c < 0) { // disk - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - - // next - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } else if (c > 0) { // memory - code = tsdbCommitTableMemData(pCommitter, pIter, pBlock->minKey); - if (code) goto _err; - - // next - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - } else { // merge - int32_t nOvlp = tsdbGetNumOfRowsLessThan(pIter, pBlock->maxKey); - - ASSERT(nOvlp > 0); - - if (pBlock->nRow + nOvlp <= pCommitter->maxRow && pBlock->nSubBlock < TSDB_MAX_SUBBLOCKS) { - code = tsdbMergeAsSubBlock(pCommitter, pIter, pBlock); - if (code) goto _err; - } else { - code = tsdbMergeCommitData(pCommitter, pIter, pBlock); - if (code) goto _err; - } - - // next - pRow = tsdbTbDataIterGet(pIter); - if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { - pRow = NULL; - } - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } - } - - while (pBlock) { - code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pBlock, tPutBlock); - if (code) goto _err; - - // next - iBlock++; - if (iBlock < pCommitter->dReader.mBlock.nItem) { - tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pBlock, tGetBlock); - } else { - pBlock = NULL; - } - } - - // .data append and .last merge - code = tsdbMergeCommitLast(pCommitter, pIter); - if (code) goto _err; - - // end - if (pCommitter->dWriter.mBlock.nItem > 0) { - SBlockIdx blockIdx = {.suid = pTbData->suid, .uid = pTbData->uid}; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); - if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } - -_exit: - pRow = tsdbTbDataIterGet(pIter); - if (pRow) { - pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); - } - - return code; - -_err: - tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbCommitFileDataEnd(SCommitter *pCommitter) { - int32_t code = 0; - - // write aBlockIdx - code = tsdbWriteBlockIdx(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockIdx); - if (code) goto _err; - - // write aBlockL - code = tsdbWriteBlockL(pCommitter->dWriter.pWriter, pCommitter->dWriter.aBlockL); - if (code) goto _err; - - // update file header - code = tsdbUpdateDFileSetHeader(pCommitter->dWriter.pWriter); - if (code) goto _err; - - // upsert SDFileSet - code = tsdbFSUpsertFSet(&pCommitter->fs, &pCommitter->dWriter.pWriter->wSet); - if (code) goto _err; - - // close and sync - code = tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 1); - if (code) goto _err; - - if (pCommitter->dReader.pReader) { - code = tsdbDataFReaderClose(&pCommitter->dReader.pReader); - if (code) goto _err; - } - -_exit: - return code; - -_err: - tsdbError("vgId:%d, commit file data end failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbMoveCommitData(SCommitter *pCommitter, TABLEID toTable) { - int32_t code = 0; - - // .data - while (true) { - if (pCommitter->dReader.pBlockIdx == NULL || tTABLEIDCmprFn(pCommitter->dReader.pBlockIdx, &toTable) >= 0) break; - - SBlockIdx blockIdx = *pCommitter->dReader.pBlockIdx; - code = tsdbWriteBlock(pCommitter->dWriter.pWriter, &pCommitter->dReader.mBlock, &blockIdx); - if (code) goto _err; - - if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; - } - - // .last - while (true) { - if (pCommitter->dReader.pRowInfo == NULL || tTABLEIDCmprFn(pCommitter->dReader.pRowInfo, &toTable) >= 0) break; - - SBlockData *pBlockDataR = &pCommitter->dReader.bDatal; - SBlockData *pBlockDataW = &pCommitter->dWriter.bDatal; - tb_uid_t suid = pCommitter->dReader.pRowInfo->suid; - tb_uid_t uid = pCommitter->dReader.pRowInfo->uid; - - ASSERT((pBlockDataR->suid && !pBlockDataR->uid) || (!pBlockDataR->suid && pBlockDataR->uid)); - ASSERT(pBlockDataR->nRow > 0); - - // commit and reset block data schema if need - if (pBlockDataW->suid || pBlockDataW->uid) { - if (pBlockDataW->suid != suid || pBlockDataW->suid == 0) { - if (pBlockDataW->nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - tBlockDataReset(pBlockDataW); - } - } - - // set block data schema if need - if (pBlockDataW->suid == 0 && pBlockDataW->uid == 0) { - code = tsdbCommitterUpdateTableSchema(pCommitter, suid, uid); - if (code) goto _err; - - code = tBlockDataInit(pBlockDataW, suid, suid ? 0 : uid, pCommitter->skmTable.pTSchema); - if (code) goto _err; - } - - // check if it can make sure that one table data in one block - int32_t nRow = 0; - if (pBlockDataR->suid) { - int32_t iRow = pCommitter->dReader.iRow; - while ((iRow < pBlockDataR->nRow) && (pBlockDataR->aUid[iRow] == uid)) { - nRow++; - iRow++; - } - } else { - ASSERT(pCommitter->dReader.iRow == 0); - nRow = pBlockDataR->nRow; - } - - ASSERT(nRow > 0 && nRow < pCommitter->minRow); - - if (pBlockDataW->nRow + nRow > pCommitter->maxRow) { - ASSERT(pBlockDataW->nRow > 0); - - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - - while (nRow > 0) { - code = tBlockDataAppendRow(pBlockDataW, &pCommitter->dReader.pRowInfo->row, NULL, uid); - if (code) goto _err; - - code = tsdbCommitterNextLastRow(pCommitter); - if (code) goto _err; - - nRow--; - } - } - - return code; - -_err: - tsdbError("vgId:%d tsdb move commit data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbCommitFileData(SCommitter *pCommitter) { - int32_t code = 0; - STsdb *pTsdb = pCommitter->pTsdb; - SMemTable *pMemTable = pTsdb->imem; - - // commit file data start - code = tsdbCommitFileDataStart(pCommitter); - if (code) goto _err; - - // commit file data impl - for (int32_t iTbData = 0; iTbData < taosArrayGetSize(pCommitter->aTbDataP); iTbData++) { - STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, iTbData); - - // move commit until current (suid, uid) - code = tsdbMoveCommitData(pCommitter, *(TABLEID *)pTbData); - if (code) goto _err; - - // commit current table data - code = tsdbCommitTableData(pCommitter, pTbData); - if (code) goto _err; - - // move next reader table data if need - if (pCommitter->dReader.pBlockIdx && tTABLEIDCmprFn(pTbData, pCommitter->dReader.pBlockIdx) == 0) { - code = tsdbCommitterNextTableData(pCommitter); - if (code) goto _err; - } - } - - code = tsdbMoveCommitData(pCommitter, (TABLEID){.suid = INT64_MAX, .uid = INT64_MAX}); - if (code) goto _err; - - if (pCommitter->dWriter.bDatal.nRow > 0) { - code = tsdbCommitLastBlock(pCommitter); - if (code) goto _err; - } - - // commit file data end - code = tsdbCommitFileDataEnd(pCommitter); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, commit file data failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - tsdbDataFReaderClose(&pCommitter->dReader.pReader); - tsdbDataFWriterClose(&pCommitter->dWriter.pWriter, 0); - return code; -} - -// ---------------------------------------------------------------------------- -static int32_t tsdbStartCommit(STsdb *pTsdb, SCommitter *pCommitter) { - int32_t code = 0; - - memset(pCommitter, 0, sizeof(*pCommitter)); - ASSERT(pTsdb->mem && pTsdb->imem == NULL); - - taosThreadRwlockWrlock(&pTsdb->rwLock); - pTsdb->imem = pTsdb->mem; - pTsdb->mem = NULL; - taosThreadRwlockUnlock(&pTsdb->rwLock); - - pCommitter->pTsdb = pTsdb; - pCommitter->commitID = pTsdb->pVnode->state.commitID; - pCommitter->minutes = pTsdb->keepCfg.days; - pCommitter->precision = pTsdb->keepCfg.precision; - pCommitter->minRow = pTsdb->pVnode->config.tsdbCfg.minRows; - pCommitter->maxRow = pTsdb->pVnode->config.tsdbCfg.maxRows; - pCommitter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; - pCommitter->aTbDataP = tsdbMemTableGetTbDataArray(pTsdb->imem); - if (pCommitter->aTbDataP == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - code = tsdbFSCopy(pTsdb, &pCommitter->fs); - if (code) goto _err; - - return code; - -_err: - tsdbError("vgId:%d, tsdb start commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -static int32_t tsdbCommitDataStart(SCommitter *pCommitter) { - int32_t code = 0; - - // Reader - pCommitter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dReader.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - code = tBlockDataCreate(&pCommitter->dReader.bData); - if (code) goto _exit; - - pCommitter->dReader.aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pCommitter->dReader.aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - code = tBlockDataCreate(&pCommitter->dReader.bDatal); - if (code) goto _exit; - - // Writer - pCommitter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pCommitter->dWriter.aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - pCommitter->dWriter.aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pCommitter->dWriter.aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - code = tBlockDataCreate(&pCommitter->dWriter.bData); - if (code) goto _exit; - - code = tBlockDataCreate(&pCommitter->dWriter.bDatal); - if (code) goto _exit; - -_exit: - return code; -} - -static void tsdbCommitDataEnd(SCommitter *pCommitter) { - // Reader - taosArrayDestroy(pCommitter->dReader.aBlockIdx); - tMapDataClear(&pCommitter->dReader.mBlock); - tBlockDataDestroy(&pCommitter->dReader.bData, 1); - taosArrayDestroy(pCommitter->dReader.aBlockL); - tBlockDataDestroy(&pCommitter->dReader.bDatal, 1); + // merger + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { + SDataIter *pIter = &pCommitter->aDataIter[iStt]; + taosArrayDestroy(pIter->aSttBlk); + tBlockDataDestroy(&pIter->bData, 1); + } - // Writer + // writer taosArrayDestroy(pCommitter->dWriter.aBlockIdx); - taosArrayDestroy(pCommitter->dWriter.aBlockL); + taosArrayDestroy(pCommitter->dWriter.aSttBlk); tMapDataClear(&pCommitter->dWriter.mBlock); tBlockDataDestroy(&pCommitter->dWriter.bData, 1); tBlockDataDestroy(&pCommitter->dWriter.bDatal, 1); @@ -1389,7 +861,7 @@ static int32_t tsdbCommitData(SCommitter *pCommitter) { tsdbCommitDataEnd(pCommitter); _exit: - tsdbDebug("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); + tsdbInfo("vgId:%d, commit data done, nRow:%" PRId64, TD_VID(pTsdb->pVnode), pMemTable->nRow); return code; _err: @@ -1515,6 +987,11 @@ static int32_t tsdbEndCommit(SCommitter *pCommitter, int32_t eno) { tsdbFSDestroy(&pCommitter->fs); taosArrayDestroy(pCommitter->aTbDataP); + // if (pCommitter->toMerge) { + // code = tsdbMerge(pTsdb); + // if (code) goto _err; + // } + tsdbInfo("vgId:%d, tsdb end commit", TD_VID(pTsdb->pVnode)); return code; @@ -1522,3 +999,473 @@ _err: tsdbError("vgId:%d, tsdb end commit failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } + +// ================================================================================ + +static FORCE_INLINE SRowInfo *tsdbGetCommitRow(SCommitter *pCommitter) { + return (pCommitter->pIter) ? &pCommitter->pIter->r : NULL; +} + +static int32_t tsdbNextCommitRow(SCommitter *pCommitter) { + int32_t code = 0; + + if (pCommitter->pIter) { + SDataIter *pIter = pCommitter->pIter; + if (pCommitter->pIter->type == MEMORY_DATA_ITER) { // memory + tsdbTbDataIterNext(&pIter->iter); + TSDBROW *pRow = tsdbTbDataIterGet(&pIter->iter); + while (true) { + if (pRow && TSDBROW_TS(pRow) > pCommitter->maxKey) { + pCommitter->nextKey = TMIN(pCommitter->nextKey, TSDBROW_TS(pRow)); + pRow = NULL; + } + + if (pRow) { + pIter->r.suid = pIter->iter.pTbData->suid; + pIter->r.uid = pIter->iter.pTbData->uid; + pIter->r.row = *pRow; + break; + } + + pIter->iTbDataP++; + if (pIter->iTbDataP < taosArrayGetSize(pCommitter->aTbDataP)) { + STbData *pTbData = (STbData *)taosArrayGetP(pCommitter->aTbDataP, pIter->iTbDataP); + TSDBKEY keyFrom = {.ts = pCommitter->minKey, .version = VERSION_MIN}; + tsdbTbDataIterOpen(pTbData, &keyFrom, 0, &pIter->iter); + pRow = tsdbTbDataIterGet(&pIter->iter); + continue; + } else { + pCommitter->pIter = NULL; + break; + } + } + } else if (pCommitter->pIter->type == STT_DATA_ITER) { // last file + pIter->iRow++; + if (pIter->iRow < pIter->bData.nRow) { + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + } else { + pIter->iSttBlk++; + if (pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk)) { + SSttBlk *pSttBlk = (SSttBlk *)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); + + code = tsdbReadSttBlock(pCommitter->dReader.pReader, pIter->iStt, pSttBlk, &pIter->bData); + if (code) goto _exit; + + pIter->iRow = 0; + pIter->r.suid = pIter->bData.suid; + pIter->r.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[0]; + pIter->r.row = tsdbRowFromBlockData(&pIter->bData, 0); + } else { + pCommitter->pIter = NULL; + } + } + } else { + ASSERT(0); + } + + // compare with min in RB Tree + pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); + if (pCommitter->pIter && pIter) { + int32_t c = tRowInfoCmprFn(&pCommitter->pIter->r, &pIter->r); + if (c > 0) { + tRBTreePut(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); + pCommitter->pIter = NULL; + } else { + ASSERT(c); + } + } + } + + if (pCommitter->pIter == NULL) { + pCommitter->pIter = (SDataIter *)tRBTreeMin(&pCommitter->rbt); + if (pCommitter->pIter) { + tRBTreeDrop(&pCommitter->rbt, (SRBTreeNode *)pCommitter->pIter); + } + } + +_exit: + return code; +} + +static int32_t tsdbCommitAheadBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { + int32_t code = 0; + SBlockData *pBlockData = &pCommitter->dWriter.bData; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; + + tBlockDataClear(pBlockData); + while (pRowInfo) { + ASSERT(pRowInfo->row.type == 0); + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo) { + if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { + pRowInfo = NULL; + } else { + TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); + if (tsdbKeyCmprFn(&tKey, &pDataBlk->minKey) >= 0) pRowInfo = NULL; + } + } + + if (pBlockData->nRow >= pCommitter->maxRow) { + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + } + } + + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBlockData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d, tsdb commit ahead block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbCommitMergeBlock(SCommitter *pCommitter, SDataBlk *pDataBlk) { + int32_t code = 0; + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; + SBlockData *pBDataR = &pCommitter->dReader.bData; + SBlockData *pBDataW = &pCommitter->dWriter.bData; + + code = tsdbReadDataBlock(pCommitter->dReader.pReader, pDataBlk, pBDataR); + if (code) goto _err; + + tBlockDataClear(pBDataW); + int32_t iRow = 0; + TSDBROW row = tsdbRowFromBlockData(pBDataR, 0); + TSDBROW *pRow = &row; + + while (pRow && pRowInfo) { + int32_t c = tsdbRowCmprFn(pRow, &pRowInfo->row); + if (c < 0) { + code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); + if (code) goto _err; + + iRow++; + if (iRow < pBDataR->nRow) { + row = tsdbRowFromBlockData(pBDataR, iRow); + } else { + pRow = NULL; + } + } else if (c > 0) { + ASSERT(pRowInfo->row.type == 0); + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + + code = tBlockDataAppendRow(pBDataW, &pRowInfo->row, pCommitter->skmRow.pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo) { + if (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid) { + pRowInfo = NULL; + } else { + TSDBKEY tKey = TSDBROW_KEY(&pRowInfo->row); + if (tsdbKeyCmprFn(&tKey, &pDataBlk->maxKey) > 0) pRowInfo = NULL; + } + } + } else { + ASSERT(0); + } + + if (pBDataW->nRow >= pCommitter->maxRow) { + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + } + } + + while (pRow) { + code = tBlockDataAppendRow(pBDataW, pRow, NULL, id.uid); + if (code) goto _err; + + iRow++; + if (iRow < pBDataR->nRow) { + row = tsdbRowFromBlockData(pBDataR, iRow); + } else { + pRow = NULL; + } + + if (pBDataW->nRow >= pCommitter->maxRow) { + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + } + } + + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBDataW, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d, tsdb commit merge block failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbMergeTableData(SCommitter *pCommitter, TABLEID id) { + int32_t code = 0; + SBlockIdx *pBlockIdx = pCommitter->dReader.pBlockIdx; + + ASSERT(pBlockIdx == NULL || tTABLEIDCmprFn(pBlockIdx, &id) >= 0); + if (pBlockIdx && pBlockIdx->suid == id.suid && pBlockIdx->uid == id.uid) { + int32_t iBlock = 0; + SDataBlk block; + SDataBlk *pDataBlk = █ + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + + ASSERT(pRowInfo->suid == id.suid && pRowInfo->uid == id.uid); + + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); + while (pDataBlk && pRowInfo) { + SDataBlk tBlock = {.minKey = TSDBROW_KEY(&pRowInfo->row), .maxKey = TSDBROW_KEY(&pRowInfo->row)}; + int32_t c = tDataBlkCmprFn(pDataBlk, &tBlock); + + if (c < 0) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); + } else { + pDataBlk = NULL; + } + } else if (c > 0) { + code = tsdbCommitAheadBlock(pCommitter, pDataBlk); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; + } else { + code = tsdbCommitMergeBlock(pCommitter, pDataBlk); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); + } else { + pDataBlk = NULL; + } + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) pRowInfo = NULL; + } + } + + while (pDataBlk) { + code = tMapDataPutItem(&pCommitter->dWriter.mBlock, pDataBlk, tPutDataBlk); + if (code) goto _err; + + iBlock++; + if (iBlock < pCommitter->dReader.mBlock.nItem) { + tMapDataGetItemByIdx(&pCommitter->dReader.mBlock, iBlock, pDataBlk, tGetDataBlk); + } else { + pDataBlk = NULL; + } + } + + code = tsdbCommitterNextTableData(pCommitter); + if (code) goto _err; + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb merge table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbInitLastBlockIfNeed(SCommitter *pCommitter, TABLEID id) { + int32_t code = 0; + + SBlockData *pBDatal = &pCommitter->dWriter.bDatal; + if (pBDatal->suid || pBDatal->uid) { + if ((pBDatal->suid != id.suid) || (id.suid == 0)) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + if (code) goto _exit; + tBlockDataReset(pBDatal); + } + } + + if (!pBDatal->suid && !pBDatal->uid) { + ASSERT(pCommitter->skmTable.suid == id.suid); + ASSERT(pCommitter->skmTable.uid == id.uid); + code = tBlockDataInit(pBDatal, id.suid, id.suid ? 0 : id.uid, pCommitter->skmTable.pTSchema); + if (code) goto _exit; + } + +_exit: + return code; +} + +static int32_t tsdbAppendLastBlock(SCommitter *pCommitter) { + int32_t code = 0; + + SBlockData *pBData = &pCommitter->dWriter.bData; + SBlockData *pBDatal = &pCommitter->dWriter.bDatal; + + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid}; + code = tsdbInitLastBlockIfNeed(pCommitter, id); + if (code) goto _err; + + for (int32_t iRow = 0; iRow < pBData->nRow; iRow++) { + TSDBROW row = tsdbRowFromBlockData(pBData, iRow); + code = tBlockDataAppendRow(pBDatal, &row, NULL, pBData->uid); + if (code) goto _err; + + if (pBDatal->nRow >= pCommitter->maxRow) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBDatal, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + if (code) goto _err; + } + } + + return code; + +_err: + return code; +} + +static int32_t tsdbCommitTableData(SCommitter *pCommitter, TABLEID id) { + int32_t code = 0; + + SRowInfo *pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pRowInfo == NULL) goto _exit; + + SBlockData *pBData; + if (pCommitter->toLastOnly) { + pBData = &pCommitter->dWriter.bDatal; + code = tsdbInitLastBlockIfNeed(pCommitter, id); + if (code) goto _err; + } else { + pBData = &pCommitter->dWriter.bData; + ASSERT(pBData->nRow == 0); + } + + while (pRowInfo) { + STSchema *pTSchema = NULL; + if (pRowInfo->row.type == 0) { + code = tsdbCommitterUpdateRowSchema(pCommitter, id.suid, id.uid, TSDBROW_SVERSION(&pRowInfo->row)); + if (code) goto _err; + pTSchema = pCommitter->skmRow.pTSchema; + } + + code = tBlockDataAppendRow(pBData, &pRowInfo->row, pTSchema, id.uid); + if (code) goto _err; + + code = tsdbNextCommitRow(pCommitter); + if (code) goto _err; + + pRowInfo = tsdbGetCommitRow(pCommitter); + if (pRowInfo && (pRowInfo->suid != id.suid || pRowInfo->uid != id.uid)) { + pRowInfo = NULL; + } + + if (pBData->nRow >= pCommitter->maxRow) { + if (pCommitter->toLastOnly) { + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, pBData, pCommitter->dWriter.aSttBlk, pCommitter->cmprAlg); + if (code) goto _err; + } else { + code = + tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + } + } + } + + if (!pCommitter->toLastOnly && pBData->nRow) { + if (pBData->nRow > pCommitter->minRow) { + code = tsdbWriteDataBlock(pCommitter->dWriter.pWriter, pBData, &pCommitter->dWriter.mBlock, pCommitter->cmprAlg); + if (code) goto _err; + } else { + code = tsdbAppendLastBlock(pCommitter); + if (code) goto _err; + } + } + +_exit: + return code; + +_err: + tsdbError("vgId:%d tsdb commit table data failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbCommitFileDataImpl(SCommitter *pCommitter) { + int32_t code = 0; + + SRowInfo *pRowInfo; + TABLEID id = {0}; + while ((pRowInfo = tsdbGetCommitRow(pCommitter)) != NULL) { + ASSERT(pRowInfo->suid != id.suid || pRowInfo->uid != id.uid); + id.suid = pRowInfo->suid; + id.uid = pRowInfo->uid; + + code = tsdbMoveCommitData(pCommitter, id); + if (code) goto _err; + + // start + tMapDataReset(&pCommitter->dWriter.mBlock); + + // impl + code = tsdbUpdateTableSchema(pCommitter->pTsdb->pVnode->pMeta, id.suid, id.uid, &pCommitter->skmTable); + if (code) goto _err; + code = tBlockDataInit(&pCommitter->dReader.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; + code = tBlockDataInit(&pCommitter->dWriter.bData, id.suid, id.uid, pCommitter->skmTable.pTSchema); + if (code) goto _err; + + /* merge with data in .data file */ + code = tsdbMergeTableData(pCommitter, id); + if (code) goto _err; + + /* handle remain table data */ + code = tsdbCommitTableData(pCommitter, id); + if (code) goto _err; + + // end + if (pCommitter->dWriter.mBlock.nItem > 0) { + SBlockIdx blockIdx = {.suid = id.suid, .uid = id.uid}; + code = tsdbWriteDataBlk(pCommitter->dWriter.pWriter, &pCommitter->dWriter.mBlock, &blockIdx); + if (code) goto _err; + + if (taosArrayPush(pCommitter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + } + + id.suid = INT64_MAX; + id.uid = INT64_MAX; + code = tsdbMoveCommitData(pCommitter, id); + if (code) goto _err; + + code = tsdbWriteSttBlock(pCommitter->dWriter.pWriter, &pCommitter->dWriter.bDatal, pCommitter->dWriter.aSttBlk, + pCommitter->cmprAlg); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb commit file data impl failed since %s", TD_VID(pCommitter->pTsdb->pVnode), tstrerror(code)); + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCompact.c b/source/dnode/vnode/src/tsdb/tsdbCompact.c new file mode 100644 index 0000000000000000000000000000000000000000..fb3917be64faa058b52f1f13a86ec0034486f279 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCompact.c @@ -0,0 +1,27 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct { + STsdb *pTsdb; + STsdbFS fs; +} STsdbCompactor; + +int32_t tsdbCompact(STsdb *pTsdb) { + int32_t code = 0; + // TODO + return code; +} diff --git a/source/dnode/vnode/src/tsdb/tsdbCompress.c b/source/dnode/vnode/src/tsdb/tsdbCompress.c new file mode 100644 index 0000000000000000000000000000000000000000..76be7c10708a1150b26953745b3f2cb21aaeb6fa --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbCompress.c @@ -0,0 +1,64 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +// Integer ===================================================== +typedef struct { + int8_t rawCopy; + int64_t prevVal; + int32_t nVal; + int32_t nBuf; + uint8_t *pBuf; +} SIntCompressor; + +#define I64_SAFE_ADD(a, b) (((a) >= 0 && (b) <= INT64_MAX - (b)) || ((a) < 0 && (b) >= INT64_MIN - (a))) +#define SIMPLE8B_MAX ((uint64_t)1152921504606846974LL) + +static int32_t tsdbCmprI64(SIntCompressor *pCompressor, int64_t val) { + int32_t code = 0; + + // raw copy + if (pCompressor->rawCopy) { + memcpy(pCompressor->pBuf + pCompressor->nBuf, &val, sizeof(val)); + pCompressor->nBuf += sizeof(val); + pCompressor->nVal++; + goto _exit; + } + + if (!I64_SAFE_ADD(val, pCompressor->prevVal)) { + pCompressor->rawCopy = 1; + // TODO: decompress and copy + pCompressor->nVal++; + goto _exit; + } + + int64_t diff = val - pCompressor->prevVal; + uint8_t zigzag = ZIGZAGE(int64_t, diff); + + if (zigzag >= SIMPLE8B_MAX) { + pCompressor->rawCopy = 1; + // TODO: decompress and copy + pCompressor->nVal++; + goto _exit; + } + +_exit: + return code; +} + +// Timestamp ===================================================== + +// Float ===================================================== \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbDiskData.c b/source/dnode/vnode/src/tsdb/tsdbDiskData.c new file mode 100644 index 0000000000000000000000000000000000000000..3bd71f0ea6465cadcda3924247715a88721be3d8 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbDiskData.c @@ -0,0 +1,84 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +typedef struct SDiskColBuilder SDiskColBuilder; +struct SDiskColBuilder { + uint8_t flags; + uint8_t *pBitMap; + int32_t *aOffset; + int32_t nData; + uint8_t *pData; +}; + +int32_t tDiskColAddVal(SDiskColBuilder *pBuilder, SColVal *pColVal) { + int32_t code = 0; + // TODO + return code; +} + +// ================================================================ +typedef struct SDiskDataBuilder SDiskDataBuilder; +struct SDiskDataBuilder { + SDiskDataHdr hdr; + SArray *aBlockCol; // SArray +}; + +int32_t tDiskDataBuilderCreate(SDiskDataBuilder **ppBuilder) { + int32_t code = 0; + // TODO + return code; +} + +void tDiskDataBuilderDestroy(SDiskDataBuilder *pBuilder) { + // TODO +} + +void tDiskDataBuilderInit(SDiskDataBuilder *pBuilder, int64_t suid, int64_t uid, STSchema *pTSchema, int8_t cmprAlg) { + pBuilder->hdr = (SDiskDataHdr){.delimiter = TSDB_FILE_DLMT, // + .fmtVer = 0, + .suid = suid, + .uid = uid, + .cmprAlg = cmprAlg}; +} + +void tDiskDataBuilderReset(SDiskDataBuilder *pBuilder) { + // TODO +} + +int32_t tDiskDataBuilderAddRow(SDiskDataBuilder *pBuilder, TSDBROW *pRow, STSchema *pTSchema, int64_t uid) { + int32_t code = 0; + + // uid (todo) + + // version (todo) + + // TSKEY (todo) + + SRowIter iter = {0}; + tRowIterInit(&iter, pRow, pTSchema); + + for (int32_t iDiskCol = 0; iDiskCol < 0; iDiskCol++) { + } + + return code; +} + +int32_t tDiskDataBuilderGet(SDiskDataBuilder *pBuilder, uint8_t **ppData) { + int32_t code = 0; + // TODO + return code; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbFS.c b/source/dnode/vnode/src/tsdb/tsdbFS.c index 247de993381d98713fa6a4ca1938c11b044c8cd6..10926ae6ad31ae8d609dd20819ad03bec2e26c57 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFS.c +++ b/source/dnode/vnode/src/tsdb/tsdbFS.c @@ -21,6 +21,9 @@ static int32_t tsdbEncodeFS(uint8_t *p, STsdbFS *pFS) { int8_t hasDel = pFS->pDelFile ? 1 : 0; uint32_t nSet = taosArrayGetSize(pFS->aDFileSet); + // version + n += tPutI8(p ? p + n : p, 0); + // SDelFile n += tPutI8(p ? p + n : p, hasDel); if (hasDel) { @@ -55,7 +58,7 @@ static int32_t tsdbGnrtCurrent(STsdb *pTsdb, STsdbFS *pFS, char *fname) { taosCalcChecksumAppend(0, pData, size); // create and write - pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE); + pFD = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); if (pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; @@ -110,7 +113,7 @@ _err: // taosRemoveFile(fname); // } -// // last +// // stt // if (isSameDisk && pFrom->pLastF->commitID == pTo->pLastF->commitID) { // if (pFrom->pLastF->size > pTo->pLastF->size) { // code = tsdbDFileRollback(pFS->pTsdb, pTo, TSDB_LAST_FILE); @@ -140,7 +143,7 @@ _err: // tsdbDataFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pDataF, fname); // taosRemoveFile(fname); -// // last +// // stt // tsdbLastFileName(pFS->pTsdb, pFrom->diskId, pFrom->fid, pFrom->pLastF, fname); // taosRemoveFile(fname); @@ -254,8 +257,10 @@ void tsdbFSDestroy(STsdbFS *pFS) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pFS->aDFileSet, iSet); taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pLastF); taosMemoryFree(pSet->pSmaF); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + taosMemoryFree(pSet->aSttF[iStt]); + } } taosArrayDestroy(pFS->aDFileSet); @@ -274,7 +279,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { goto _err; } - if (size != pTsdb->fs.pDelFile->size) { + if (size != tsdbLogicToFileSize(pTsdb->fs.pDelFile->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -290,7 +295,7 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size != pSet->pHeadF->size) { + if (size != tsdbLogicToFileSize(pSet->pHeadF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; } @@ -301,38 +306,40 @@ static int32_t tsdbScanAndTryFixFS(STsdb *pTsdb) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size < pSet->pDataF->size) { + if (size < tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; - } else if (size > pSet->pDataF->size) { + } else if (size > tsdbLogicToFileSize(pSet->pDataF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_DATA_FILE); if (code) goto _err; } - // last =========== - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); - if (taosStatFile(fname, &size, NULL)) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (size != pSet->pLastF->size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - // sma ============= tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); if (taosStatFile(fname, &size, NULL)) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - if (size < pSet->pSmaF->size) { + if (size < tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = TSDB_CODE_FILE_CORRUPTED; goto _err; - } else if (size > pSet->pSmaF->size) { + } else if (size > tsdbLogicToFileSize(pSet->pSmaF->size, pTsdb->pVnode->config.tsdbPageSize)) { code = tsdbDFileRollback(pTsdb, pSet, TSDB_SMA_FILE); if (code) goto _err; } + + // stt =========== + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); + if (taosStatFile(fname, &size, NULL)) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + if (size != tsdbLogicToFileSize(pSet->aSttF[iStt]->size, pTsdb->pVnode->config.tsdbPageSize)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _err; + } + } } { @@ -360,10 +367,12 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) { int32_t code = 0; int8_t hasDel; uint32_t nSet; - int32_t n; + int32_t n = 0; + + // version + n += tGetI8(pData + n, NULL); // SDelFile - n = 0; n += tGetI8(pData + n, &hasDel); if (hasDel) { pTsdb->fs.pDelFile = (SDelFile *)taosMemoryMalloc(sizeof(SDelFile)); @@ -382,41 +391,15 @@ static int32_t tsdbRecoverFS(STsdb *pTsdb, uint8_t *pData, int64_t nData) { taosArrayClear(pTsdb->fs.aDFileSet); n += tGetU32v(pData + n, &nSet); for (uint32_t iSet = 0; iSet < nSet; iSet++) { - SDFileSet fSet; - - // head - fSet.pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); - if (fSet.pHeadF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - fSet.pHeadF->nRef = 1; - - // data - fSet.pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); - if (fSet.pDataF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - fSet.pDataF->nRef = 1; - - // last - fSet.pLastF = (SLastFile *)taosMemoryCalloc(1, sizeof(SLastFile)); - if (fSet.pLastF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - fSet.pLastF->nRef = 1; + SDFileSet fSet = {0}; - // sma - fSet.pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { + int32_t nt = tGetDFileSet(pData + n, &fSet); + if (nt < 0) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - fSet.pSmaF->nRef = 1; - n += tGetDFileSet(pData + n, &fSet); + n += nt; if (taosArrayPush(pTsdb->fs.aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -532,13 +515,15 @@ int32_t tsdbFSClose(STsdb *pTsdb) { ASSERT(pSet->pDataF->nRef == 1); taosMemoryFree(pSet->pDataF); - // last - ASSERT(pSet->pLastF->nRef == 1); - taosMemoryFree(pSet->pLastF); - // sma ASSERT(pSet->pSmaF->nRef == 1); taosMemoryFree(pSet->pSmaF); + + // stt + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + ASSERT(pSet->aSttF[iStt]->nRef == 1); + taosMemoryFree(pSet->aSttF[iStt]); + } } taosArrayDestroy(pTsdb->fs.aDFileSet); @@ -586,15 +571,7 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pDataF = *pSet->pDataF; - // data - fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.pLastF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - *fSet.pLastF = *pSet->pLastF; - - // last + // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -602,6 +579,16 @@ int32_t tsdbFSCopy(STsdb *pTsdb, STsdbFS *pFS) { } *fSet.pSmaF = *pSet->pSmaF; + // stt + for (fSet.nSttF = 0; fSet.nSttF < pSet->nSttF; fSet.nSttF++) { + fSet.aSttF[fSet.nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[fSet.nSttF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + *fSet.aSttF[fSet.nSttF] = *pSet->aSttF[fSet.nSttF]; + } + if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -651,14 +638,38 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { if (c == 0) { *pDFileSet->pHeadF = *pSet->pHeadF; *pDFileSet->pDataF = *pSet->pDataF; - *pDFileSet->pLastF = *pSet->pLastF; *pDFileSet->pSmaF = *pSet->pSmaF; + // stt + if (pSet->nSttF > pDFileSet->nSttF) { + ASSERT(pSet->nSttF == pDFileSet->nSttF + 1); + + pDFileSet->aSttF[pDFileSet->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pDFileSet->aSttF[pDFileSet->nSttF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + *pDFileSet->aSttF[pDFileSet->nSttF] = *pSet->aSttF[pSet->nSttF - 1]; + pDFileSet->nSttF++; + } else if (pSet->nSttF < pDFileSet->nSttF) { + ASSERT(pSet->nSttF == 1); + for (int32_t iStt = 1; iStt < pDFileSet->nSttF; iStt++) { + taosMemoryFree(pDFileSet->aSttF[iStt]); + } + + *pDFileSet->aSttF[0] = *pSet->aSttF[0]; + pDFileSet->nSttF = 1; + } else { + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + *pDFileSet->aSttF[iStt] = *pSet->aSttF[iStt]; + } + } goto _exit; } } - SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid}; + ASSERT(pSet->nSttF == 1); + SDFileSet fSet = {.diskId = pSet->diskId, .fid = pSet->fid, .nSttF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -676,21 +687,21 @@ int32_t tsdbFSUpsertFSet(STsdbFS *pFS, SDFileSet *pSet) { } *fSet.pDataF = *pSet->pDataF; - // data - fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.pLastF == NULL) { + // sma + fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); + if (fSet.pSmaF == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.pLastF = *pSet->pLastF; + *fSet.pSmaF = *pSet->pSmaF; - // last - fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); - if (fSet.pSmaF == NULL) { + // stt + fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[0] == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; } - *fSet.pSmaF = *pSet->pSmaF; + *fSet.aSttF[0] = *pSet->aSttF[0]; if (taosArrayInsert(pFS->aDFileSet, idx, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -836,27 +847,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->pDataF->size = pSetNew->pDataF->size; } - // last - fSet.pLastF = pSetOld->pLastF; - if ((!sameDisk) || (pSetOld->pLastF->commitID != pSetNew->pLastF->commitID)) { - pSetOld->pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (pSetOld->pLastF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *pSetOld->pLastF = *pSetNew->pLastF; - pSetOld->pLastF->nRef = 1; - - nRef = atomic_sub_fetch_32(&fSet.pLastF->nRef, 1); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, fSet.pLastF, fname); - taosRemoveFile(fname); - taosMemoryFree(fSet.pLastF); - } - } else { - ASSERT(pSetOld->pLastF->size == pSetNew->pLastF->size); - } - // sma fSet.pSmaF = pSetOld->pSmaF; if ((!sameDisk) || (pSetOld->pSmaF->commitID != pSetNew->pSmaF->commitID)) { @@ -879,6 +869,84 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { pSetOld->pSmaF->size = pSetNew->pSmaF->size; } + // stt + if (sameDisk) { + if (pSetNew->nSttF > pSetOld->nSttF) { + ASSERT(pSetNew->nSttF = pSetOld->nSttF + 1); + pSetOld->aSttF[pSetOld->nSttF] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[pSetOld->nSttF] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[pSetOld->nSttF] = *pSetNew->aSttF[pSetOld->nSttF]; + pSetOld->aSttF[pSetOld->nSttF]->nRef = 1; + pSetOld->nSttF++; + } else if (pSetNew->nSttF < pSetOld->nSttF) { + ASSERT(pSetNew->nSttF == 1); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + pSetOld->aSttF[iStt] = NULL; + } + + pSetOld->nSttF = 1; + pSetOld->aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[0] = *pSetNew->aSttF[0]; + pSetOld->aSttF[0]->nRef = 1; + } else { + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + if (pSetOld->aSttF[iStt]->commitID != pSetNew->aSttF[iStt]->commitID) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } else { + ASSERT(pSetOld->aSttF[iStt]->size == pSetOld->aSttF[iStt]->size); + ASSERT(pSetOld->aSttF[iStt]->offset == pSetOld->aSttF[iStt]->offset); + } + } + } + } else { + ASSERT(pSetOld->nSttF == pSetNew->nSttF); + for (int32_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + SSttFile *pSttFile = pSetOld->aSttF[iStt]; + nRef = atomic_sub_fetch_32(&pSttFile->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSttFile, fname); + taosRemoveFile(fname); + taosMemoryFree(pSttFile); + } + + pSetOld->aSttF[iStt] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (pSetOld->aSttF[iStt] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *pSetOld->aSttF[iStt] = *pSetNew->aSttF[iStt]; + pSetOld->aSttF[iStt]->nRef = 1; + } + } + if (!sameDisk) { pSetOld->diskId = pSetNew->diskId; } @@ -902,13 +970,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pDataF); } - nRef = atomic_sub_fetch_32(&pSetOld->pLastF->nRef, 1); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pLastF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSetOld->pLastF); - } - nRef = atomic_sub_fetch_32(&pSetOld->pSmaF->nRef, 1); if (nRef == 0) { tsdbSmaFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->pSmaF, fname); @@ -916,12 +977,20 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { taosMemoryFree(pSetOld->pSmaF); } + for (int8_t iStt = 0; iStt < pSetOld->nSttF; iStt++) { + nRef = atomic_sub_fetch_32(&pSetOld->aSttF[iStt]->nRef, 1); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSetOld->diskId, pSetOld->fid, pSetOld->aSttF[iStt], fname); + taosRemoveFile(fname); + taosMemoryFree(pSetOld->aSttF[iStt]); + } + } + taosArrayRemove(pTsdb->fs.aDFileSet, iOld); continue; _add_new: - fSet.diskId = pSetNew->diskId; - fSet.fid = pSetNew->fid; + fSet = (SDFileSet){.diskId = pSetNew->diskId, .fid = pSetNew->fid, .nSttF = 1}; // head fSet.pHeadF = (SHeadFile *)taosMemoryMalloc(sizeof(SHeadFile)); @@ -941,15 +1010,6 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *fSet.pDataF = *pSetNew->pDataF; fSet.pDataF->nRef = 1; - // last - fSet.pLastF = (SLastFile *)taosMemoryMalloc(sizeof(SLastFile)); - if (fSet.pLastF == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - *fSet.pLastF = *pSetNew->pLastF; - fSet.pLastF->nRef = 1; - // sma fSet.pSmaF = (SSmaFile *)taosMemoryMalloc(sizeof(SSmaFile)); if (fSet.pSmaF == NULL) { @@ -959,6 +1019,16 @@ int32_t tsdbFSCommit2(STsdb *pTsdb, STsdbFS *pFSNew) { *fSet.pSmaF = *pSetNew->pSmaF; fSet.pSmaF->nRef = 1; + // stt + ASSERT(pSetNew->nSttF == 1); + fSet.aSttF[0] = (SSttFile *)taosMemoryMalloc(sizeof(SSttFile)); + if (fSet.aSttF[0] == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + *fSet.aSttF[0] = *pSetNew->aSttF[0]; + fSet.aSttF[0]->nRef = 1; + if (taosArrayInsert(pTsdb->fs.aDFileSet, iOld, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; @@ -1002,12 +1072,14 @@ int32_t tsdbFSRef(STsdb *pTsdb, STsdbFS *pFS) { nRef = atomic_fetch_add_32(&pSet->pDataF->nRef, 1); ASSERT(nRef > 0); - nRef = atomic_fetch_add_32(&pSet->pLastF->nRef, 1); - ASSERT(nRef > 0); - nRef = atomic_fetch_add_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef > 0); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + nRef = atomic_fetch_add_32(&pSet->aSttF[iStt]->nRef, 1); + ASSERT(nRef > 0); + } + if (taosArrayPush(pFS->aDFileSet, &fSet) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _exit; @@ -1053,15 +1125,6 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { taosMemoryFree(pSet->pDataF); } - // last - nRef = atomic_sub_fetch_32(&pSet->pLastF->nRef, 1); - ASSERT(nRef >= 0); - if (nRef == 0) { - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); - taosRemoveFile(fname); - taosMemoryFree(pSet->pLastF); - } - // sma nRef = atomic_sub_fetch_32(&pSet->pSmaF->nRef, 1); ASSERT(nRef >= 0); @@ -1070,6 +1133,18 @@ void tsdbFSUnref(STsdb *pTsdb, STsdbFS *pFS) { taosRemoveFile(fname); taosMemoryFree(pSet->pSmaF); } + + // stt + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + nRef = atomic_sub_fetch_32(&pSet->aSttF[iStt]->nRef, 1); + ASSERT(nRef >= 0); + if (nRef == 0) { + tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); + taosRemoveFile(fname); + taosMemoryFree(pSet->aSttF[iStt]); + /* code */ + } + } } taosArrayDestroy(pFS->aDFileSet); diff --git a/source/dnode/vnode/src/tsdb/tsdbFile.c b/source/dnode/vnode/src/tsdb/tsdbFile.c index 00d2ac848f6d599fef54d9957047521e27062c89..3c944584de7ae10b21cb75913d1f920198288fe4 100644 --- a/source/dnode/vnode/src/tsdb/tsdbFile.c +++ b/source/dnode/vnode/src/tsdb/tsdbFile.c @@ -53,22 +53,22 @@ static int32_t tGetDataFile(uint8_t *p, SDataFile *pDataFile) { return n; } -int32_t tPutLastFile(uint8_t *p, SLastFile *pLastFile) { +int32_t tPutSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t n = 0; - n += tPutI64v(p ? p + n : p, pLastFile->commitID); - n += tPutI64v(p ? p + n : p, pLastFile->size); - n += tPutI64v(p ? p + n : p, pLastFile->offset); + n += tPutI64v(p ? p + n : p, pSttFile->commitID); + n += tPutI64v(p ? p + n : p, pSttFile->size); + n += tPutI64v(p ? p + n : p, pSttFile->offset); return n; } -static int32_t tGetLastFile(uint8_t *p, SLastFile *pLastFile) { +static int32_t tGetSttFile(uint8_t *p, SSttFile *pSttFile) { int32_t n = 0; - n += tGetI64v(p + n, &pLastFile->commitID); - n += tGetI64v(p + n, &pLastFile->size); - n += tGetI64v(p + n, &pLastFile->offset); + n += tGetI64v(p + n, &pSttFile->commitID); + n += tGetI64v(p + n, &pSttFile->size); + n += tGetI64v(p + n, &pSttFile->offset); return n; } @@ -102,9 +102,9 @@ void tsdbDataFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SDataFile *pDataF, TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pDataF->commitID, ".data"); } -void tsdbLastFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SLastFile *pLastF, char fname[]) { +void tsdbSttFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSttFile *pSttF, char fname[]) { snprintf(fname, TSDB_FILENAME_LEN - 1, "%s%s%s%sv%df%dver%" PRId64 "%s", tfsGetDiskPath(pTsdb->pVnode->pTfs, did), - TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pLastF->commitID, ".last"); + TD_DIRSEP, pTsdb->path, TD_DIRSEP, TD_VID(pTsdb->pVnode), fid, pSttF->commitID, ".stt"); } void tsdbSmaFileName(STsdb *pTsdb, SDiskID did, int32_t fid, SSmaFile *pSmaF, char fname[]) { @@ -148,7 +148,7 @@ int32_t tsdbDFileRollback(STsdb *pTsdb, SDFileSet *pSet, EDataFileT ftype) { } // ftruncate - if (taosFtruncateFile(pFD, size) < 0) { + if (taosFtruncateFile(pFD, tsdbLogicToFileSize(size, pTsdb->pVnode->config.tsdbPageSize)) < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } @@ -194,9 +194,11 @@ int32_t tPutDFileSet(uint8_t *p, SDFileSet *pSet) { n += tPutDataFile(p ? p + n : p, pSet->pDataF); n += tPutSmaFile(p ? p + n : p, pSet->pSmaF); - // last - n += tPutU8(p ? p + n : p, 1); // for future compatibility - n += tPutLastFile(p ? p + n : p, pSet->pLastF); + // stt + n += tPutU8(p ? p + n : p, pSet->nSttF); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + n += tPutSttFile(p ? p + n : p, pSet->aSttF[iStt]); + } return n; } @@ -208,15 +210,40 @@ int32_t tGetDFileSet(uint8_t *p, SDFileSet *pSet) { n += tGetI32v(p + n, &pSet->diskId.id); n += tGetI32v(p + n, &pSet->fid); - // data + // head + pSet->pHeadF = (SHeadFile *)taosMemoryCalloc(1, sizeof(SHeadFile)); + if (pSet->pHeadF == NULL) { + return -1; + } + pSet->pHeadF->nRef = 1; n += tGetHeadFile(p + n, pSet->pHeadF); + + // data + pSet->pDataF = (SDataFile *)taosMemoryCalloc(1, sizeof(SDataFile)); + if (pSet->pDataF == NULL) { + return -1; + } + pSet->pDataF->nRef = 1; n += tGetDataFile(p + n, pSet->pDataF); + + // sma + pSet->pSmaF = (SSmaFile *)taosMemoryCalloc(1, sizeof(SSmaFile)); + if (pSet->pSmaF == NULL) { + return -1; + } + pSet->pSmaF->nRef = 1; n += tGetSmaFile(p + n, pSet->pSmaF); - // last - uint8_t nLast; - n += tGetU8(p + n, &nLast); - n += tGetLastFile(p + n, pSet->pLastF); + // stt + n += tGetU8(p + n, &pSet->nSttF); + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + pSet->aSttF[iStt] = (SSttFile *)taosMemoryCalloc(1, sizeof(SSttFile)); + if (pSet->aSttF[iStt] == NULL) { + return -1; + } + pSet->aSttF[iStt]->nRef = 1; + n += tGetSttFile(p + n, pSet->aSttF[iStt]); + } return n; } diff --git a/source/dnode/vnode/src/tsdb/tsdbMergeTree.c b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c new file mode 100644 index 0000000000000000000000000000000000000000..45fe29f0faba9fbcb43e81ad1e3022d408d8d927 --- /dev/null +++ b/source/dnode/vnode/src/tsdb/tsdbMergeTree.c @@ -0,0 +1,561 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tsdb.h" + +// SLDataIter ================================================= +struct SLDataIter { + SRBTreeNode node; + SSttBlk *pSttBlk; + SDataFReader *pReader; + int32_t iStt; + int8_t backward; + int32_t iSttBlk; + int32_t iRow; + SRowInfo rInfo; + uint64_t uid; + STimeWindow timeWindow; + SVersionRange verRange; + + SSttBlockLoadInfo* pBlockLoadInfo; +}; + +SSttBlockLoadInfo* tCreateLastBlockLoadInfo() { + SSttBlockLoadInfo* pLoadInfo = taosMemoryCalloc(TSDB_DEFAULT_STT_FILE, sizeof(SSttBlockLoadInfo)); + if (pLoadInfo == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + pLoadInfo[i].blockIndex[0] = -1; + pLoadInfo[i].blockIndex[1] = -1; + pLoadInfo[i].currentLoadBlockIndex = 1; + + int32_t code = tBlockDataCreate(&pLoadInfo[i].blockData[0]); + if (code) { + terrno = code; + } + + code = tBlockDataCreate(&pLoadInfo[i].blockData[1]); + if (code) { + terrno = code; + } + + pLoadInfo[i].aSttBlk = taosArrayInit(4, sizeof(SSttBlk)); + } + + return pLoadInfo; +} + +void resetLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + pLoadInfo[i].currentLoadBlockIndex = 1; + pLoadInfo[i].blockIndex[0] = -1; + pLoadInfo[i].blockIndex[1] = -1; + + taosArrayClear(pLoadInfo[i].aSttBlk); + + pLoadInfo[i].elapsedTime = 0; + pLoadInfo[i].loadBlocks = 0; + } +} + +void getLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo, int64_t* blocks, double* el) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + *el += pLoadInfo[i].elapsedTime; + *blocks += pLoadInfo[i].loadBlocks; + } +} + +void* destroyLastBlockLoadInfo(SSttBlockLoadInfo* pLoadInfo) { + for(int32_t i = 0; i < TSDB_DEFAULT_STT_FILE; ++i) { + pLoadInfo[i].currentLoadBlockIndex = 1; + pLoadInfo[i].blockIndex[0] = -1; + pLoadInfo[i].blockIndex[1] = -1; + + tBlockDataDestroy(&pLoadInfo[i].blockData[0], true); + tBlockDataDestroy(&pLoadInfo[i].blockData[1], true); + + taosArrayDestroy(pLoadInfo[i].aSttBlk); + } + + taosMemoryFree(pLoadInfo); + return NULL; +} + +static SBlockData* loadLastBlock(SLDataIter *pIter, const char* idStr) { + int32_t code = 0; + + SSttBlockLoadInfo* pInfo = pIter->pBlockLoadInfo; + if (pInfo->blockIndex[0] == pIter->iSttBlk) { + return &pInfo->blockData[0]; + } + + if (pInfo->blockIndex[1] == pIter->iSttBlk) { + return &pInfo->blockData[1]; + } + + pInfo->currentLoadBlockIndex ^= 1; + if (pIter->pSttBlk != NULL) { // current block not loaded yet + int64_t st = taosGetTimestampUs(); + code = tsdbReadSttBlock(pIter->pReader, pIter->iStt, pIter->pSttBlk, &pInfo->blockData[pInfo->currentLoadBlockIndex]); + double el = (taosGetTimestampUs() - st)/ 1000.0; + pInfo->elapsedTime += el; + pInfo->loadBlocks += 1; + + tsdbDebug("read last block, index:%d, last file index:%d, elapsed time:%.2f ms, %s", pIter->iSttBlk, pIter->iStt, el, idStr); + if (code != TSDB_CODE_SUCCESS) { + goto _exit; + } + + pInfo->blockIndex[pInfo->currentLoadBlockIndex] = pIter->iSttBlk; + pIter->iRow = (pIter->backward) ? pInfo->blockData[pInfo->currentLoadBlockIndex].nRow : -1; + } + + return &pInfo->blockData[pInfo->currentLoadBlockIndex]; + + _exit: + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + } + + return NULL; +} + +// find the earliest block that contains the required records +static FORCE_INLINE int32_t findEarliestIndex(int32_t index, uint64_t uid, const SSttBlk* pBlockList, int32_t num, int32_t backward) { + int32_t i = index; + int32_t step = backward? 1:-1; + while (i >= 0 && i < num && uid >= pBlockList[i].minUid && uid <= pBlockList[i].maxUid) { + i += step; + } + return i - step; +} + +static int32_t binarySearchForStartBlock(SSttBlk*pBlockList, int32_t num, uint64_t uid, int32_t backward) { + int32_t midPos = -1; + if (num <= 0) { + return -1; + } + + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + // find the first position which is bigger than the key + if ((uid > pBlockList[lastPos].maxUid) || (uid < pBlockList[firstPos].minUid)) { + return -1; + } + + while (1) { + if (uid >= pBlockList[firstPos].minUid && uid <= pBlockList[firstPos].maxUid) { + return findEarliestIndex(firstPos, uid, pBlockList, num, backward); + } + + if (uid > pBlockList[lastPos].maxUid || uid < pBlockList[firstPos].minUid) { + return -1; + } + + int32_t numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (uid < pBlockList[midPos].minUid) { + lastPos = midPos - 1; + } else if (uid > pBlockList[midPos].maxUid) { + firstPos = midPos + 1; + } else { + return findEarliestIndex(midPos, uid, pBlockList, num, backward); + } + } +} + +static FORCE_INLINE int32_t findEarliestRow(int32_t index, uint64_t uid, const uint64_t* uidList, int32_t num, int32_t backward) { + int32_t i = index; + int32_t step = backward? 1:-1; + while (i >= 0 && i < num && uid == uidList[i]) { + i += step; + } + return i - step; +} + +static int32_t binarySearchForStartRowIndex(uint64_t* uidList, int32_t num, uint64_t uid, int32_t backward) { + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + // find the first position which is bigger than the key + if ((uid > uidList[lastPos]) || (uid < uidList[firstPos])) { + return -1; + } + + while (1) { + if (uid == uidList[firstPos]) { + return findEarliestRow(firstPos, uid, uidList, num, backward); + } + + if (uid > uidList[lastPos] || uid < uidList[firstPos]) { + return -1; + } + + int32_t numOfRows = lastPos - firstPos + 1; + int32_t midPos = (numOfRows >> 1u) + firstPos; + + if (uid < uidList[midPos]) { + lastPos = midPos - 1; + } else if (uid > uidList[midPos]) { + firstPos = midPos + 1; + } else { + return findEarliestRow(midPos, uid, uidList, num, backward); + } + } +} + +int32_t tLDataIterOpen(struct SLDataIter **pIter, SDataFReader *pReader, int32_t iStt, int8_t backward, uint64_t suid, + uint64_t uid, STimeWindow *pTimeWindow, SVersionRange *pRange, SSttBlockLoadInfo* pBlockLoadInfo) { + int32_t code = 0; + *pIter = taosMemoryCalloc(1, sizeof(SLDataIter)); + if (*pIter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + (*pIter)->uid = uid; + (*pIter)->pReader = pReader; + (*pIter)->iStt = iStt; + (*pIter)->backward = backward; + (*pIter)->verRange = *pRange; + (*pIter)->timeWindow = *pTimeWindow; + + (*pIter)->pBlockLoadInfo = pBlockLoadInfo; + if (taosArrayGetSize(pBlockLoadInfo->aSttBlk) == 0) { + code = tsdbReadSttBlk(pReader, iStt, pBlockLoadInfo->aSttBlk); + if (code) { + goto _exit; + } else { + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + SArray* pTmp = taosArrayInit(size, sizeof(SSttBlk)); + for(int32_t i = 0; i < size; ++i) { + SSttBlk* p = taosArrayGet(pBlockLoadInfo->aSttBlk, i); + if (p->suid == suid) { + taosArrayPush(pTmp, p); + } + } + + taosArrayDestroy(pBlockLoadInfo->aSttBlk); + pBlockLoadInfo->aSttBlk = pTmp; + } + } + + size_t size = taosArrayGetSize(pBlockLoadInfo->aSttBlk); + + // find the start block + (*pIter)->iSttBlk = binarySearchForStartBlock(pBlockLoadInfo->aSttBlk->pData, size, uid, backward); + if ((*pIter)->iSttBlk != -1) { + (*pIter)->pSttBlk = taosArrayGet(pBlockLoadInfo->aSttBlk, (*pIter)->iSttBlk); + (*pIter)->iRow = ((*pIter)->backward) ? (*pIter)->pSttBlk->nRow : -1; + } + +_exit: + return code; +} + +void tLDataIterClose(SLDataIter *pIter) { + taosMemoryFree(pIter); +} + +void tLDataIterNextBlock(SLDataIter *pIter) { + int32_t step = pIter->backward ? -1 : 1; + pIter->iSttBlk += step; + + int32_t index = -1; + size_t size = pIter->pBlockLoadInfo->aSttBlk->size; + for (int32_t i = pIter->iSttBlk; i < size && i >= 0; i += step) { + SSttBlk *p = taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, i); + if ((!pIter->backward) && p->minUid > pIter->uid) { + break; + } + + if (pIter->backward && p->maxUid < pIter->uid) { + break; + } + + // check uid firstly + if (p->minUid <= pIter->uid && p->maxUid >= pIter->uid) { + if ((!pIter->backward) && p->minKey > pIter->timeWindow.ekey) { + break; + } + + if (pIter->backward && p->maxKey < pIter->timeWindow.skey) { + break; + } + + // check time range secondly + if (p->minKey <= pIter->timeWindow.ekey && p->maxKey >= pIter->timeWindow.skey) { + if ((!pIter->backward) && p->minVer > pIter->verRange.maxVer) { + break; + } + + if (pIter->backward && p->maxVer < pIter->verRange.minVer) { + break; + } + + if (p->minVer <= pIter->verRange.maxVer && p->maxVer >= pIter->verRange.minVer) { + index = i; + break; + } + } + } + } + + pIter->pSttBlk = NULL; + if (index != -1) { + pIter->pSttBlk = (SSttBlk *)taosArrayGet(pIter->pBlockLoadInfo->aSttBlk, pIter->iSttBlk); + } +} + +static void findNextValidRow(SLDataIter *pIter, const char* idStr) { + int32_t step = pIter->backward ? -1 : 1; + + bool hasVal = false; + int32_t i = pIter->iRow; + + SBlockData *pBlockData = loadLastBlock(pIter, idStr); + + // mostly we only need to find the start position for a given table + if ((((i == 0) && (!pIter->backward)) || (i == pBlockData->nRow - 1 && pIter->backward)) && pBlockData->aUid != NULL) { + i = binarySearchForStartRowIndex((uint64_t*)pBlockData->aUid, pBlockData->nRow, pIter->uid, pIter->backward); + if (i == -1) { + pIter->iRow = -1; + return; + } + } + + for (; i < pBlockData->nRow && i >= 0; i += step) { + if (pBlockData->aUid != NULL) { + if (!pIter->backward) { + /*if (pBlockData->aUid[i] < pIter->uid) { + continue; + } else */if (pBlockData->aUid[i] > pIter->uid) { + break; + } + } else { + /*if (pBlockData->aUid[i] > pIter->uid) { + continue; + } else */if (pBlockData->aUid[i] < pIter->uid) { + break; + } + } + } + + int64_t ts = pBlockData->aTSKEY[i]; + if (!pIter->backward) { // asc + if (ts > pIter->timeWindow.ekey) { // no more data + break; + } else if (ts < pIter->timeWindow.skey) { + continue; + } + } else { + if (ts < pIter->timeWindow.skey) { + break; + } else if (ts > pIter->timeWindow.ekey) { + continue; + } + } + + int64_t ver = pBlockData->aVersion[i]; + if (ver < pIter->verRange.minVer) { + continue; + } + + // todo opt handle desc case + if (ver > pIter->verRange.maxVer) { + continue; + } + + hasVal = true; + break; + } + + pIter->iRow = (hasVal) ? i : -1; +} + +bool tLDataIterNextRow(SLDataIter *pIter, const char* idStr) { + int32_t code = 0; + int32_t step = pIter->backward ? -1 : 1; + + // no qualified last file block in current file, no need to fetch row + if (pIter->pSttBlk == NULL) { + return false; + } + + int32_t iBlockL = pIter->iSttBlk; + SBlockData *pBlockData = loadLastBlock(pIter, idStr); + pIter->iRow += step; + + while (1) { + findNextValidRow(pIter, idStr); + + if (pIter->iRow >= pBlockData->nRow || pIter->iRow < 0) { + tLDataIterNextBlock(pIter); + if (pIter->pSttBlk == NULL) { // no more data + goto _exit; + } + } else { + break; + } + + if (iBlockL != pIter->iSttBlk) { + pBlockData = loadLastBlock(pIter, idStr); + pIter->iRow += step; + } + } + + pIter->rInfo.suid = pBlockData->suid; + pIter->rInfo.uid = pBlockData->uid; + pIter->rInfo.row = tsdbRowFromBlockData(pBlockData, pIter->iRow); + +_exit: + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + } + + return (code == TSDB_CODE_SUCCESS) && (pIter->pSttBlk != NULL); +} + +SRowInfo *tLDataIterGet(SLDataIter *pIter) { return &pIter->rInfo; } + +// SMergeTree ================================================= +static FORCE_INLINE int32_t tLDataIterCmprFn(const void *p1, const void *p2) { + SLDataIter *pIter1 = (SLDataIter *)(((uint8_t *)p1) - sizeof(SRBTreeNode)); + SLDataIter *pIter2 = (SLDataIter *)(((uint8_t *)p2) - sizeof(SRBTreeNode)); + + TSDBKEY key1 = TSDBROW_KEY(&pIter1->rInfo.row); + TSDBKEY key2 = TSDBROW_KEY(&pIter2->rInfo.row); + + if (key1.ts < key2.ts) { + return -1; + } else if (key1.ts > key2.ts) { + return 1; + } else { + if (key1.version < key2.version) { + return -1; + } else if (key1.version > key2.version) { + return 1; + } else { + return 0; + } + } +} + +int32_t tMergeTreeOpen(SMergeTree *pMTree, int8_t backward, SDataFReader *pFReader, uint64_t suid, uint64_t uid, + STimeWindow *pTimeWindow, SVersionRange *pVerRange, void* pBlockLoadInfo, const char* idStr) { + pMTree->backward = backward; + pMTree->pIter = NULL; + pMTree->pIterList = taosArrayInit(4, POINTER_BYTES); + if (pMTree->pIterList == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + pMTree->idStr = idStr; + + tRBTreeCreate(&pMTree->rbt, tLDataIterCmprFn); + int32_t code = TSDB_CODE_SUCCESS; + + SSttBlockLoadInfo* pLoadInfo = NULL; + if (pBlockLoadInfo == NULL) { + if (pMTree->pLoadInfo == NULL) { + pMTree->destroyLoadInfo = true; + pMTree->pLoadInfo = tCreateLastBlockLoadInfo(); + } + + pLoadInfo = pMTree->pLoadInfo; + } else { + pLoadInfo = pBlockLoadInfo; + } + + for (int32_t i = 0; i < pFReader->pSet->nSttF; ++i) { // open all last file + struct SLDataIter* pIter = NULL; + code = tLDataIterOpen(&pIter, pFReader, i, pMTree->backward, suid, uid, pTimeWindow, pVerRange, &pLoadInfo[i]); + if (code != TSDB_CODE_SUCCESS) { + goto _end; + } + + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); + if (hasVal) { + taosArrayPush(pMTree->pIterList, &pIter); + tMergeTreeAddIter(pMTree, pIter); + } else { + tLDataIterClose(pIter); + } + } + + return code; + +_end: + tMergeTreeClose(pMTree); + return code; +} + +void tMergeTreeAddIter(SMergeTree *pMTree, SLDataIter *pIter) { tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pIter); } + +bool tMergeTreeNext(SMergeTree *pMTree) { + int32_t code = TSDB_CODE_SUCCESS; + if (pMTree->pIter) { + SLDataIter *pIter = pMTree->pIter; + + bool hasVal = tLDataIterNextRow(pIter, pMTree->idStr); + if (!hasVal) { + pMTree->pIter = NULL; + } + + // compare with min in RB Tree + pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); + if (pMTree->pIter && pIter) { + int32_t c = pMTree->rbt.cmprFn(RBTREE_NODE_PAYLOAD(&pMTree->pIter->node), RBTREE_NODE_PAYLOAD(&pIter->node)); + if (c > 0) { + tRBTreePut(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); + pMTree->pIter = NULL; + } else { + ASSERT(c); + } + } + } + + if (pMTree->pIter == NULL) { + pMTree->pIter = (SLDataIter *)tRBTreeMin(&pMTree->rbt); + if (pMTree->pIter) { + tRBTreeDrop(&pMTree->rbt, (SRBTreeNode *)pMTree->pIter); + } + } + + return pMTree->pIter != NULL; +} + +TSDBROW tMergeTreeGetRow(SMergeTree *pMTree) { return pMTree->pIter->rInfo.row; } + +void tMergeTreeClose(SMergeTree *pMTree) { + size_t size = taosArrayGetSize(pMTree->pIterList); + for (int32_t i = 0; i < size; ++i) { + SLDataIter *pIter = taosArrayGetP(pMTree->pIterList, i); + tLDataIterClose(pIter); + } + + pMTree->pIterList = taosArrayDestroy(pMTree->pIterList); + pMTree->pIter = NULL; + + if (pMTree->destroyLoadInfo) { + pMTree->pLoadInfo = destroyLastBlockLoadInfo(pMTree->pLoadInfo); + pMTree->destroyLoadInfo = false; + } +} diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index a92e8189a1fed7a23be79ded705ad852b74b3a8c..e94cdfde37fc9f6ff3f0006947e39e48c5cd9065 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,9 +16,7 @@ #include "osDef.h" #include "tsdb.h" -#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define ALL_ROWS_CHECKED_INDEX (INT16_MIN) -#define INITIAL_ROW_INDEX_VAL (-1) +#define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) typedef enum { EXTERNAL_ROWS_PREV = 0x1, @@ -34,21 +32,20 @@ typedef struct { typedef struct { int32_t numOfBlocks; - int32_t numOfLastBlocks; + int32_t numOfLastFiles; } SBlockNumber; typedef struct STableBlockScanInfo { uint64_t uid; TSKEY lastKey; - SMapData mapData; // block info (compressed) - SArray* pBlockList; // block data index list - SIterInfo iter; // mem buffer skip list iterator - SIterInfo iiter; // imem buffer skip list iterator - SArray* delSkyline; // delete info for this table - int32_t fileDelIndex; // file block delete index - int32_t lastBlockDelIndex;// delete index for last block - bool iterInit; // whether to initialize the in-memory skip list iterator or not - int16_t indexInBlockL;// row position in last block + SMapData mapData; // block info (compressed) + SArray* pBlockList; // block data index list + SIterInfo iter; // mem buffer skip list iterator + SIterInfo iiter; // imem buffer skip list iterator + SArray* delSkyline; // delete info for this table + int32_t fileDelIndex; // file block delete index + int32_t lastBlockDelIndex; // delete index for last block + bool iterInit; // whether to initialize the in-memory skip list iterator or not } STableBlockScanInfo; typedef struct SBlockOrderWrapper { @@ -73,6 +70,8 @@ typedef struct SIOCostSummary { double smaLoadTime; int64_t lastBlockLoad; double lastBlockLoadTime; + int64_t composedBlocks; + double buildComposedBlockTime; } SIOCostSummary; typedef struct SBlockLoadSuppInfo { @@ -83,28 +82,21 @@ typedef struct SBlockLoadSuppInfo { char** buildBuf; // build string tmp buffer, todo remove it later after all string format being updated. } SBlockLoadSuppInfo; -typedef struct SVersionRange { - uint64_t minVer; - uint64_t maxVer; -} SVersionRange; - typedef struct SLastBlockReader { - SArray* pBlockL; - int32_t currentBlockIndex; - SBlockData lastBlockData; - STimeWindow window; - SVersionRange verRange; - int32_t order; - uint64_t uid; - int16_t* rowIndex; // row index ptr, usually from the STableBlockScanInfo->indexInBlockL + STimeWindow window; + SVersionRange verRange; + int32_t order; + uint64_t uid; + SMergeTree mergeTree; + SSttBlockLoadInfo* pInfo; } SLastBlockReader; typedef struct SFilesetIter { - int32_t numOfFiles; // number of total files - int32_t index; // current accessed index in the list - SArray* pFileList; // data file list + int32_t numOfFiles; // number of total files + int32_t index; // current accessed index in the list + SArray* pFileList; // data file list int32_t order; - SLastBlockReader* pLastBlockReader; // last file block reader + SLastBlockReader* pLastBlockReader; // last file block reader } SFilesetIter; typedef struct SFileDataBlockInfo { @@ -116,9 +108,9 @@ typedef struct SFileDataBlockInfo { typedef struct SDataBlockIter { int32_t numOfBlocks; int32_t index; - SArray* blockList; // SArray + SArray* blockList; // SArray int32_t order; - SBlock block; // current SBlock data + SDataBlk block; // current SDataBlk data SHashObj* pTableMap; } SDataBlockIter; @@ -130,8 +122,8 @@ typedef struct SFileBlockDumpInfo { } SFileBlockDumpInfo; typedef struct SUidOrderCheckInfo { - uint64_t* tableUidList; // access table uid list in uid ascending order list - int32_t currentIndex; // index in table uid list + uint64_t* tableUidList; // access table uid list in uid ascending order list + int32_t currentIndex; // index in table uid list } SUidOrderCheckInfo; typedef struct SReaderStatus { @@ -139,9 +131,9 @@ typedef struct SReaderStatus { bool composedDataBlock; // the returned data block is a composed block or not SHashObj* pTableMap; // SHash STableBlockScanInfo* pTableIter; // table iterator used in building in-memory buffer data blocks. - SUidOrderCheckInfo uidCheckInfo; // check all table in uid order + SUidOrderCheckInfo uidCheckInfo; // check all table in uid order SFileBlockDumpInfo fBlockDumpInfo; - SDFileSet* pCurrentFileset; // current opened file set + SDFileSet* pCurrentFileset; // current opened file set SBlockData fileBlockData; SFilesetIter fileIter; SDataBlockIter blockIter; @@ -175,29 +167,34 @@ static int buildDataBlockFromBufImpl(STableBlockScanInfo* pBlockScanInfo, i static TSDBROW* getValidMemRow(SIterInfo* pIter, const SArray* pDelList, STsdbReader* pReader); static int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pScanInfo, STsdbReader* pReader, SRowMerger* pMerger); -static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger); +static int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger); static int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDelList, SRowMerger* pMerger, STsdbReader* pReader); static int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* pTSRow, uint64_t uid); static int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, - int32_t rowIndex); + int32_t rowIndex); static void setComposedBlockFlag(STsdbReader* pReader, bool composed); static bool hasBeenDropped(const SArray* pDelList, int32_t* index, TSDBKEY* pKey, int32_t order); -static void doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader, bool* freeTSRow); -static void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow); -static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader); +static int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, + STSRow** pTSRow, STsdbReader* pReader, bool* freeTSRow); +static int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, + STsdbReader* pReader, STSRow** pTSRow); +static int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, + STsdbReader* pReader); static int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, STbData* piMemTbData); static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* retentions, const char* idstr, int8_t* pLevel); static SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_t level); -static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); -static int32_t doBuildDataBlock(STsdbReader* pReader); +static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader); +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader); +static int32_t doBuildDataBlock(STsdbReader* pReader); +static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader); + +static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } static int32_t setColumnIdSlotList(STsdbReader* pReader, SSDataBlock* pBlock) { SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; @@ -234,15 +231,13 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK } for (int32_t j = 0; j < numOfTables; ++j) { - STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL}; + STableBlockScanInfo info = {.lastKey = 0, .uid = idList[j].uid}; if (ASCENDING_TRAVERSE(pTsdbReader->order)) { - if (info.lastKey == INT64_MIN || info.lastKey < pTsdbReader->window.skey) { - info.lastKey = pTsdbReader->window.skey; - } - - ASSERT(info.lastKey >= pTsdbReader->window.skey && info.lastKey <= pTsdbReader->window.ekey); + int64_t skey = pTsdbReader->window.skey; + info.lastKey = (skey > INT64_MIN) ? (skey - 1) : skey; } else { - info.lastKey = pTsdbReader->window.skey; + int64_t ekey = pTsdbReader->window.ekey; + info.lastKey = (ekey < INT64_MAX) ? (ekey + 1) : ekey; } taosHashPut(pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); @@ -256,7 +251,7 @@ static SHashObj* createDataBlockScanInfo(STsdbReader* pTsdbReader, const STableK return pTableMap; } -static void resetDataBlockScanInfo(SHashObj* pTableMap) { +static void resetDataBlockScanInfo(SHashObj* pTableMap, int64_t ts) { STableBlockScanInfo* p = NULL; while ((p = taosHashIterate(pTableMap, p)) != NULL) { @@ -266,9 +261,8 @@ static void resetDataBlockScanInfo(SHashObj* pTableMap) { p->iter.iter = tsdbTbDataIterDestroy(p->iter.iter); } - p->fileDelIndex = -1; - p->delSkyline = taosArrayDestroy(p->delSkyline); - p->lastBlockDelIndex = INITIAL_ROW_INDEX_VAL; + p->delSkyline = taosArrayDestroy(p->delSkyline); + p->lastKey = ts; } } @@ -330,7 +324,7 @@ static void limitOutputBufferSize(const SQueryTableDataCond* pCond, int32_t* cap } // init file iterator -static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader/*int32_t order, const char* idstr*/) { +static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdbReader* pReader) { size_t numOfFileset = taosArrayGetSize(aDFileSet); pIter->index = ASCENDING_TRAVERSE(pReader->order) ? -1 : numOfFileset; @@ -345,17 +339,21 @@ static int32_t initFilesetIterator(SFilesetIter* pIter, SArray* aDFileSet, STsdb tsdbError("failed to prepare the last block iterator, code:%d %s", tstrerror(code), pReader->idStr); return code; } + } - SLastBlockReader* pLReader = pIter->pLastBlockReader; - pLReader->pBlockL = taosArrayInit(4, sizeof(SBlockL)); - pLReader->order = pReader->order; - pLReader->window = pReader->window; - pLReader->verRange = pReader->verRange; - pLReader->currentBlockIndex = -1; + SLastBlockReader* pLReader = pIter->pLastBlockReader; + pLReader->order = pReader->order; + pLReader->window = pReader->window; + pLReader->verRange = pReader->verRange; - int32_t code = tBlockDataCreate(&pLReader->lastBlockData); - if (code != TSDB_CODE_SUCCESS) { - return code; + pLReader->uid = 0; + tMergeTreeClose(&pLReader->mergeTree); + + if (pLReader->pInfo == NULL) { + pLReader->pInfo = tCreateLastBlockLoadInfo(); + if (pLReader->pInfo == NULL) { + tsdbDebug("init fileset iterator failed, code:%s %s", tstrerror(terrno), pReader->idStr); + return terrno; } } @@ -372,6 +370,13 @@ static bool filesetIteratorNext(SFilesetIter* pIter, STsdbReader* pReader) { return false; } + SIOCostSummary* pSum = &pReader->cost; + getLastBlockLoadInfo(pIter->pLastBlockReader->pInfo, &pSum->lastBlockLoad, &pReader->cost.lastBlockLoadTime); + + pIter->pLastBlockReader->uid = 0; + tMergeTreeClose(&pIter->pLastBlockReader->mergeTree); + resetLastBlockLoadInfo(pIter->pLastBlockReader->pInfo); + // check file the time range of coverage STimeWindow win = {0}; @@ -580,14 +585,12 @@ static void cleanupTableScanInfo(SHashObj* pTableMap) { } // reset the index in last block when handing a new file - px->indexInBlockL = INITIAL_ROW_INDEX_VAL; tMapDataClear(&px->mapData); taosArrayClear(px->pBlockList); } } -static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* pLastBlockIndex, - SBlockNumber * pBlockNum, SArray* pQualifiedLastBlock) { +static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SBlockNumber* pBlockNum) { int32_t numOfQTable = 0; size_t sizeInDisk = 0; size_t numOfTables = taosArrayGetSize(pIndexList); @@ -601,12 +604,12 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* STableBlockScanInfo* pScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockIdx->uid, sizeof(int64_t)); tMapDataReset(&pScanInfo->mapData); - tsdbReadBlock(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); + tsdbReadDataBlk(pReader->pFileReader, pBlockIdx, &pScanInfo->mapData); sizeInDisk += pScanInfo->mapData.nData; for (int32_t j = 0; j < pScanInfo->mapData.nItem; ++j) { - SBlock block = {0}; - tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetBlock); + SDataBlk block = {0}; + tMapDataGetItemByIdx(&pScanInfo->mapData, j, &block, tGetDataBlk); // 1. time range check if (block.minKey.ts > pReader->window.ekey || block.maxKey.ts < pReader->window.skey) { @@ -632,36 +635,14 @@ static int32_t doLoadFileBlock(STsdbReader* pReader, SArray* pIndexList, SArray* } } - size_t numOfLast = taosArrayGetSize(pLastBlockIndex); - for(int32_t i = 0; i < numOfLast; ++i) { - SBlockL* pLastBlock = taosArrayGet(pLastBlockIndex, i); - if (pLastBlock->suid != pReader->suid) { - continue; - } - - { - // 1. time range check - if (pLastBlock->minKey > pReader->window.ekey || pLastBlock->maxKey < pReader->window.skey) { - continue; - } - - // 2. version range check - if (pLastBlock->minVer > pReader->verRange.maxVer || pLastBlock->maxVer < pReader->verRange.minVer) { - continue; - } - - pBlockNum->numOfLastBlocks += 1; - taosArrayPush(pQualifiedLastBlock, pLastBlock); - } - } - - int32_t total = pBlockNum->numOfLastBlocks + pBlockNum->numOfBlocks; + pBlockNum->numOfLastFiles = pReader->pFileReader->pSet->nSttF; + int32_t total = pBlockNum->numOfLastFiles + pBlockNum->numOfBlocks; double el = (taosGetTimestampUs() - st) / 1000.0; tsdbDebug( - "load block of %d tables completed, blocks:%d in %d tables, lastBlock:%d, block-info-size:%.2f Kb, elapsed " + "load block of %d tables completed, blocks:%d in %d tables, last-files:%d, block-info-size:%.2f Kb, elapsed " "time:%.2f ms %s", - numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastBlocks, sizeInDisk / 1000.0, el, + numOfTables, pBlockNum->numOfBlocks, numOfQTable, pBlockNum->numOfLastFiles, sizeInDisk / 1000.0, el, pReader->idStr); pReader->cost.numOfBlocks += total; @@ -702,15 +683,156 @@ static SFileDataBlockInfo* getCurrentBlockInfo(SDataBlockIter* pBlockIter) { return pBlockInfo; } -static SBlock* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } +static SDataBlk* getCurrentBlock(SDataBlockIter* pBlockIter) { return &pBlockIter->block; } + +int32_t binarySearchForTs(char* pValue, int num, TSKEY key, int order) { + int32_t midPos = -1; + int32_t numOfRows; + + ASSERT(order == TSDB_ORDER_ASC || order == TSDB_ORDER_DESC); + + TSKEY* keyList = (TSKEY*)pValue; + int32_t firstPos = 0; + int32_t lastPos = num - 1; + + if (order == TSDB_ORDER_DESC) { + // find the first position which is smaller than the key + while (1) { + if (key >= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key < keyList[lastPos]) { + lastPos += 1; + if (lastPos >= num) { + return -1; + } else { + return lastPos; + } + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1) + firstPos; + + if (key < keyList[midPos]) { + firstPos = midPos + 1; + } else if (key > keyList[midPos]) { + lastPos = midPos - 1; + } else { + break; + } + } + + } else { + // find the first position which is bigger than the key + while (1) { + if (key <= keyList[firstPos]) return firstPos; + if (key == keyList[lastPos]) return lastPos; + + if (key > keyList[lastPos]) { + lastPos = lastPos + 1; + if (lastPos >= num) + return -1; + else + return lastPos; + } + + numOfRows = lastPos - firstPos + 1; + midPos = (numOfRows >> 1u) + firstPos; + + if (key < keyList[midPos]) { + lastPos = midPos - 1; + } else if (key > keyList[midPos]) { + firstPos = midPos + 1; + } else { + break; + } + } + } + + return midPos; +} + +static int doBinarySearchKey(TSKEY* keyList, int num, int pos, TSKEY key, int order) { + // start end position + int s, e; + s = pos; + + // check + assert(pos >=0 && pos < num); + assert(num > 0); + + if (order == TSDB_ORDER_ASC) { + // find the first position which is smaller than the key + e = num - 1; + if (key < keyList[pos]) + return -1; + while (1) { + // check can return + if (key >= keyList[e]) + return e; + if (key <= keyList[s]) + return s; + if (e - s <= 1) + return s; + + // change start or end position + int mid = s + (e - s + 1)/2; + if (keyList[mid] > key) + e = mid; + else if(keyList[mid] < key) + s = mid; + else + return mid; + } + } else { // DESC + // find the first position which is bigger than the key + e = 0; + if (key > keyList[pos]) + return -1; + while (1) { + // check can return + if (key <= keyList[e]) + return e; + if (key >= keyList[s]) + return s; + if (s - e <= 1) + return s; + + // change start or end position + int mid = s - (s - e + 1)/2; + if (keyList[mid] < key) + e = mid; + else if(keyList[mid] > key) + s = mid; + else + return mid; + } + } +} + +int32_t getEndPosInDataBlock(STsdbReader* pReader, SBlockData* pBlockData, SDataBlk* pBlock, int32_t pos) { + // NOTE: reverse the order to find the end position in data block + int32_t endPos = -1; + bool asc = ASCENDING_TRAVERSE(pReader->order); + + if (asc && pReader->window.ekey >= pBlock->maxKey.ts) { + endPos = pBlock->nRow - 1; + } else if (!asc && pReader->window.skey <= pBlock->minKey.ts) { + endPos = 0; + } else { + endPos = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.ekey, pReader->order); + } + + return endPos; +} static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo) { SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; SBlockData* pBlockData = &pStatus->fileBlockData; - SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(pBlockIter); - SBlock* pBlock = getCurrentBlock(pBlockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); SSDataBlock* pResBlock = pReader->pResBlock; int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); @@ -722,23 +844,42 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn bool asc = ASCENDING_TRAVERSE(pReader->order); int32_t step = asc ? 1 : -1; - int32_t rowIndex = 0; - int32_t remain = asc ? (pBlockData->nRow - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex + 1); - - int32_t endIndex = 0; - if (remain <= pReader->capacity) { - endIndex = pBlockData->nRow; + if (asc && pReader->window.skey <= pBlock->minKey.ts) { + pDumpInfo->rowIndex = 0; + } else if (!asc && pReader->window.ekey >= pBlock->maxKey.ts) { + pDumpInfo->rowIndex = pBlock->nRow - 1; } else { - endIndex = pDumpInfo->rowIndex + step * pReader->capacity; + int32_t pos = asc? pBlock->nRow-1:0; + int32_t order = (pReader->order == TSDB_ORDER_ASC)? TSDB_ORDER_DESC:TSDB_ORDER_ASC; + pDumpInfo->rowIndex = doBinarySearchKey(pBlockData->aTSKEY, pBlock->nRow, pos, pReader->window.skey, order); + } + + // time window check + int32_t endIndex = getEndPosInDataBlock(pReader, pBlockData, pBlock, pDumpInfo->rowIndex); + if (endIndex == -1) { + setBlockAllDumped(pDumpInfo, pReader->window.ekey, pReader->order); + return TSDB_CODE_SUCCESS; + } + + endIndex += step; + int32_t remain = asc ? (endIndex - pDumpInfo->rowIndex) : (pDumpInfo->rowIndex - endIndex); + if (remain > pReader->capacity) { // output buffer check remain = pReader->capacity; } - int32_t i = 0; + int32_t rowIndex = 0; + + int32_t i = 0; SColumnInfoData* pColData = taosArrayGet(pResBlock->pDataBlock, i); if (pColData->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) { - colDataAppend(pColData, rowIndex++, (const char*)&pBlockData->aTSKEY[j], false); + if (asc) { + memcpy(pColData->pData, &pBlockData->aTSKEY[pDumpInfo->rowIndex], remain * sizeof(int64_t)); + } else { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { + colDataAppendInt64(pColData, rowIndex++, &pBlockData->aTSKEY[j]); + } } + i += 1; } @@ -752,13 +893,32 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn if (pData->cid < pColData->info.colId) { colIndex += 1; } else if (pData->cid == pColData->info.colId) { - for (int32_t j = pDumpInfo->rowIndex; j < endIndex && j >= 0; j += step) { - tColDataGetValue(pData, j, &cv); - doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + if (pData->flag == HAS_NONE || pData->flag == HAS_NULL) { + colDataAppendNNULL(pColData, 0, remain); + } else { + if (IS_NUMERIC_TYPE(pColData->info.type) && asc) { + uint8_t* p = pData->pData + tDataTypes[pData->type].bytes * pDumpInfo->rowIndex; + memcpy(pColData->pData, p, remain * tDataTypes[pData->type].bytes); + + // null value exists, check one-by-one + if (pData->flag != HAS_VALUE) { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step, rowIndex++) { + uint8_t v = tColDataGetBitValue(pData, j); + if (v == 0 || v == 1) { + colDataSetNull_f(pColData->nullbitmap, rowIndex); + } + } + } + } else { + for (int32_t j = pDumpInfo->rowIndex; rowIndex < remain; j += step) { + tColDataGetValue(pData, j, &cv); + doCopyColVal(pColData, rowIndex++, i, &cv, pSupInfo); + } + } } + colIndex += 1; i += 1; - ASSERT(rowIndex == remain); } else { // the specified column does not exist in file block, fill with null data colDataAppendNNULL(pColData, 0, remain); i += 1; @@ -774,7 +934,13 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn pResBlock->info.rows = remain; pDumpInfo->rowIndex += step * remain; - setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); + if (pDumpInfo->rowIndex >= 0 && pDumpInfo->rowIndex < pBlock->nRow) { + int64_t ts = pBlockData->aTSKEY[pDumpInfo->rowIndex]; + setBlockAllDumped(pDumpInfo, ts, pReader->order); + } else { + int64_t k = asc? pBlock->maxKey.ts:pBlock->minKey.ts; + setBlockAllDumped(pDumpInfo, k, pReader->order); + } double elapsedTime = (taosGetTimestampUs() - st) / 1000.0; pReader->cost.blockLoadTime += elapsedTime; @@ -782,7 +948,7 @@ static int32_t copyBlockDataToSDataBlock(STsdbReader* pReader, STableBlockScanIn int32_t unDumpedRows = asc ? pBlock->nRow - pDumpInfo->rowIndex : pDumpInfo->rowIndex + 1; tsdbDebug("%p copy file block to sdatablock, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, remain:%d, minVer:%" PRId64 ", maxVer:%" PRId64 ", elapsed time:%.2f ms, %s", - pReader, pBlockIter->index, pFBlock->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, + pReader, pBlockIter->index, pBlockInfo->tbBlockIdx, pBlock->minKey.ts, pBlock->maxKey.ts, remain, unDumpedRows, pBlock->minVer, pBlock->maxVer, elapsedTime, pReader->idStr); return TSDB_CODE_SUCCESS; @@ -795,8 +961,8 @@ static int32_t doLoadFileBlockData(STsdbReader* pReader, SDataBlockIter* pBlockI SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; ASSERT(pBlockInfo != NULL); - SBlock* pBlock = getCurrentBlock(pBlockIter); - int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); + int32_t code = tsdbReadDataBlock(pReader->pFileReader, pBlock, pBlockData); if (code != TSDB_CODE_SUCCESS) { tsdbError("%p error occurs in loading file block, global index:%d, table index:%d, brange:%" PRId64 "-%" PRId64 ", rows:%d, code:%s %s", @@ -872,8 +1038,8 @@ static int32_t doSetCurrentBlock(SDataBlockIter* pBlockIter) { SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(pBlockIter); if (pBlockInfo != NULL) { STableBlockScanInfo* pScanInfo = taosHashGet(pBlockIter->pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); - tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetBlock); + int32_t* mapDataIndex = taosArrayGet(pScanInfo->pBlockList, pBlockInfo->tbBlockIdx); + tMapDataGetItemByIdx(&pScanInfo->mapData, *mapDataIndex, &pBlockIter->block, tGetDataBlk); } #if 0 @@ -924,12 +1090,12 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } sup.pDataBlockInfo[sup.numOfTables] = (SBlockOrderWrapper*)buf; - SBlock block = {0}; + SDataBlk block = {0}; for (int32_t k = 0; k < num; ++k) { SBlockOrderWrapper wrapper = {0}; int32_t* mapDataIndex = taosArrayGet(pTableScanInfo->pBlockList, k); - tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetBlock); + tMapDataGetItemByIdx(&pTableScanInfo->mapData, *mapDataIndex, &block, tGetDataBlk); wrapper.uid = pTableScanInfo->uid; wrapper.offset = block.aSubBlock[0].offset; @@ -990,8 +1156,8 @@ static int32_t initBlockIterator(STsdbReader* pReader, SDataBlockIter* pBlockIte } int64_t et = taosGetTimestampUs(); - tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, (et - st) / 1000.0, - pReader->idStr); + tsdbDebug("%p %d data blocks access order completed, elapsed time:%.2f ms %s", pReader, numOfBlocks, + (et - st) / 1000.0, pReader->idStr); cleanupBlockOrderSupporter(&sup); taosMemoryFree(pTree); @@ -1018,15 +1184,15 @@ static bool blockIteratorNext(SDataBlockIter* pBlockIter) { /** * This is an two rectangles overlap cases. */ -static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SBlock* pBlock) { +static int32_t dataBlockPartiallyRequired(STimeWindow* pWindow, SVersionRange* pVerRange, SDataBlk* pBlock) { return (pWindow->ekey < pBlock->maxKey.ts && pWindow->ekey >= pBlock->minKey.ts) || (pWindow->skey > pBlock->minKey.ts && pWindow->skey <= pBlock->maxKey.ts) || (pVerRange->minVer > pBlock->minVer && pVerRange->minVer <= pBlock->maxVer) || (pVerRange->maxVer < pBlock->maxVer && pVerRange->maxVer >= pBlock->minVer); } -static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, - int32_t* nextIndex, int32_t order) { +static SDataBlk* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STableBlockScanInfo* pTableBlockScanInfo, + int32_t* nextIndex, int32_t order) { bool asc = ASCENDING_TRAVERSE(order); if (asc && pFBlockInfo->tbBlockIdx >= taosArrayGetSize(pTableBlockScanInfo->pBlockList) - 1) { return NULL; @@ -1039,10 +1205,10 @@ static SBlock* getNeighborBlockOfSameTable(SFileDataBlockInfo* pFBlockInfo, STab int32_t step = asc ? 1 : -1; *nextIndex = pFBlockInfo->tbBlockIdx + step; - SBlock* pBlock = taosMemoryCalloc(1, sizeof(SBlock)); - int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); + SDataBlk* pBlock = taosMemoryCalloc(1, sizeof(SDataBlk)); + int32_t* indexInMapdata = taosArrayGet(pTableBlockScanInfo->pBlockList, *nextIndex); - tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetBlock); + tMapDataGetItemByIdx(&pTableBlockScanInfo->mapData, *indexInMapdata, pBlock, tGetDataBlk); return pBlock; } @@ -1085,7 +1251,7 @@ static int32_t setFileBlockActiveInBlockIter(SDataBlockIter* pBlockIter, int32_t return TSDB_CODE_SUCCESS; } -static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t order) { +static bool overlapWithNeighborBlock(SDataBlk* pBlock, SDataBlk* pNeighbor, int32_t order) { // it is the last block in current file, no chance to overlap with neighbor blocks. if (ASCENDING_TRAVERSE(order)) { return pBlock->maxKey.ts == pNeighbor->minKey.ts; @@ -1094,19 +1260,19 @@ static bool overlapWithNeighborBlock(SBlock* pBlock, SBlock* pNeighbor, int32_t } } -static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SBlock* pBlock) { +static bool bufferDataInFileBlockGap(int32_t order, TSDBKEY key, SDataBlk* pBlock) { bool ascScan = ASCENDING_TRAVERSE(order); return (ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts <= pBlock->minKey.ts)) || (!ascScan && (key.ts != TSKEY_INITIAL_VAL && key.ts >= pBlock->maxKey.ts)); } -static bool keyOverlapFileBlock(TSDBKEY key, SBlock* pBlock, SVersionRange* pVerRange) { +static bool keyOverlapFileBlock(TSDBKEY key, SDataBlk* pBlock, SVersionRange* pVerRange) { return (key.ts >= pBlock->minKey.ts && key.ts <= pBlock->maxKey.ts) && (pBlock->maxVer >= pVerRange->minVer) && (pBlock->minVer <= pVerRange->maxVer); } -static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock) { +static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock) { size_t num = taosArrayGetSize(pBlockScanInfo->delSkyline); for (int32_t i = pBlockScanInfo->fileDelIndex; i < num; i += 1) { @@ -1140,7 +1306,7 @@ static bool doCheckforDatablockOverlap(STableBlockScanInfo* pBlockScanInfo, cons return false; } -static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBlock* pBlock, int32_t order) { +static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SDataBlk* pBlock, int32_t order) { if (pBlockScanInfo->delSkyline == NULL) { return false; } @@ -1170,53 +1336,79 @@ static bool overlapWithDelSkyline(STableBlockScanInfo* pBlockScanInfo, const SBl } } -// 1. the version of all rows should be less than the endVersion -// 2. current block should not overlap with next neighbor block -// 3. current timestamp should not be overlap with each other -// 4. output buffer should be large enough to hold all rows in current block -// 5. delete info should not overlap with current block data -static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pFBlock, SBlock* pBlock, - STableBlockScanInfo* pScanInfo, TSDBKEY key, SLastBlockReader* pLastBlockReader) { - int32_t neighborIndex = 0; - SBlock* pNeighbor = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &neighborIndex, pReader->order); +typedef struct { + bool overlapWithNeighborBlock; + bool hasDupTs; + bool overlapWithDelInfo; + bool overlapWithLastBlock; + bool overlapWithKeyInBuf; + bool partiallyRequired; + bool moreThanCapcity; +} SDataBlockToLoadInfo; + +static void getBlockToLoadInfo(SDataBlockToLoadInfo* pInfo, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader, + STsdbReader* pReader) { + int32_t neighborIndex = 0; + SDataBlk* pNeighbor = getNeighborBlockOfSameTable(pBlockInfo, pScanInfo, &neighborIndex, pReader->order); // overlap with neighbor - bool overlapWithNeighbor = false; if (pNeighbor) { - overlapWithNeighbor = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); + pInfo->overlapWithNeighborBlock = overlapWithNeighborBlock(pBlock, pNeighbor, pReader->order); taosMemoryFree(pNeighbor); } // has duplicated ts of different version in this block - bool hasDup = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; - bool overlapWithDel = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); + pInfo->hasDupTs = (pBlock->nSubBlock == 1) ? pBlock->hasDup : true; + pInfo->overlapWithDelInfo = overlapWithDelSkyline(pScanInfo, pBlock, pReader->order); - // todo here we need to each key in the last files to identify if it is really overlapped with last block - bool overlapWithlastBlock = false; - if (taosArrayGetSize(pLastBlockReader->pBlockL) > 0 && (pLastBlockReader->currentBlockIndex != -1)) { - SBlockL *pBlockL = taosArrayGet(pLastBlockReader->pBlockL, pLastBlockReader->currentBlockIndex); - overlapWithlastBlock = !(pBlock->maxKey.ts < pBlockL->minKey || pBlock->minKey.ts > pBlockL->maxKey); + if (hasDataInLastBlock(pLastBlockReader)) { + int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); + pInfo->overlapWithLastBlock = !(pBlock->maxKey.ts < tsLast || pBlock->minKey.ts > tsLast); } - bool moreThanOutputCapacity = pBlock->nRow > pReader->capacity; - bool partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); - bool overlapWithKey = keyOverlapFileBlock(key, pBlock, &pReader->verRange); + pInfo->moreThanCapcity = pBlock->nRow > pReader->capacity; + pInfo->partiallyRequired = dataBlockPartiallyRequired(&pReader->window, &pReader->verRange, pBlock); + pInfo->overlapWithKeyInBuf = keyOverlapFileBlock(keyInBuf, pBlock, &pReader->verRange); +} + +// 1. the version of all rows should be less than the endVersion +// 2. current block should not overlap with next neighbor block +// 3. current timestamp should not be overlap with each other +// 4. output buffer should be large enough to hold all rows in current block +// 5. delete info should not overlap with current block data +// 6. current block should not contain the duplicated ts +static bool fileBlockShouldLoad(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); - bool loadDataBlock = (overlapWithNeighbor || hasDup || partiallyRequired || overlapWithKey || - moreThanOutputCapacity || overlapWithDel || overlapWithlastBlock); + bool loadDataBlock = + (info.overlapWithNeighborBlock || info.hasDupTs || info.partiallyRequired || info.overlapWithKeyInBuf || + info.moreThanCapcity || info.overlapWithDelInfo || info.overlapWithLastBlock); // log the reason why load the datablock for profile if (loadDataBlock) { tsdbDebug("%p uid:%" PRIu64 " need to load the datablock, overlapwithneighborblock:%d, hasDup:%d, partiallyRequired:%d, " "overlapWithKey:%d, greaterThanBuf:%d, overlapWithDel:%d, overlapWithlastBlock:%d, %s", - pReader, pFBlock->uid, overlapWithNeighbor, hasDup, partiallyRequired, overlapWithKey, - moreThanOutputCapacity, overlapWithDel, overlapWithlastBlock, pReader->idStr); + pReader, pBlockInfo->uid, info.overlapWithNeighborBlock, info.hasDupTs, info.partiallyRequired, + info.overlapWithKeyInBuf, info.moreThanCapcity, info.overlapWithDelInfo, info.overlapWithLastBlock, + pReader->idStr); } return loadDataBlock; } +static bool isCleanFileDataBlock(STsdbReader* pReader, SFileDataBlockInfo* pBlockInfo, SDataBlk* pBlock, + STableBlockScanInfo* pScanInfo, TSDBKEY keyInBuf, SLastBlockReader* pLastBlockReader) { + SDataBlockToLoadInfo info = {0}; + getBlockToLoadInfo(&info, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader, pReader); + bool isCleanFileBlock = !(info.overlapWithNeighborBlock || info.hasDupTs || info.overlapWithKeyInBuf || + info.overlapWithDelInfo || info.overlapWithLastBlock); + return isCleanFileBlock; +} + static int32_t buildDataBlockFromBuf(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, int64_t endKey) { if (!(pBlockScanInfo->iiter.hasVal || pBlockScanInfo->iter.hasVal)) { return TSDB_CODE_SUCCESS; @@ -1262,6 +1454,38 @@ static bool tryCopyDistinctRowFromFileBlock(STsdbReader* pReader, SBlockData* pB return false; } +static bool nextRowFromLastBlocks(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { + while (1) { + bool hasVal = tMergeTreeNext(&pLastBlockReader->mergeTree); + if (!hasVal) { + return false; + } + + TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + TSDBKEY k = TSDBROW_KEY(&row); + if (!hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { + return true; + } + } +} + +static bool tryCopyDistinctRowFromSttBlock(TSDBROW* fRow, SLastBlockReader* pLastBlockReader, + STableBlockScanInfo* pScanInfo, int64_t ts, STsdbReader* pReader) { + bool hasVal = nextRowFromLastBlocks(pLastBlockReader, pScanInfo); + if (hasVal) { + int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); + if (next1 != ts) { + doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + return true; + } + } else { + doAppendRowFromFileBlock(pReader->pResBlock, pReader, fRow->pBlockData, fRow->iRow); + return true; + } + + return false; +} + static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { @@ -1295,18 +1519,16 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; int64_t tsLast = INT64_MIN; - if ((pLastBlockReader->lastBlockData.nRow > 0) && hasDataInLastBlock(pLastBlockReader)) { + if (hasDataInLastBlock(pLastBlockReader)) { tsLast = getCurrentKeyInLastBlock(pLastBlockReader); } - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); int64_t minKey = 0; if (pReader->order == TSDB_ORDER_ASC) { - minKey = INT64_MAX; // chosen the minimum value + minKey = INT64_MAX; // chosen the minimum value if (minKey > tsLast && hasDataInLastBlock(pLastBlockReader)) { minKey = tsLast; } @@ -1336,7 +1558,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* bool init = false; // ASC: file block ---> last block -----> imem -----> mem - //DESC: mem -----> imem -----> last block -----> file block + // DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { init = true; @@ -1345,7 +1567,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1374,7 +1596,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1395,7 +1617,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* } } - tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); @@ -1406,27 +1632,55 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, bool mergeBlockData) { - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; - int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); + SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; + int64_t tsLastBlock = getCurrentKeyInLastBlock(pLastBlockReader); STSRow* pTSRow = NULL; SRowMerger merge = {0}; + TSDBROW fRow = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + + // only last block exists + if ((!mergeBlockData) || (tsLastBlock != pBlockData->aTSKEY[pDumpInfo->rowIndex])) { + if (tryCopyDistinctRowFromSttBlock(&fRow, pLastBlockReader, pBlockScanInfo, tsLastBlock, pReader)) { + return TSDB_CODE_SUCCESS; + } else { + tRowMergerInit(&merge, &fRow, pReader->pSchema); - TSDBROW fRow = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tRowMerge(&merge, &fRow1); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } - // merge with block data if ts == key - if (mergeBlockData) { - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + } + } else { // not merge block data + tRowMergerInit(&merge, &fRow, pReader->pSchema); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge); + ASSERT(mergeBlockData); + + // merge with block data if ts == key + if (tsLastBlock == pBlockData->aTSKEY[pDumpInfo->rowIndex]) { + doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + } + + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); + + taosMemoryFree(pTSRow); + tRowMergerClear(&merge); + } - taosMemoryFree(pTSRow); - tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } @@ -1436,7 +1690,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader if (pBlockData->nRow > 0) { // no last block available, only data block exists - if (pLastBlockReader->lastBlockData.nRow == 0 || (!hasDataInLastBlock(pLastBlockReader))) { + if (!hasDataInLastBlock(pLastBlockReader)) { return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } @@ -1446,7 +1700,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader ASSERT(ts >= key); if (ASCENDING_TRAVERSE(pReader->order)) { - if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist + if (key < ts) { // imem, mem are all empty, file blocks (data blocks and last block) exist return mergeRowsInFileBlocks(pBlockData, pBlockScanInfo, key, pReader); } else if (key == ts) { STSRow* pTSRow = NULL; @@ -1454,14 +1708,22 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); + + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + tRowMerge(&merge, &fRow1); + doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge); - tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; + return code; } else { ASSERT(0); return TSDB_CODE_SUCCESS; @@ -1486,7 +1748,6 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); ASSERT(pRow != NULL && piRow != NULL); - SBlockData* pLastBlockData = &pLastBlockReader->lastBlockData; int64_t tsLast = INT64_MIN; if (hasDataInLastBlock(pLastBlockReader)) { tsLast = getCurrentKeyInLastBlock(pLastBlockReader); @@ -1516,7 +1777,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* minKey = tsLast; } } else { - minKey = INT64_MIN; // let find the maximum ts value + minKey = INT64_MIN; // let find the maximum ts value if (minKey < k.ts) { minKey = k.ts; } @@ -1547,7 +1808,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1597,7 +1858,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } if (minKey == tsLast) { - TSDBROW fRow1 = tsdbRowFromBlockData(pLastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { tRowMerge(&merge, &fRow1); } else { @@ -1618,158 +1879,81 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } } - tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); - return TSDB_CODE_SUCCESS; + return code; } -#if 0 -static int32_t doMergeThreeLevelRows(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData) { - SRowMerger merge = {0}; - STSRow* pTSRow = NULL; - - SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - SArray* pDelList = pBlockScanInfo->delSkyline; - - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pDelList, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pDelList, pReader); - ASSERT(pRow != NULL && piRow != NULL); - - int64_t key = pBlockData->aTSKEY[pDumpInfo->rowIndex]; - bool freeTSRow = false; +static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { + if (pBlockScanInfo->iterInit) { + return TSDB_CODE_SUCCESS; + } - uint64_t uid = pBlockScanInfo->uid; + int32_t code = TSDB_CODE_SUCCESS; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY startKey = {0}; if (ASCENDING_TRAVERSE(pReader->order)) { - // [1&2] key <= [k.ts && ik.ts] - if (key <= k.ts && key <= ik.ts) { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - - if (ik.ts == key) { - tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - } - - if (k.ts == key) { - tRowMerge(&merge, pRow); - doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - } - - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - return TSDB_CODE_SUCCESS; - } else { // key > ik.ts || key > k.ts - ASSERT(key != ik.ts); - - // [3] ik.ts < key <= k.ts - // [4] ik.ts < k.ts <= key - if (ik.ts < k.ts) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - if (freeTSRow) { - taosMemoryFree(pTSRow); - } - return TSDB_CODE_SUCCESS; - } + startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; + } else { + startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; + } - // [5] k.ts < key <= ik.ts - // [6] k.ts < ik.ts <= key - if (k.ts < ik.ts) { - doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - if (freeTSRow) { - taosMemoryFree(pTSRow); - } - return TSDB_CODE_SUCCESS; - } + int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); - // [7] k.ts == ik.ts < key - if (k.ts == ik.ts) { - ASSERT(key > ik.ts && key > k.ts); + STbData* d = NULL; + if (pReader->pReadSnap->pMem != NULL) { + d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); + if (d != NULL) { + code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); + if (code == TSDB_CODE_SUCCESS) { + pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); - doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); - return TSDB_CODE_SUCCESS; + tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 " %s", + pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr); + } else { + tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid, + tstrerror(code), pReader->idStr); + return code; } } - } else { // descending order scan - // [1/2] k.ts >= ik.ts && k.ts >= key - if (k.ts >= ik.ts && k.ts >= key) { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - - tRowMergerInit(&merge, pRow, pSchema); - doMergeRowsInBuf(&pBlockScanInfo->iter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - - if (ik.ts == k.ts) { - tRowMerge(&merge, piRow); - doMergeRowsInBuf(&pBlockScanInfo->iiter, uid, key, pBlockScanInfo->delSkyline, &merge, pReader); - } - - if (k.ts == key) { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMerge(&merge, &fRow); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - } - - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - return TSDB_CODE_SUCCESS; - } else { - ASSERT(ik.ts != k.ts); // this case has been included in the previous if branch - - // [3] ik.ts > k.ts >= Key - // [4] ik.ts > key >= k.ts - if (ik.ts > key) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - if (freeTSRow) { - taosMemoryFree(pTSRow); - } - return TSDB_CODE_SUCCESS; - } - - // [5] key > ik.ts > k.ts - // [6] key > k.ts > ik.ts - if (key > ik.ts) { - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMergerInit(&merge, &fRow, pReader->pSchema); - - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); - taosMemoryFree(pTSRow); - return TSDB_CODE_SUCCESS; - } - - //[7] key = ik.ts > k.ts - if (key == ik.ts) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, &pTSRow, pReader, &freeTSRow); + } else { + tsdbDebug("%p uid:%" PRId64 ", no data in mem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); + } - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - tRowMerge(&merge, &fRow); - doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); - doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, uid); + STbData* di = NULL; + if (pReader->pReadSnap->pIMem != NULL) { + di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); + if (di != NULL) { + code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); + if (code == TSDB_CODE_SUCCESS) { + pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL); - taosMemoryFree(pTSRow); - return TSDB_CODE_SUCCESS; + tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 + "-%" PRId64 " %s", + pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr); + } else { + tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid, + tstrerror(code), pReader->idStr); + return code; } } + } else { + tsdbDebug("%p uid:%" PRId64 ", no data in imem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); } - ASSERT(0); - return -1; + initDelSkylineIterator(pBlockScanInfo, pReader, d, di); + + pBlockScanInfo->iterInit = true; + return TSDB_CODE_SUCCESS; } -#endif static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDumpInfo, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { @@ -1800,135 +1984,87 @@ static bool isValidFileBlockRow(SBlockData* pBlockData, SFileBlockDumpInfo* pDum return true; } -static bool outOfTimeWindow(int64_t ts, STimeWindow* pWindow) { return (ts > pWindow->ekey) || (ts < pWindow->skey); } - -static void initLastBlockReader(SLastBlockReader* pLastBlockReader, uint64_t uid, int16_t* startPos) { - pLastBlockReader->uid = uid; - pLastBlockReader->rowIndex = startPos; - - if (*startPos == -1) { - if (ASCENDING_TRAVERSE(pLastBlockReader->order)) { - // do nothing - } else { - *startPos = pLastBlockReader->lastBlockData.nRow; - } +static bool initLastBlockReader(SLastBlockReader* pLBlockReader, STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + // the last block reader has been initialized for this table. + if (pLBlockReader->uid == pScanInfo->uid) { + return true; } -} - -static void setAllRowsChecked(SLastBlockReader *pLastBlockReader) { - *pLastBlockReader->rowIndex = ALL_ROWS_CHECKED_INDEX; -} -static bool nextRowInLastBlock(SLastBlockReader *pLastBlockReader, STableBlockScanInfo* pBlockScanInfo) { - bool asc = ASCENDING_TRAVERSE(pLastBlockReader->order); - int32_t step = (asc) ? 1 : -1; - if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { - return false; + if (pLBlockReader->uid != 0) { + tMergeTreeClose(&pLBlockReader->mergeTree); } - *(pLastBlockReader->rowIndex) += step; - - SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - for(int32_t i = *(pLastBlockReader->rowIndex); i < pBlockData->nRow && i >= 0; i += step) { - if (pBlockData->aUid != NULL) { - if (asc) { - if (pBlockData->aUid[i] < pLastBlockReader->uid) { - continue; - } else if (pBlockData->aUid[i] > pLastBlockReader->uid) { - break; - } - } else { - if (pBlockData->aUid[i] > pLastBlockReader->uid) { - continue; - } else if (pBlockData->aUid[i] < pLastBlockReader->uid) { - break; - } - } - } - - int64_t ts = pBlockData->aTSKEY[i]; - if (ts < pLastBlockReader->window.skey) { - continue; - } - - int64_t ver = pBlockData->aVersion[i]; - if (ver < pLastBlockReader->verRange.minVer) { - continue; - } - - // no data any more, todo opt handle desc case - if (ts > pLastBlockReader->window.ekey) { - continue; - } - - // todo opt handle desc case - if (ver > pLastBlockReader->verRange.maxVer) { - continue; - } + initMemDataIterator(pScanInfo, pReader); + pLBlockReader->uid = pScanInfo->uid; - TSDBKEY k = {.ts = ts, .version = ver}; - if (hasBeenDropped(pBlockScanInfo->delSkyline, &pBlockScanInfo->lastBlockDelIndex, &k, pLastBlockReader->order)) { - continue; - } + int32_t step = ASCENDING_TRAVERSE(pLBlockReader->order) ? 1 : -1; + STimeWindow w = pLBlockReader->window; + if (ASCENDING_TRAVERSE(pLBlockReader->order)) { + w.skey = pScanInfo->lastKey + step; + } else { + w.ekey = pScanInfo->lastKey + step; + } - *(pLastBlockReader->rowIndex) = i; - return true; + int32_t code = + tMergeTreeOpen(&pLBlockReader->mergeTree, (pLBlockReader->order == TSDB_ORDER_DESC), pReader->pFileReader, + pReader->suid, pScanInfo->uid, &w, &pLBlockReader->verRange, pLBlockReader->pInfo, pReader->idStr); + if (code != TSDB_CODE_SUCCESS) { + return false; } - // set all data is consumed in last block - setAllRowsChecked(pLastBlockReader); - return false; + return nextRowFromLastBlocks(pLBlockReader, pScanInfo); } static int64_t getCurrentKeyInLastBlock(SLastBlockReader* pLastBlockReader) { - SBlockData* pBlockData = &pLastBlockReader->lastBlockData; - return pBlockData->aTSKEY[*pLastBlockReader->rowIndex]; + TSDBROW row = tMergeTreeGetRow(&pLastBlockReader->mergeTree); + return TSDBROW_TS(&row); } -static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { - if (*pLastBlockReader->rowIndex == ALL_ROWS_CHECKED_INDEX) { - return false; - } - - ASSERT(pLastBlockReader->lastBlockData.nRow > 0); - return true; -} +static bool hasDataInLastBlock(SLastBlockReader* pLastBlockReader) { return pLastBlockReader->mergeTree.pIter != NULL; } -int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, STsdbReader* pReader) { +int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBlockScanInfo, int64_t key, + STsdbReader* pReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - - TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - if (tryCopyDistinctRowFromFileBlock(pReader, pBlockData, key, pDumpInfo)) { return TSDB_CODE_SUCCESS; } else { + TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); + STSRow* pTSRow = NULL; SRowMerger merge = {0}; tRowMergerInit(&merge, &fRow, pReader->pSchema); doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); - tRowMergerGetRow(&merge, &pTSRow); + int32_t code = tRowMergerGetRow(&merge, &pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doAppendRowFromTSRow(pReader->pResBlock, pReader, pTSRow, pBlockScanInfo->uid); taosMemoryFree(pTSRow); tRowMergerClear(&merge); return TSDB_CODE_SUCCESS; } - - return TSDB_CODE_SUCCESS; } static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanInfo* pBlockScanInfo, SBlockData* pBlockData, SLastBlockReader* pLastBlockReader) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; - int64_t key = (pBlockData->nRow > 0)? pBlockData->aTSKEY[pDumpInfo->rowIndex]:INT64_MIN; - TSDBROW* pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); - TSDBROW* piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); - + int64_t key = (pBlockData->nRow > 0) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; if (pBlockScanInfo->iter.hasVal && pBlockScanInfo->iiter.hasVal) { return doMergeMultiLevelRows(pReader, pBlockScanInfo, pBlockData, pLastBlockReader); } else { + TSDBROW *pRow = NULL, *piRow = NULL; + if (pBlockScanInfo->iter.hasVal) { + pRow = getValidMemRow(&pBlockScanInfo->iter, pBlockScanInfo->delSkyline, pReader); + } + + if (pBlockScanInfo->iiter.hasVal) { + piRow = getValidMemRow(&pBlockScanInfo->iiter, pBlockScanInfo->delSkyline, pReader); + } + // imem + file + last block if (pBlockScanInfo->iiter.hasVal) { return doMergeBufAndFileRows(pReader, pBlockScanInfo, piRow, &pBlockScanInfo->iiter, key, pLastBlockReader); @@ -1947,24 +2083,31 @@ static int32_t buildComposedDataBlockImpl(STsdbReader* pReader, STableBlockScanI static int32_t buildComposedDataBlock(STsdbReader* pReader) { SSDataBlock* pResBlock = pReader->pResBlock; - SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SFileDataBlockInfo* pBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); + SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; + + int64_t st = taosGetTimestampUs(); STableBlockScanInfo* pBlockScanInfo = NULL; if (pBlockInfo != NULL) { pBlockScanInfo = taosHashGet(pReader->status.pTableMap, &pBlockInfo->uid, sizeof(pBlockInfo->uid)); - } else { + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pBlockScanInfo, pReader); + + // it is a clean block, load it directly + if (isCleanFileDataBlock(pReader, pBlockInfo, pBlock, pBlockScanInfo, keyInBuf, pLastBlockReader)) { + copyBlockDataToSDataBlock(pReader, pBlockScanInfo); + goto _end; + } + } else { // file blocks not exist pBlockScanInfo = pReader->status.pTableIter; } - SLastBlockReader* pLastBlockReader = pReader->status.fileIter.pLastBlockReader; SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; SBlockData* pBlockData = &pReader->status.fileBlockData; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int64_t st = taosGetTimestampUs(); - while (1) { - // todo check the validate of row in file block bool hasBlockData = false; { while (pBlockData->nRow > 0) { // find the first qualified row in data block @@ -1975,7 +2118,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { pDumpInfo->rowIndex += step; - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); if (pDumpInfo->rowIndex >= pBlock->nRow || pDumpInfo->rowIndex < 0) { setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; @@ -1994,7 +2137,7 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { // currently loaded file data block is consumed if ((pBlockData->nRow > 0) && (pDumpInfo->rowIndex >= pBlockData->nRow || pDumpInfo->rowIndex < 0)) { - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); setBlockAllDumped(pDumpInfo, pBlock->maxKey.ts, pReader->order); break; } @@ -2004,17 +2147,21 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { } } + _end: pResBlock->info.uid = pBlockScanInfo->uid; blockDataUpdateTsWindow(pResBlock, 0); setComposedBlockFlag(pReader, true); - int64_t et = taosGetTimestampUs(); + double el = (taosGetTimestampUs() - st)/1000.0; + + pReader->cost.composedBlocks += 1; + pReader->cost.buildComposedBlockTime += el; if (pResBlock->info.rows > 0) { tsdbDebug("%p uid:%" PRIu64 ", composed data block created, brange:%" PRIu64 "-%" PRIu64 " rows:%d, elapsed time:%.2f ms %s", pReader, pBlockScanInfo->uid, pResBlock->info.window.skey, pResBlock->info.window.ekey, - pResBlock->info.rows, (et - st) / 1000.0, pReader->idStr); + pResBlock->info.rows, el, pReader->idStr); } return TSDB_CODE_SUCCESS; @@ -2022,70 +2169,6 @@ static int32_t buildComposedDataBlock(STsdbReader* pReader) { void setComposedBlockFlag(STsdbReader* pReader, bool composed) { pReader->status.composedDataBlock = composed; } -static int32_t initMemDataIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader) { - if (pBlockScanInfo->iterInit) { - return TSDB_CODE_SUCCESS; - } - - int32_t code = TSDB_CODE_SUCCESS; - - TSDBKEY startKey = {0}; - if (ASCENDING_TRAVERSE(pReader->order)) { - startKey = (TSDBKEY){.ts = pReader->window.skey, .version = pReader->verRange.minVer}; - } else { - startKey = (TSDBKEY){.ts = pReader->window.ekey, .version = pReader->verRange.maxVer}; - } - - int32_t backward = (!ASCENDING_TRAVERSE(pReader->order)); - - STbData* d = NULL; - if (pReader->pReadSnap->pMem != NULL) { - d = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pMem, pReader->suid, pBlockScanInfo->uid); - if (d != NULL) { - code = tsdbTbDataIterCreate(d, &startKey, backward, &pBlockScanInfo->iter.iter); - if (code == TSDB_CODE_SUCCESS) { - pBlockScanInfo->iter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iter.iter) != NULL); - - tsdbDebug("%p uid:%" PRId64 ", check data in mem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 " %s", - pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, d->minKey, d->maxKey, pReader->idStr); - } else { - tsdbError("%p uid:%" PRId64 ", failed to create iterator for imem, code:%s, %s", pReader, pBlockScanInfo->uid, - tstrerror(code), pReader->idStr); - return code; - } - } - } else { - tsdbDebug("%p uid:%" PRId64 ", no data in mem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); - } - - STbData* di = NULL; - if (pReader->pReadSnap->pIMem != NULL) { - di = tsdbGetTbDataFromMemTable(pReader->pReadSnap->pIMem, pReader->suid, pBlockScanInfo->uid); - if (di != NULL) { - code = tsdbTbDataIterCreate(di, &startKey, backward, &pBlockScanInfo->iiter.iter); - if (code == TSDB_CODE_SUCCESS) { - pBlockScanInfo->iiter.hasVal = (tsdbTbDataIterGet(pBlockScanInfo->iiter.iter) != NULL); - - tsdbDebug("%p uid:%" PRId64 ", check data in imem from skey:%" PRId64 ", order:%d, ts range in buf:%" PRId64 - "-%" PRId64 " %s", - pReader, pBlockScanInfo->uid, startKey.ts, pReader->order, di->minKey, di->maxKey, pReader->idStr); - } else { - tsdbError("%p uid:%" PRId64 ", failed to create iterator for mem, code:%s, %s", pReader, pBlockScanInfo->uid, - tstrerror(code), pReader->idStr); - return code; - } - } - } else { - tsdbDebug("%p uid:%" PRId64 ", no data in imem, %s", pReader, pBlockScanInfo->uid, pReader->idStr); - } - - initDelSkylineIterator(pBlockScanInfo, pReader, d, di); - - pBlockScanInfo->iterInit = true; - return TSDB_CODE_SUCCESS; -} - int32_t initDelSkylineIterator(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STbData* pMemTbData, STbData* piMemTbData) { if (pBlockScanInfo->delSkyline != NULL) { @@ -2168,10 +2251,8 @@ _err: return code; } -static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { - TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; - - initMemDataIterator(pScanInfo, pReader); +TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* pReader) { + TSDBKEY key = {.ts = TSKEY_INITIAL_VAL}; TSDBROW* pRow = getValidMemRow(&pScanInfo->iter, pScanInfo->delSkyline, pReader); if (pRow != NULL) { key = TSDBROW_KEY(pRow); @@ -2191,12 +2272,10 @@ static TSDBKEY getCurrentKeyInBuf(STableBlockScanInfo* pScanInfo, STsdbReader* p static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { SReaderStatus* pStatus = &pReader->status; pBlockNum->numOfBlocks = 0; - pBlockNum->numOfLastBlocks = 0; + pBlockNum->numOfLastFiles = 0; size_t numOfTables = taosHashGetSize(pReader->status.pTableMap); SArray* pIndexList = taosArrayInit(numOfTables, sizeof(SBlockIdx)); - SArray* pLastBlocks = pStatus->fileIter.pLastBlockReader->pBlockL; - taosArrayClear(pLastBlocks); while (1) { bool hasNext = filesetIteratorNext(&pStatus->fileIter, pReader); @@ -2211,32 +2290,16 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return code; } - code = tsdbReadBlockL(pReader->pFileReader, pLastBlocks); - if (code != TSDB_CODE_SUCCESS) { - taosArrayDestroy(pIndexList); - return code; - } - - if (taosArrayGetSize(pIndexList) > 0 || taosArrayGetSize(pLastBlocks) > 0) { - SArray* pQLastBlock = taosArrayInit(4, sizeof(SBlockL)); - - code = doLoadFileBlock(pReader, pIndexList, pLastBlocks, pBlockNum, pQLastBlock); + if (taosArrayGetSize(pIndexList) > 0 || pReader->pFileReader->pSet->nSttF > 0) { + code = doLoadFileBlock(pReader, pIndexList, pBlockNum); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(pIndexList); - taosArrayDestroy(pQLastBlock); return code; } - if (pBlockNum->numOfBlocks + pBlockNum->numOfLastBlocks > 0) { - ASSERT(taosArrayGetSize(pQLastBlock) == pBlockNum->numOfLastBlocks); - taosArrayClear(pLastBlocks); - taosArrayAddAll(pLastBlocks, pQLastBlock); - - taosArrayDestroy(pQLastBlock); + if (pBlockNum->numOfBlocks + pBlockNum->numOfLastFiles > 0) { break; } - - taosArrayDestroy(pQLastBlock); } // no blocks in current file, try next files @@ -2246,81 +2309,22 @@ static int32_t moveToNextFile(STsdbReader* pReader, SBlockNumber* pBlockNum) { return TSDB_CODE_SUCCESS; } -static int32_t doLoadRelatedLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo *pBlockScanInfo, STsdbReader* pReader) { - SArray* pBlocks = pLastBlockReader->pBlockL; - SBlockL* pBlock = NULL; - - uint64_t uid = pBlockScanInfo->uid; - int32_t totalLastBlocks = (int32_t)taosArrayGetSize(pBlocks); - - initMemDataIterator(pBlockScanInfo, pReader); - - // find the correct SBlockL. todo binary search - int32_t index = -1; - for (int32_t i = 0; i < totalLastBlocks; ++i) { - SBlockL* p = taosArrayGet(pBlocks, i); - if (p->minUid <= uid && p->maxUid >= uid) { - index = i; - pBlock = p; - break; - } - } - - if (index == -1) { - pLastBlockReader->currentBlockIndex = index; - tBlockDataReset(&pLastBlockReader->lastBlockData); - return TSDB_CODE_SUCCESS; - } - - // the required last datablock has already loaded - if (index == pLastBlockReader->currentBlockIndex) { - return TSDB_CODE_SUCCESS; - } - - int64_t st = taosGetTimestampUs(); - int32_t code = tBlockDataInit(&pLastBlockReader->lastBlockData, pReader->suid, pReader->suid ? 0 : uid, pReader->pSchema); - if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p init block data failed, code:%s %s", pReader, tstrerror(code), pReader->idStr); - return code; - } - - code = tsdbReadLastBlock(pReader->pFileReader, pBlock, &pLastBlockReader->lastBlockData); - - double el = (taosGetTimestampUs() - st) / 1000.0; - if (code != TSDB_CODE_SUCCESS) { - tsdbError("%p error occurs in loading last block into buffer, last block index:%d, total:%d code:%s %s", pReader, - pLastBlockReader->currentBlockIndex, totalLastBlocks, tstrerror(code), pReader->idStr); - } else { - tsdbDebug("%p load last block completed, uid:%" PRIu64 - " last block index:%d, total:%d rows:%d, minVer:%d, maxVer:%d, brange:%" PRId64 "-%" PRId64 - " elapsed time:%.2f ms, %s", - pReader, uid, index, totalLastBlocks, pBlock->nRow, pBlock->minVer, pBlock->maxVer, pBlock->minKey, - pBlock->maxKey, el, pReader->idStr); - } - - pLastBlockReader->currentBlockIndex = index; - pReader->cost.lastBlockLoad += 1; - pReader->cost.lastBlockLoadTime += el; - - return TSDB_CODE_SUCCESS; -} - static int32_t uidComparFunc(const void* p1, const void* p2) { - uint64_t pu1 = *(uint64_t*) p1; - uint64_t pu2 = *(uint64_t*) p2; + uint64_t pu1 = *(uint64_t*)p1; + uint64_t pu2 = *(uint64_t*)p2; if (pu1 == pu2) { return 0; } else { - return (pu1 < pu2)? -1:1; + return (pu1 < pu2) ? -1 : 1; } } -static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SReaderStatus* pStatus) { +static void extractOrderedTableUidList(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) { int32_t index = 0; int32_t total = taosHashGetSize(pStatus->pTableMap); void* p = taosHashIterate(pStatus->pTableMap, NULL); - while(p != NULL) { + while (p != NULL) { STableBlockScanInfo* pScanInfo = p; pOrderCheckInfo->tableUidList[index++] = pScanInfo->uid; p = taosHashIterate(pStatus->pTableMap, p); @@ -2330,9 +2334,12 @@ static void extractOrderedTableUidList(SUidOrderCheckInfo *pOrderCheckInfo, SRea } static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderStatus* pStatus) { - if (pOrderCheckInfo->tableUidList == NULL) { - int32_t total = taosHashGetSize(pStatus->pTableMap); + int32_t total = taosHashGetSize(pStatus->pTableMap); + if (total == 0) { + return TSDB_CODE_SUCCESS; + } + if (pOrderCheckInfo->tableUidList == NULL) { pOrderCheckInfo->currentIndex = 0; pOrderCheckInfo->tableUidList = taosMemoryMalloc(total * sizeof(uint64_t)); if (pOrderCheckInfo->tableUidList == NULL) { @@ -2340,20 +2347,17 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt } extractOrderedTableUidList(pOrderCheckInfo, pStatus); - uint64_t uid = pOrderCheckInfo->tableUidList[0]; pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); } else { if (pStatus->pTableIter == NULL) { // it is the last block of a new file -// ASSERT(pOrderCheckInfo->currentIndex == taosHashGetSize(pStatus->pTableMap)); - pOrderCheckInfo->currentIndex = 0; uint64_t uid = pOrderCheckInfo->tableUidList[pOrderCheckInfo->currentIndex]; pStatus->pTableIter = taosHashGet(pStatus->pTableMap, &uid, sizeof(uid)); // the tableMap has already updated if (pStatus->pTableIter == NULL) { - void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, taosHashGetSize(pStatus->pTableMap)*sizeof(uint64_t)); + void* p = taosMemoryRealloc(pOrderCheckInfo->tableUidList, total * sizeof(uint64_t)); if (p == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -2370,7 +2374,7 @@ static int32_t initOrderCheckInfo(SUidOrderCheckInfo* pOrderCheckInfo, SReaderSt return TSDB_CODE_SUCCESS; } -static bool moveToNextTable(SUidOrderCheckInfo *pOrderedCheckInfo, SReaderStatus* pStatus) { +static bool moveToNextTable(SUidOrderCheckInfo* pOrderedCheckInfo, SReaderStatus* pStatus) { pOrderedCheckInfo->currentIndex += 1; if (pOrderedCheckInfo->currentIndex >= taosHashGetSize(pStatus->pTableMap)) { pStatus->pTableIter = NULL; @@ -2384,38 +2388,20 @@ static bool moveToNextTable(SUidOrderCheckInfo *pOrderedCheckInfo, SReaderStatus } static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { - SReaderStatus* pStatus = &pReader->status; + SReaderStatus* pStatus = &pReader->status; SLastBlockReader* pLastBlockReader = pStatus->fileIter.pLastBlockReader; - SUidOrderCheckInfo *pOrderedCheckInfo = &pStatus->uidCheckInfo; - int32_t code = initOrderCheckInfo(pOrderedCheckInfo, pStatus); + SUidOrderCheckInfo* pOrderedCheckInfo = &pStatus->uidCheckInfo; + int32_t code = initOrderCheckInfo(pOrderedCheckInfo, pStatus); if (code != TSDB_CODE_SUCCESS || (taosHashGetSize(pStatus->pTableMap) == 0)) { return code; } - while(1) { + while (1) { // load the last data block of current table STableBlockScanInfo* pScanInfo = pStatus->pTableIter; - code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - if (pLastBlockReader->currentBlockIndex != -1) { - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - int32_t index = pScanInfo->indexInBlockL; - - if (index == INITIAL_ROW_INDEX_VAL || index == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); - if (!hasData) { // current table does not have rows in last block, try next table - bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); - if (!hasNexTable) { - return TSDB_CODE_SUCCESS; - } - continue; - } - } - } else { // no data in last block, try next table + bool hasVal = initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + if (!hasVal) { bool hasNexTable = moveToNextTable(pOrderedCheckInfo, pStatus); if (!hasNexTable) { return TSDB_CODE_SUCCESS; @@ -2441,9 +2427,8 @@ static int32_t doLoadLastBlockSequentially(STsdbReader* pReader) { } static int32_t doBuildDataBlock(STsdbReader* pReader) { - TSDBKEY key = {0}; - int32_t code = TSDB_CODE_SUCCESS; - SBlock* pBlock = NULL; + int32_t code = TSDB_CODE_SUCCESS; + SDataBlk* pBlock = NULL; SReaderStatus* pStatus = &pReader->status; SDataBlockIter* pBlockIter = &pStatus->blockIter; @@ -2461,26 +2446,13 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { pBlock = getCurrentBlock(pBlockIter); } - { - key = getCurrentKeyInBuf(pScanInfo, pReader); - - // load the last data block of current table - code = doLoadRelatedLastBlock(pLastBlockReader, pScanInfo, pReader); - if (code != TSDB_CODE_SUCCESS) { - return code; - } - - // note: the lastblock may be null here - initLastBlockReader(pLastBlockReader, pScanInfo->uid, &pScanInfo->indexInBlockL); - if (pScanInfo->indexInBlockL == INITIAL_ROW_INDEX_VAL || pScanInfo->indexInBlockL == pLastBlockReader->lastBlockData.nRow) { - bool hasData = nextRowInLastBlock(pLastBlockReader, pScanInfo); - } - } + initLastBlockReader(pLastBlockReader, pScanInfo, pReader); + TSDBKEY keyInBuf = getCurrentKeyInBuf(pScanInfo, pReader); if (pBlockInfo == NULL) { // build data block from last data file ASSERT(pBlockIter->numOfBlocks == 0); code = buildComposedDataBlock(pReader); - } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, key, pLastBlockReader)) { + } else if (fileBlockShouldLoad(pReader, pBlockInfo, pBlock, pScanInfo, keyInBuf, pLastBlockReader)) { tBlockDataReset(&pStatus->fileBlockData); code = tBlockDataInit(&pStatus->fileBlockData, pReader->suid, pScanInfo->uid, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { @@ -2494,20 +2466,20 @@ static int32_t doBuildDataBlock(STsdbReader* pReader) { // build composed data block code = buildComposedDataBlock(pReader); - } else if (bufferDataInFileBlockGap(pReader->order, key, pBlock)) { + } else if (bufferDataInFileBlockGap(pReader->order, keyInBuf, pBlock)) { // data in memory that are earlier than current file block - // todo rows in buffer should be less than the file block in asc, greater than file block in desc + // rows in buffer should be less than the file block in asc, greater than file block in desc int64_t endKey = (ASCENDING_TRAVERSE(pReader->order)) ? pBlock->minKey.ts : pBlock->maxKey.ts; code = buildDataBlockFromBuf(pReader, pScanInfo, endKey); } else { if (hasDataInLastBlock(pLastBlockReader) && !ASCENDING_TRAVERSE(pReader->order)) { // only return the rows in last block int64_t tsLast = getCurrentKeyInLastBlock(pLastBlockReader); - ASSERT (tsLast >= pBlock->maxKey.ts); + ASSERT(tsLast >= pBlock->maxKey.ts); tBlockDataReset(&pReader->status.fileBlockData); code = buildComposedDataBlock(pReader); - } else { // whole block is required, return it directly + } else { // whole block is required, return it directly SDataBlockInfo* pInfo = &pReader->pResBlock->info; pInfo->rows = pBlock->nRow; pInfo->uid = pScanInfo->uid; @@ -2554,7 +2526,7 @@ static int32_t buildBlockFromBufferSequentially(STsdbReader* pReader) { // set the correct start position in case of the first/last file block, according to the query time window static void initBlockDumpInfo(STsdbReader* pReader, SDataBlockIter* pBlockIter) { - SBlock* pBlock = getCurrentBlock(pBlockIter); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); SReaderStatus* pStatus = &pReader->status; @@ -2574,7 +2546,7 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl } // all data files are consumed, try data in buffer - if (num.numOfBlocks + num.numOfLastBlocks == 0) { + if (num.numOfBlocks + num.numOfLastFiles == 0) { pReader->status.loadFromFile = false; return code; } @@ -2582,14 +2554,11 @@ static int32_t initForFirstBlockInFile(STsdbReader* pReader, SDataBlockIter* pBl // initialize the block iterator for a new fileset if (num.numOfBlocks > 0) { code = initBlockIterator(pReader, pBlockIter, num.numOfBlocks); - } else { // no block data, only last block exists + } else { // no block data, only last block exists tBlockDataReset(&pReader->status.fileBlockData); resetDataBlockIterator(pBlockIter, pReader->order); } - SLastBlockReader* pLReader = pReader->status.fileIter.pLastBlockReader; - pLReader->currentBlockIndex = -1; - // set the correct start position according to the query time window initBlockDumpInfo(pReader, pBlockIter); return code; @@ -2607,7 +2576,7 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { SDataBlockIter* pBlockIter = &pReader->status.blockIter; if (pBlockIter->numOfBlocks == 0) { - _begin: + _begin: code = doLoadLastBlockSequentially(pReader); if (code != TSDB_CODE_SUCCESS) { return code; @@ -2654,21 +2623,24 @@ static int32_t buildBlockFromFiles(STsdbReader* pReader) { bool hasNext = blockIteratorNext(&pReader->status.blockIter); if (hasNext) { // check for the next block in the block accessed order list initBlockDumpInfo(pReader, pBlockIter); - } else if (taosArrayGetSize(pReader->status.fileIter.pLastBlockReader->pBlockL) > 0) { // data blocks in current file are exhausted, let's try the next file now - tBlockDataReset(&pReader->status.fileBlockData); - resetDataBlockIterator(pBlockIter, pReader->order); - goto _begin; } else { - code = initForFirstBlockInFile(pReader, pBlockIter); + if (pReader->status.pCurrentFileset->nSttF > 0) { + // data blocks in current file are exhausted, let's try the next file now + tBlockDataReset(&pReader->status.fileBlockData); + resetDataBlockIterator(pBlockIter, pReader->order); + goto _begin; + } else { + code = initForFirstBlockInFile(pReader, pBlockIter); - // error happens or all the data files are completely checked - if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { - return code; - } + // error happens or all the data files are completely checked + if ((code != TSDB_CODE_SUCCESS) || (pReader->status.loadFromFile == false)) { + return code; + } - // this file does not have blocks, let's start check the last block file - if (pBlockIter->numOfBlocks == 0) { - goto _begin; + // this file does not have blocks, let's start check the last block file + if (pBlockIter->numOfBlocks == 0) { + goto _begin; + } } } } @@ -2691,6 +2663,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret if (VND_IS_RSMA(pVnode)) { int8_t level = 0; int64_t now = taosGetTimestamp(pVnode->config.tsdbCfg.precision); + int64_t offset = TSDB_TICK_PER_SECOND(pVnode->config.tsdbCfg.precision); for (int8_t i = 0; i < TSDB_RETENTION_MAX; ++i) { SRetention* pRetention = retentions + level; @@ -2700,7 +2673,7 @@ static STsdb* getTsdbByRetentions(SVnode* pVnode, TSKEY winSKey, SRetention* ret } break; } - if ((now - pRetention->keep) <= winSKey) { + if ((now - pRetention->keep) <= (winSKey + offset)) { break; } ++level; @@ -2927,7 +2900,7 @@ typedef enum { CHECK_FILEBLOCK_QUIT = 0x2, } CHECK_FILEBLOCK_STATE; -static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SBlock* pBlock, +static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanInfo* pScanInfo, SDataBlk* pBlock, SFileDataBlockInfo* pFBlock, SRowMerger* pMerger, int64_t key, CHECK_FILEBLOCK_STATE* state) { SFileBlockDumpInfo* pDumpInfo = &pReader->status.fBlockDumpInfo; @@ -2936,8 +2909,8 @@ static int32_t checkForNeighborFileBlock(STsdbReader* pReader, STableBlockScanIn *state = CHECK_FILEBLOCK_QUIT; int32_t step = ASCENDING_TRAVERSE(pReader->order) ? 1 : -1; - int32_t nextIndex = -1; - SBlock* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); + int32_t nextIndex = -1; + SDataBlk* pNeighborBlock = getNeighborBlockOfSameTable(pFBlock, pScanInfo, &nextIndex, pReader->order); if (pNeighborBlock == NULL) { // do nothing return 0; } @@ -3001,7 +2974,7 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc CHECK_FILEBLOCK_STATE st; SFileDataBlockInfo* pFileBlockInfo = getCurrentBlockInfo(&pReader->status.blockIter); - SBlock* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); + SDataBlk* pCurrentBlock = getCurrentBlock(&pReader->status.blockIter); checkForNeighborFileBlock(pReader, pScanInfo, pCurrentBlock, pFileBlockInfo, pMerger, key, &st); if (st == CHECK_FILEBLOCK_QUIT) { break; @@ -3012,11 +2985,13 @@ int32_t doMergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pSc return TSDB_CODE_SUCCESS; } -int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, SRowMerger* pMerger) { - while(nextRowInLastBlock(pLastBlockReader, pScanInfo)) { +int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockScanInfo* pScanInfo, int64_t ts, + SRowMerger* pMerger) { + pScanInfo->lastKey = ts; + while (nextRowFromLastBlocks(pLastBlockReader, pScanInfo)) { int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { - TSDBROW fRow1 = tsdbRowFromBlockData(&pLastBlockReader->lastBlockData, *pLastBlockReader->rowIndex); + TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); tRowMerge(pMerger, &fRow1); } else { break; @@ -3026,8 +3001,8 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc return TSDB_CODE_SUCCESS; } -void doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, - STsdbReader* pReader, bool* freeTSRow) { +int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SArray* pDelList, STSRow** pTSRow, + STsdbReader* pReader, bool* freeTSRow) { TSDBROW* pNextRow = NULL; TSDBROW current = *pRow; @@ -3037,19 +3012,19 @@ void doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SAr if (!pIter->hasVal) { *pTSRow = current.pTSRow; *freeTSRow = false; - return; + return TSDB_CODE_SUCCESS; } else { // has next point in mem/imem pNextRow = getValidMemRow(pIter, pDelList, pReader); if (pNextRow == NULL) { *pTSRow = current.pTSRow; *freeTSRow = false; - return; + return TSDB_CODE_SUCCESS; } if (current.pTSRow->ts != pNextRow->pTSRow->ts) { *pTSRow = current.pTSRow; *freeTSRow = false; - return; + return TSDB_CODE_SUCCESS; } } } @@ -3069,14 +3044,18 @@ void doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, SAr tRowMergerAdd(&merge, pNextRow, pTSchema1); doMergeRowsInBuf(pIter, uid, current.pTSRow->ts, pDelList, &merge, pReader); - tRowMergerGetRow(&merge, pTSRow); - tRowMergerClear(&merge); + int32_t code = tRowMergerGetRow(&merge, pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + tRowMergerClear(&merge); *freeTSRow = true; + return TSDB_CODE_SUCCESS; } -void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, - STSRow** pTSRow) { +int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, + STSRow** pTSRow) { SRowMerger merge = {0}; TSDBKEY k = TSDBROW_KEY(pRow); @@ -3100,7 +3079,8 @@ void doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* pBlo doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, k.ts, pBlockScanInfo->delSkyline, &merge, pReader); } - tRowMergerGetRow(&merge, pTSRow); + int32_t code = tRowMergerGetRow(&merge, pTSRow); + return code; } int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pReader, STSRow** pTSRow, int64_t endKey, @@ -3130,28 +3110,31 @@ int32_t tsdbGetNextRowInMem(STableBlockScanInfo* pBlockScanInfo, STsdbReader* pR TSDBKEY k = TSDBROW_KEY(pRow); TSDBKEY ik = TSDBROW_KEY(piRow); + int32_t code = TSDB_CODE_SUCCESS; if (ik.ts != k.ts) { if (((ik.ts < k.ts) && asc) || ((ik.ts > k.ts) && (!asc))) { // ik.ts < k.ts - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); } else if (((k.ts < ik.ts) && asc) || ((k.ts > ik.ts) && (!asc))) { - doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); + code = doMergeMemTableMultiRows(pRow, uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); } } else { // ik.ts == k.ts - doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); *freeTSRow = true; + code = doMergeMemIMemRows(pRow, piRow, pBlockScanInfo, pReader, pTSRow); + if (code != TSDB_CODE_SUCCESS) { + return code; + } } - return TSDB_CODE_SUCCESS; + return code; } if (pBlockScanInfo->iter.hasVal && pRow != NULL) { - doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, freeTSRow); - return TSDB_CODE_SUCCESS; + return doMergeMemTableMultiRows(pRow, pBlockScanInfo->uid, &pBlockScanInfo->iter, pDelList, pTSRow, pReader, + freeTSRow); } if (pBlockScanInfo->iiter.hasVal && piRow != NULL) { - doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); - return TSDB_CODE_SUCCESS; + return doMergeMemTableMultiRows(piRow, uid, &pBlockScanInfo->iiter, pDelList, pTSRow, pReader, freeTSRow); } return TSDB_CODE_SUCCESS; @@ -3201,7 +3184,8 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, STSRow* return TSDB_CODE_SUCCESS; } -int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, int32_t rowIndex) { +int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, SBlockData* pBlockData, + int32_t rowIndex) { int32_t i = 0, j = 0; int32_t outputRowIndex = pResBlock->info.rows; @@ -3214,18 +3198,23 @@ int32_t doAppendRowFromFileBlock(SSDataBlock* pResBlock, STsdbReader* pReader, S } SColVal cv = {0}; - int32_t numOfInputCols = taosArrayGetSize(pBlockData->aIdx); - int32_t numOfOutputCols = blockDataGetNumOfCols(pResBlock); + int32_t numOfInputCols = pBlockData->aIdx->size; + int32_t numOfOutputCols = pResBlock->pDataBlock->size; while (i < numOfOutputCols && j < numOfInputCols) { - SColumnInfoData* pCol = taosArrayGet(pResBlock->pDataBlock, i); + SColumnInfoData* pCol = TARRAY_GET_ELEM(pResBlock->pDataBlock, i); SColData* pData = tBlockDataGetColDataByIdx(pBlockData, j); + if (pData->cid < pCol->info.colId) { + j += 1; + continue; + } + if (pData->cid == pCol->info.colId) { tColDataGetValue(pData, rowIndex, &cv); doCopyColVal(pCol, outputRowIndex, i, &cv, pSupInfo); j += 1; - } else { // the specified column does not exist in file block, fill with null data + } else if (pData->cid > pCol->info.colId) { // the specified column does not exist in file block, fill with null data colDataAppendNULL(pCol, outputRowIndex); } @@ -3278,7 +3267,7 @@ int32_t tsdbSetTableId(STsdbReader* pReader, int64_t uid) { ASSERT(pReader != NULL); taosHashClear(pReader->status.pTableMap); - STableBlockScanInfo info = {.lastKey = 0, .uid = uid, .indexInBlockL = INITIAL_ROW_INDEX_VAL}; + STableBlockScanInfo info = {.lastKey = 0, .uid = uid}; taosHashPut(pReader->status.pTableMap, &info.uid, sizeof(uint64_t), &info, sizeof(info)); return TDB_CODE_SUCCESS; } @@ -3299,7 +3288,6 @@ void* tsdbGetIvtIdx(SMeta* pMeta) { uint64_t getReaderMaxVersion(STsdbReader* pReader) { return pReader->verRange.maxVer; } - // ====================================== EXPOSED APIs ====================================== int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTableList, STsdbReader** ppReader, const char* idstr) { @@ -3348,11 +3336,18 @@ int32_t tsdbReaderOpen(SVnode* pVnode, SQueryTableDataCond* pCond, SArray* pTabl } } + // NOTE: the endVersion in pCond is the data version not schema version, so pCond->endVersion is not correct here. if (pCond->suid != 0) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, pCond->schemaVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pReader->suid, /*pCond->endVersion*/ -1); + if (pReader->pSchema == NULL) { + tsdbError("failed to get table schema, suid:%"PRIu64", ver:%"PRId64" , %s", pReader->suid, -1, pReader->idStr); + } } else if (taosArrayGetSize(pTableList) > 0) { STableKeyInfo* pKey = taosArrayGet(pTableList, 0); - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, pCond->schemaVersion); + pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, pKey->uid, /*pCond->endVersion*/ -1); + if (pReader->pSchema == NULL) { + tsdbError("failed to get table schema, uid:%"PRIu64", ver:%"PRId64" , %s", pKey->uid, -1, pReader->idStr); + } } int32_t numOfTables = taosArrayGetSize(pTableList); @@ -3422,7 +3417,6 @@ void tsdbReaderClose(STsdbReader* pReader) { } SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; - tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); taosMemoryFreeClear(pSupInfo->plist); taosMemoryFree(pSupInfo->colIds); @@ -3433,6 +3427,7 @@ void tsdbReaderClose(STsdbReader* pReader) { taosMemoryFreeClear(pSupInfo->buildBuf[i]); } } + taosMemoryFree(pSupInfo->buildBuf); tBlockDataDestroy(&pReader->status.fileBlockData, true); @@ -3446,25 +3441,30 @@ void tsdbReaderClose(STsdbReader* pReader) { tsdbDataFReaderClose(&pReader->pFileReader); } + tsdbUntakeReadSnap(pReader->pTsdb, pReader->pReadSnap); + taosMemoryFree(pReader->status.uidCheckInfo.tableUidList); + SIOCostSummary* pCost = &pReader->cost; SFilesetIter* pFilesetIter = &pReader->status.fileIter; if (pFilesetIter->pLastBlockReader != NULL) { - tBlockDataDestroy(&pFilesetIter->pLastBlockReader->lastBlockData, true); - taosArrayDestroy(pFilesetIter->pLastBlockReader->pBlockL); - taosMemoryFree(pFilesetIter->pLastBlockReader); - } + SLastBlockReader* pLReader = pFilesetIter->pLastBlockReader; + tMergeTreeClose(&pLReader->mergeTree); - SIOCostSummary* pCost = &pReader->cost; + getLastBlockLoadInfo(pLReader->pInfo, &pCost->lastBlockLoad, &pCost->lastBlockLoadTime); + + pLReader->pInfo = destroyLastBlockLoadInfo(pLReader->pInfo); + taosMemoryFree(pLReader); + } - tsdbDebug("%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 - " SMA-time:%.2f ms, fileBlocks:%" PRId64 - ", fileBlocks-time:%.2f ms, " - "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 - ", lastBlocks-time:%.2f ms, STableBlockScanInfo size:%.2f Kb %s", - pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, - pCost->numOfBlocks, pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, - pCost->lastBlockLoadTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); + tsdbDebug( + "%p :io-cost summary: head-file:%" PRIu64 ", head-file time:%.2f ms, SMA:%" PRId64 + " SMA-time:%.2f ms, fileBlocks:%" PRId64 ", fileBlocks-load-time:%.2f ms, " + "build in-memory-block-time:%.2f ms, lastBlocks:%" PRId64 ", lastBlocks-time:%.2f ms, composed-blocks:%" PRId64 + ", composed-blocks-time:%.2fms, STableBlockScanInfo size:%.2f Kb %s", + pReader, pCost->headFileLoad, pCost->headFileLoadTime, pCost->smaDataLoad, pCost->smaLoadTime, pCost->numOfBlocks, + pCost->blockLoadTime, pCost->buildmemBlock, pCost->lastBlockLoad, pCost->lastBlockLoadTime, pCost->composedBlocks, + pCost->buildComposedBlockTime, numOfTables * sizeof(STableBlockScanInfo) / 1000.0, pReader->idStr); taosMemoryFree(pReader->idStr); taosMemoryFree(pReader->pSchema); @@ -3575,12 +3575,12 @@ int32_t tsdbRetrieveDatablockSMA(STsdbReader* pReader, SColumnDataAgg*** pBlockS SFileDataBlockInfo* pFBlock = getCurrentBlockInfo(&pReader->status.blockIter); - SBlock* pBlock = getCurrentBlock(&pReader->status.blockIter); - int64_t stime = taosGetTimestampUs(); + SDataBlk* pBlock = getCurrentBlock(&pReader->status.blockIter); + int64_t stime = taosGetTimestampUs(); SBlockLoadSuppInfo* pSup = &pReader->suppInfo; - if (tBlockHasSma(pBlock)) { + if (tDataBlkHasSma(pBlock)) { code = tsdbReadBlockSma(pReader->pFileReader, pBlock, pSup->pColAgg); if (code != TSDB_CODE_SUCCESS) { tsdbDebug("vgId:%d, failed to load block SMA for uid %" PRIu64 ", code:%s, %s", 0, pFBlock->uid, tstrerror(code), @@ -3695,11 +3695,12 @@ int32_t tsdbReaderReset(STsdbReader* pReader, SQueryTableDataCond* pCond) { tsdbDataFReaderClose(&pReader->pFileReader); int32_t numOfTables = taosHashGetSize(pReader->status.pTableMap); - tsdbDataFReaderClose(&pReader->pFileReader); initFilesetIterator(&pReader->status.fileIter, pReader->pReadSnap->fs.aDFileSet, pReader); resetDataBlockIterator(&pReader->status.blockIter, pReader->order); - resetDataBlockScanInfo(pReader->status.pTableMap); + + int64_t ts = ASCENDING_TRAVERSE(pReader->order) ? pReader->window.skey - 1 : pReader->window.ekey + 1; + resetDataBlockScanInfo(pReader->status.pTableMap, ts); int32_t code = 0; SDataBlockIter* pBlockIter = &pReader->status.blockIter; @@ -3757,7 +3758,7 @@ int32_t tsdbGetFileBlocksDistInfo(STsdbReader* pReader, STableBlockDistInfo* pTa while (true) { if (hasNext) { - SBlock* pBlock = getCurrentBlock(pBlockIter); + SDataBlk* pBlock = getCurrentBlock(pBlockIter); int32_t numOfRows = pBlock->nRow; pTableBlockInfo->totalRows += numOfRows; diff --git a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c index c8f3862071b3628fdefd26df58ea3cb01e80d302..5fe0b408b1396c52fd1fbdf5ad2b6e37c0e5e7be 100644 --- a/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c +++ b/source/dnode/vnode/src/tsdb/tsdbReaderWriter.c @@ -15,743 +15,933 @@ #include "tsdb.h" -// SDelFWriter ==================================================== -int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; - SDelFWriter *pDelFWriter; - int64_t n; +// =============== PAGE-WISE FILE =============== +static int32_t tsdbOpenFile(const char *path, int32_t szPage, int32_t flag, STsdbFD **ppFD) { + int32_t code = 0; + STsdbFD *pFD; - // alloc - pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); - if (pDelFWriter == NULL) { + *ppFD = NULL; + + pFD = (STsdbFD *)taosMemoryCalloc(1, sizeof(*pFD) + strlen(path) + 1); + if (pFD == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + goto _exit; } - pDelFWriter->pTsdb = pTsdb; - pDelFWriter->fDel = *pFile; - tsdbDelFileName(pTsdb, pFile, fname); - pDelFWriter->pWriteH = taosOpenFile(fname, TD_FILE_WRITE | TD_FILE_CREATE); - if (pDelFWriter->pWriteH == NULL) { + pFD->path = (char *)&pFD[1]; + strcpy(pFD->path, path); + pFD->szPage = szPage; + pFD->flag = flag; + pFD->pFD = taosOpenFile(path, flag); + if (pFD->pFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + goto _exit; } - - // update header - n = taosWriteFile(pDelFWriter->pWriteH, &hdr, TSDB_FHDR_SIZE); - if (n < 0) { + pFD->szPage = szPage; + pFD->pgno = 0; + pFD->pBuf = taosMemoryCalloc(1, szPage); + if (pFD->pBuf == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pFD); + goto _exit; + } + if (taosStatFile(path, &pFD->szFile, NULL) < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + goto _exit; } + ASSERT(pFD->szFile % szPage == 0); + pFD->szFile = pFD->szFile / szPage; + *ppFD = pFD; - pDelFWriter->fDel.size = TSDB_FHDR_SIZE; - pDelFWriter->fDel.offset = 0; - - *ppWriter = pDelFWriter; +_exit: return code; +} -_err: - tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; +static void tsdbCloseFile(STsdbFD **ppFD) { + STsdbFD *pFD = *ppFD; + taosMemoryFree(pFD->pBuf); + taosCloseFile(&pFD->pFD); + taosMemoryFree(pFD); + *ppFD = NULL; +} + +static int32_t tsdbWriteFilePage(STsdbFD *pFD) { + int32_t code = 0; + + if (pFD->pgno > 0) { + int64_t n = taosLSeekFile(pFD->pFD, PAGE_OFFSET(pFD->pgno, pFD->szPage), SEEK_SET); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + taosCalcChecksumAppend(0, pFD->pBuf, pFD->szPage); + + n = taosWriteFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + + if (pFD->szFile < pFD->pgno) { + pFD->szFile = pFD->pgno; + } + } + pFD->pgno = 0; + +_exit: return code; } -int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { - int32_t code = 0; - SDelFWriter *pWriter = *ppWriter; - STsdb *pTsdb = pWriter->pTsdb; +static int32_t tsdbReadFilePage(STsdbFD *pFD, int64_t pgno) { + int32_t code = 0; - // sync - if (sync && taosFsyncFile(pWriter->pWriteH) < 0) { + ASSERT(pgno <= pFD->szFile); + + // seek + int64_t offset = PAGE_OFFSET(pgno, pFD->szPage); + int64_t n = taosLSeekFile(pFD->pFD, offset, SEEK_SET); + if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + goto _exit; } - // close - if (taosCloseFile(&pWriter->pWriteH) < 0) { + // read + n = taosReadFile(pFD->pFD, pFD->pBuf, pFD->szPage); + if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); - goto _err; + goto _exit; + } else if (n < pFD->szPage) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; } - for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pWriter->aBuf[iBuf]); + // check + if (!taosCheckChecksumWhole(pFD->pBuf, pFD->szPage)) { + code = TSDB_CODE_FILE_CORRUPTED; + goto _exit; } - taosMemoryFree(pWriter); - *ppWriter = NULL; - return code; + pFD->pgno = pgno; -_err: - tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); +_exit: return code; } -int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { +static int32_t tsdbWriteFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { int32_t code = 0; - int64_t size; - int64_t n; + int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); + int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); + int64_t bOffset = fOffset % pFD->szPage; + int64_t n = 0; + + do { + if (pFD->pgno != pgno) { + code = tsdbWriteFilePage(pFD); + if (code) goto _exit; + + if (pgno <= pFD->szFile) { + code = tsdbReadFilePage(pFD, pgno); + if (code) goto _exit; + } else { + pFD->pgno = pgno; + } + } - // prepare - size = sizeof(uint32_t); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); - } - size += sizeof(TSCKSUM); + int64_t nWrite = TMIN(PAGE_CONTENT_SIZE(pFD->szPage) - bOffset, size - n); + memcpy(pFD->pBuf + bOffset, pBuf + n, nWrite); - // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; + pgno++; + bOffset = 0; + n += nWrite; + } while (n < size); - // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { - n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); - } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); +_exit: + return code; +} - ASSERT(n + sizeof(TSCKSUM) == size); +static int32_t tsdbReadFile(STsdbFD *pFD, int64_t offset, uint8_t *pBuf, int64_t size) { + int32_t code = 0; + int64_t n = 0; + int64_t fOffset = LOGIC_TO_FILE_OFFSET(offset, pFD->szPage); + int64_t pgno = OFFSET_PGNO(fOffset, pFD->szPage); + int32_t szPgCont = PAGE_CONTENT_SIZE(pFD->szPage); + int64_t bOffset = fOffset % pFD->szPage; + + ASSERT(pgno && pgno <= pFD->szFile); + ASSERT(bOffset < szPgCont); + + while (n < size) { + if (pFD->pgno != pgno) { + code = tsdbReadFilePage(pFD, pgno); + if (code) goto _exit; + } - // write - n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + int64_t nRead = TMIN(szPgCont - bOffset, size - n); + memcpy(pBuf + n, pFD->pBuf + bOffset, nRead); + + n += nRead; + pgno++; + bOffset = 0; } - ASSERT(n == size); +_exit: + return code; +} - // update - pDelIdx->offset = pWriter->fDel.size; - pDelIdx->size = size; - pWriter->fDel.size += size; +static int32_t tsdbFsyncFile(STsdbFD *pFD) { + int32_t code = 0; - return code; + code = tsdbWriteFilePage(pFD); + if (code) goto _exit; -_err: - tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + if (taosFsyncFile(pFD->pFD) < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _exit; + } + +_exit: return code; } -int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { - int32_t code = 0; - int64_t size; - int64_t n; - SDelIdx *pDelIdx; +// SDataFWriter ==================================================== +int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { + int32_t code = 0; + int32_t flag; + int64_t n; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; + SDataFWriter *pWriter = NULL; + char fname[TSDB_FILENAME_LEN]; + char hdr[TSDB_FHDR_SIZE] = {0}; - // prepare - size = sizeof(uint32_t); - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); + // alloc + pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); + if (pWriter == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + pWriter->pTsdb = pTsdb; + pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, + .fid = pSet->fid, + .pHeadF = &pWriter->fHead, + .pDataF = &pWriter->fData, + .pSmaF = &pWriter->fSma, + .nSttF = pSet->nSttF}; + pWriter->fHead = *pSet->pHeadF; + pWriter->fData = *pSet->pDataF; + pWriter->fSma = *pSet->pSmaF; + for (int8_t iStt = 0; iStt < pSet->nSttF; iStt++) { + pWriter->wSet.aSttF[iStt] = &pWriter->fStt[iStt]; + pWriter->fStt[iStt] = *pSet->aSttF[iStt]; } - size += sizeof(TSCKSUM); - // alloc - code = tRealloc(&pWriter->aBuf[0], size); + // head + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pHeadFD); if (code) goto _err; - // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { - n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); + code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; + pWriter->fHead.size += TSDB_FHDR_SIZE; + + // data + if (pWriter->fData.size == 0) { + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + } else { + flag = TD_FILE_READ | TD_FILE_WRITE; + } + tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pDataFD); + if (code) goto _err; + if (pWriter->fData.size == 0) { + code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; + pWriter->fData.size += TSDB_FHDR_SIZE; } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - ASSERT(n + sizeof(TSCKSUM) == size); + // sma + if (pWriter->fSma.size == 0) { + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + } else { + flag = TD_FILE_READ | TD_FILE_WRITE; + } + tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSmaFD); + if (code) goto _err; + if (pWriter->fSma.size == 0) { + code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; - // write - n = taosWriteFile(pWriter->pWriteH, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + pWriter->fSma.size += TSDB_FHDR_SIZE; } - // update - pWriter->fDel.offset = pWriter->fDel.size; - pWriter->fDel.size += size; + // stt + ASSERT(pWriter->fStt[pSet->nSttF - 1].size == 0); + flag = TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; + tsdbSttFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fStt[pSet->nSttF - 1], fname); + code = tsdbOpenFile(fname, szPage, flag, &pWriter->pSttFD); + if (code) goto _err; + code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; + pWriter->fStt[pWriter->wSet.nSttF - 1].size += TSDB_FHDR_SIZE; + *ppWriter = pWriter; return code; _err: - tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppWriter = NULL; return code; } -int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { +int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { int32_t code = 0; - char hdr[TSDB_FHDR_SIZE]; - int64_t size = TSDB_FHDR_SIZE; - int64_t n; + STsdb *pTsdb = NULL; - // build - memset(hdr, 0, size); - tPutDelFile(hdr, &pWriter->fDel); - taosCalcChecksumAppend(0, hdr, size); + if (*ppWriter == NULL) goto _exit; - // seek - if (taosLSeekFile(pWriter->pWriteH, 0, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + pTsdb = (*ppWriter)->pTsdb; + if (sync) { + code = tsdbFsyncFile((*ppWriter)->pHeadFD); + if (code) goto _err; - // write - n = taosWriteFile(pWriter->pWriteH, hdr, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + code = tsdbFsyncFile((*ppWriter)->pDataFD); + if (code) goto _err; + + code = tsdbFsyncFile((*ppWriter)->pSmaFD); + if (code) goto _err; + + code = tsdbFsyncFile((*ppWriter)->pSttFD); + if (code) goto _err; } + tsdbCloseFile(&(*ppWriter)->pHeadFD); + tsdbCloseFile(&(*ppWriter)->pDataFD); + tsdbCloseFile(&(*ppWriter)->pSmaFD); + tsdbCloseFile(&(*ppWriter)->pSttFD); + + for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppWriter)->aBuf[iBuf]); + } + taosMemoryFree(*ppWriter); +_exit: + *ppWriter = NULL; return code; _err: - tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } -// SDelFReader ==================================================== -struct SDelFReader { - STsdb *pTsdb; - SDelFile fDel; - TdFilePtr pReadH; +int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { + int32_t code = 0; + int64_t n; + char hdr[TSDB_FHDR_SIZE]; - uint8_t *aBuf[1]; -}; + // head ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutHeadFile(hdr, &pWriter->fHead); + code = tsdbWriteFile(pWriter->pHeadFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; -int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { - int32_t code = 0; - char fname[TSDB_FILENAME_LEN]; - SDelFReader *pDelFReader; - int64_t n; + // data ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutDataFile(hdr, &pWriter->fData); + code = tsdbWriteFile(pWriter->pDataFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; - // alloc - pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); - if (pDelFReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - // open impl - pDelFReader->pTsdb = pTsdb; - pDelFReader->fDel = *pFile; + // sma ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutSmaFile(hdr, &pWriter->fSma); + code = tsdbWriteFile(pWriter->pSmaFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; - tsdbDelFileName(pTsdb, pFile, fname); - pDelFReader->pReadH = taosOpenFile(fname, TD_FILE_READ); - if (pDelFReader->pReadH == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - taosMemoryFree(pDelFReader); - goto _err; - } + // stt ============== + memset(hdr, 0, TSDB_FHDR_SIZE); + tPutSttFile(hdr, &pWriter->fStt[pWriter->wSet.nSttF - 1]); + code = tsdbWriteFile(pWriter->pSttFD, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; -_exit: - *ppReader = pDelFReader; return code; _err: - tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; + tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { - int32_t code = 0; - SDelFReader *pReader = *ppReader; +int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + int64_t size; + int64_t n; - if (pReader) { - if (taosCloseFile(&pReader->pReadH) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree(pReader->aBuf[iBuf]); - } - taosMemoryFree(pReader); + // check + if (taosArrayGetSize(aBlockIdx) == 0) { + pHeadFile->offset = pHeadFile->size; + goto _exit; } - *ppReader = NULL; - -_exit: - return code; -} - -int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { - int32_t code = 0; - int64_t offset = pDelIdx->offset; - int64_t size = pDelIdx->size; - int64_t n; - - taosArrayClear(aDelData); - // seek - if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + // prepare + size = 0; + for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { + size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); } // alloc - code = tRealloc(&pReader->aBuf[0], size); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; - // read - n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // // decode + // build n = 0; - - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - while (n < size - sizeof(TSCKSUM)) { - SDelData delData; - n += tGetDelData(pReader->aBuf[0] + n, &delData); - - if (taosArrayPush(aDelData, &delData) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { + n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); } + ASSERT(n == size); + + // write + code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); + if (code) goto _err; - ASSERT(n == size - sizeof(TSCKSUM)); + // update + pHeadFile->offset = pHeadFile->size; + pHeadFile->size += size; +_exit: + // tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", + // TD_VID(pWriter->pTsdb->pVnode), + // pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); return code; _err: - tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { - int32_t code = 0; - int32_t n; - int64_t offset = pReader->fDel.offset; - int64_t size = pReader->fDel.size - offset; - - taosArrayClear(aDelIdx); +int32_t tsdbWriteDataBlk(SDataFWriter *pWriter, SMapData *mDataBlk, SBlockIdx *pBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = &pWriter->fHead; + int64_t size; + int64_t n; - // seek - if (taosLSeekFile(pReader->pReadH, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + ASSERT(mDataBlk->nItem > 0); // alloc - code = tRealloc(&pReader->aBuf[0], size); + size = tPutMapData(NULL, mDataBlk); + code = tRealloc(&pWriter->aBuf[0], size); if (code) goto _err; - // read - n = taosReadFile(pReader->pReadH, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // decode - n = 0; - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - - while (n < size - sizeof(TSCKSUM)) { - SDelIdx delIdx; - - n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); + // build + n = tPutMapData(pWriter->aBuf[0], mDataBlk); - if (taosArrayPush(aDelIdx, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } + // write + code = tsdbWriteFile(pWriter->pHeadFD, pHeadFile->size, pWriter->aBuf[0], size); + if (code) goto _err; - ASSERT(n == size - sizeof(TSCKSUM)); + // update + pBlockIdx->offset = pHeadFile->size; + pBlockIdx->size = size; + pHeadFile->size += size; + tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 + " size:%" PRId64 " nItem:%d", + TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, + pBlockIdx->offset, pBlockIdx->size, mDataBlk->nItem); return code; _err: - tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -// SDataFReader ==================================================== -struct SDataFReader { - STsdb *pTsdb; - SDFileSet *pSet; - TdFilePtr pHeadFD; - TdFilePtr pDataFD; - TdFilePtr pLastFD; - TdFilePtr pSmaFD; - - uint8_t *aBuf[3]; -}; - -int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { - int32_t code = 0; - SDataFReader *pReader; - char fname[TSDB_FILENAME_LEN]; +int32_t tsdbWriteSttBlk(SDataFWriter *pWriter, SArray *aSttBlk) { + int32_t code = 0; + SSttFile *pSttFile = &pWriter->fStt[pWriter->wSet.nSttF - 1]; + int64_t size; + int64_t n; - // alloc - pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); - if (pReader == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + // check + if (taosArrayGetSize(aSttBlk) == 0) { + pSttFile->offset = pSttFile->size; + goto _exit; } - pReader->pTsdb = pTsdb; - pReader->pSet = pSet; - // open impl - // head - tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); - pReader->pHeadFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pHeadFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + // size + size = 0; + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { + size += tPutSttBlk(NULL, taosArrayGet(aSttBlk, iBlockL)); } - // data - tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); - pReader->pDataFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pDataFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // alloc + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; - // last - tsdbLastFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pLastF, fname); - pReader->pLastFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + // encode + n = 0; + for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aSttBlk); iBlockL++) { + n += tPutSttBlk(pWriter->aBuf[0] + n, taosArrayGet(aSttBlk, iBlockL)); } - // sma - tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); - pReader->pSmaFD = taosOpenFile(fname, TD_FILE_READ); - if (pReader->pSmaFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // write + code = tsdbWriteFile(pWriter->pSttFD, pSttFile->size, pWriter->aBuf[0], size); + if (code) goto _err; - *ppReader = pReader; + // update + pSttFile->offset = pSttFile->size; + pSttFile->size += size; + +_exit: + tsdbTrace("vgId:%d tsdb write stt block, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), + pSttFile->offset, size); return code; _err: - tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppReader = NULL; + tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { +static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { int32_t code = 0; - if (*ppReader == NULL) goto _exit; - if (taosCloseFile(&(*ppReader)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + pSmaInfo->offset = 0; + pSmaInfo->size = 0; - if (taosCloseFile(&(*ppReader)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // encode + for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { + SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - if (taosCloseFile(&(*ppReader)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - if (taosCloseFile(&(*ppReader)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + SColumnDataAgg sma; + tsdbCalcColDataSMA(pColData, &sma); - for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree((*ppReader)->aBuf[iBuf]); + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); + if (code) goto _err; + pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); } - taosMemoryFree(*ppReader); + // write + if (pSmaInfo->size) { + code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size); + if (code) goto _err; + + code = tsdbWriteFile(pWriter->pSmaFD, pWriter->fSma.size, pWriter->aBuf[0], pSmaInfo->size); + if (code) goto _err; + + pSmaInfo->offset = pWriter->fSma.size; + pWriter->fSma.size += pSmaInfo->size; + } -_exit: - *ppReader = NULL; return code; _err: - tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { - int32_t code = 0; - int64_t offset = pReader->pSet->pHeadF->offset; - int64_t size = pReader->pSet->pHeadF->size - offset; - int64_t n; - uint32_t delimiter; +int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, + int8_t cmprAlg, int8_t toLast) { + int32_t code = 0; - taosArrayClear(aBlockIdx); - if (size == 0) { - goto _exit; - } + ASSERT(pBlockData->nRow > 0); - // alloc - code = tRealloc(&pReader->aBuf[0], size); + if (toLast) { + pBlkInfo->offset = pWriter->fStt[pWriter->wSet.nSttF - 1].size; + } else { + pBlkInfo->offset = pWriter->fData.size; + } + pBlkInfo->szBlock = 0; + pBlkInfo->szKey = 0; + + int32_t aBufN[4] = {0}; + code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); if (code) goto _err; - // seek - if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // write ================= + STsdbFD *pFD = toLast ? pWriter->pSttFD : pWriter->pDataFD; - // read - n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + pBlkInfo->szKey = aBufN[3] + aBufN[2]; + pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + int64_t offset = pBlkInfo->offset; + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[3], aBufN[3]); + if (code) goto _err; + offset += aBufN[3]; - // decode - n = 0; - n = tGetU32(pReader->aBuf[0] + n, &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[2], aBufN[2]); + if (code) goto _err; + offset += aBufN[2]; - while (n < size - sizeof(TSCKSUM)) { - SBlockIdx blockIdx; - n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); + if (aBufN[1]) { + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[1], aBufN[1]); + if (code) goto _err; + offset += aBufN[1]; + } - if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + if (aBufN[0]) { + code = tsdbWriteFile(pFD, offset, pWriter->aBuf[0], aBufN[0]); + if (code) goto _err; + } + + // update info + if (toLast) { + pWriter->fStt[pWriter->wSet.nSttF - 1].size += pBlkInfo->szBlock; + } else { + pWriter->fData.size += pBlkInfo->szBlock; } - ASSERT(n + sizeof(TSCKSUM) == size); + // ================= SMA ==================== + if (pSmaInfo) { + code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); + if (code) goto _err; + } _exit: + tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", + TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, + pBlkInfo->szBlock); return code; _err: - tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbReadBlockL(SDataFReader *pReader, SArray *aBlockL) { - int32_t code = 0; - int64_t offset = pReader->pSet->pLastF->offset; - int64_t size = pReader->pSet->pLastF->size - offset; - int64_t n; - uint32_t delimiter; +int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { + int32_t code = 0; + int64_t n; + int64_t size; + TdFilePtr pOutFD = NULL; + TdFilePtr PInFD = NULL; + int32_t szPage = pTsdb->pVnode->config.szPage; + char fNameFrom[TSDB_FILENAME_LEN]; + char fNameTo[TSDB_FILENAME_LEN]; - taosArrayClear(aBlockL); - if (size == 0) { - goto _exit; + // head + tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); + tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; } - - // alloc - code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; - - // seek - if (taosLSeekFile(pReader->pLastFD, offset, SEEK_SET) < 0) { + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { code = TAOS_SYSTEM_ERROR(errno); goto _err; } - - // read - n = taosReadFile(pReader->pLastFD, pReader->aBuf[0], size); + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pHeadF->size, szPage)); if (n < 0) { code = TAOS_SYSTEM_ERROR(errno); goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; + // data + tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); + tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); goto _err; } + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + n = taosFSendFile(pOutFD, PInFD, 0, LOGIC_TO_FILE_OFFSET(pSetFrom->pDataF->size, szPage)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); - // decode - n = 0; - n = tGetU32(pReader->aBuf[0] + n, &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); - - while (n < size - sizeof(TSCKSUM)) { - SBlockL blockl; - n += tGetBlockL(pReader->aBuf[0] + n, &blockl); + // sma + tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); + tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->pSmaF->size, szPage)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); - if (taosArrayPush(aBlockL, &blockl) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; + // stt + for (int8_t iStt = 0; iStt < pSetFrom->nSttF; iStt++) { + tsdbSttFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->aSttF[iStt], fNameFrom); + tsdbSttFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->aSttF[iStt], fNameTo); + pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); + if (pOutFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); + if (PInFD == NULL) { + code = TAOS_SYSTEM_ERROR(errno); + goto _err; + } + n = taosFSendFile(pOutFD, PInFD, 0, tsdbLogicToFileSize(pSetFrom->aSttF[iStt]->size, szPage)); + if (n < 0) { + code = TAOS_SYSTEM_ERROR(errno); goto _err; } + taosCloseFile(&pOutFD); + taosCloseFile(&PInFD); } - ASSERT(n + sizeof(TSCKSUM) == size); - -_exit: return code; _err: - tsdbError("vgId:%d read blockl failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbReadBlock(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mBlock) { - int32_t code = 0; - int64_t offset = pBlockIdx->offset; - int64_t size = pBlockIdx->size; - int64_t n; - int64_t tn; +// SDataFReader ==================================================== +int32_t tsdbDataFReaderOpen(SDataFReader **ppReader, STsdb *pTsdb, SDFileSet *pSet) { + int32_t code = 0; + SDataFReader *pReader; + int32_t szPage = pTsdb->pVnode->config.tsdbPageSize; + char fname[TSDB_FILENAME_LEN]; // alloc - code = tRealloc(&pReader->aBuf[0], size); - if (code) goto _err; - - // seek - if (taosLSeekFile(pReader->pHeadFD, offset, SEEK_SET) < 0) { - code = TAOS_SYSTEM_ERROR(errno); + pReader = (SDataFReader *)taosMemoryCalloc(1, sizeof(*pReader)); + if (pReader == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } + pReader->pTsdb = pTsdb; + pReader->pSet = pSet; - // read - n = taosReadFile(pReader->pHeadFD, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + // head + tsdbHeadFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pHeadF, fname); + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pHeadFD); + if (code) goto _err; - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; + // data + tsdbDataFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pDataF, fname); + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pDataFD); + if (code) goto _err; + + // sma + tsdbSmaFileName(pTsdb, pSet->diskId, pSet->fid, pSet->pSmaF, fname); + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->pSmaFD); + if (code) goto _err; + + // stt + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + tsdbSttFileName(pTsdb, pSet->diskId, pSet->fid, pSet->aSttF[iStt], fname); + code = tsdbOpenFile(fname, szPage, TD_FILE_READ, &pReader->aSttFD[iStt]); + if (code) goto _err; } - // decode - n = 0; + *ppReader = pReader; + return code; - uint32_t delimiter; - n += tGetU32(pReader->aBuf[0] + n, &delimiter); - ASSERT(delimiter == TSDB_FILE_DLMT); +_err: + tsdbError("vgId:%d, tsdb data file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppReader = NULL; + return code; +} - tn = tGetMapData(pReader->aBuf[0] + n, mBlock); - if (tn < 0) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; +int32_t tsdbDataFReaderClose(SDataFReader **ppReader) { + int32_t code = 0; + if (*ppReader == NULL) return code; + + // head + tsdbCloseFile(&(*ppReader)->pHeadFD); + + // data + tsdbCloseFile(&(*ppReader)->pDataFD); + + // sma + tsdbCloseFile(&(*ppReader)->pSmaFD); + + // stt + for (int32_t iStt = 0; iStt < TSDB_MAX_STT_TRIGGER; iStt++) { + if ((*ppReader)->aSttFD[iStt]) { + tsdbCloseFile(&(*ppReader)->aSttFD[iStt]); + } } - n += tn; - ASSERT(n + sizeof(TSCKSUM) == size); + for (int32_t iBuf = 0; iBuf < sizeof((*ppReader)->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree((*ppReader)->aBuf[iBuf]); + } + taosMemoryFree(*ppReader); + *ppReader = NULL; return code; _err: - tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, data file reader close failed since %s", TD_VID((*ppReader)->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbReadBlockSma(SDataFReader *pReader, SBlock *pBlock, SArray *aColumnDataAgg) { - int32_t code = 0; - SSmaInfo *pSmaInfo = &pBlock->smaInfo; - - ASSERT(pSmaInfo->size > 0); +int32_t tsdbReadBlockIdx(SDataFReader *pReader, SArray *aBlockIdx) { + int32_t code = 0; + SHeadFile *pHeadFile = pReader->pSet->pHeadF; + int64_t offset = pHeadFile->offset; + int64_t size = pHeadFile->size - offset; - taosArrayClear(aColumnDataAgg); + taosArrayClear(aBlockIdx); + if (size == 0) return code; // alloc - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // seek - int64_t n = taosLSeekFile(pReader->pSmaFD, pSmaInfo->offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < pSmaInfo->offset) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - // read - n = taosReadFile(pReader->pSmaFD, pReader->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } - - // check - if (!taosCheckChecksumWhole(pReader->aBuf[0], size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _err; - } + code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); + if (code) goto _err; // decode - n = 0; - while (n < pSmaInfo->size) { - SColumnDataAgg sma; + int64_t n = 0; + while (n < size) { + SBlockIdx blockIdx; + n += tGetBlockIdx(pReader->aBuf[0] + n, &blockIdx); - n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma); - if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { + if (taosArrayPush(aBlockIdx, &blockIdx) == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } + ASSERT(n == size); return code; _err: - tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, read block idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } -static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, int8_t fromLast, - SBlockData *pBlockData) { - int32_t code = 0; - - tBlockDataClear(pBlockData); +int32_t tsdbReadSttBlk(SDataFReader *pReader, int32_t iStt, SArray *aSttBlk) { + int32_t code = 0; + SSttFile *pSttFile = pReader->pSet->aSttF[iStt]; + int64_t offset = pSttFile->offset; + int64_t size = pSttFile->size - offset; - TdFilePtr pFD = fromLast ? pReader->pLastFD : pReader->pDataFD; + taosArrayClear(aSttBlk); + if (size == 0) return code; - // uid + version + tskey - code = tsdbReadAndCheck(pFD, pBlkInfo->offset, &pReader->aBuf[0], pBlkInfo->szKey, 1); + // alloc + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - SDiskDataHdr hdr; - uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); + + // read + code = tsdbReadFile(pReader->aSttFD[iStt], offset, pReader->aBuf[0], size); + if (code) goto _err; + + // decode + int64_t n = 0; + while (n < size) { + SSttBlk sttBlk; + n += tGetSttBlk(pReader->aBuf[0] + n, &sttBlk); + + if (taosArrayPush(aSttBlk, &sttBlk) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + ASSERT(n == size); + + return code; + +_err: + tsdbError("vgId:%d read stt blk failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbReadDataBlk(SDataFReader *pReader, SBlockIdx *pBlockIdx, SMapData *mDataBlk) { + int32_t code = 0; + int64_t offset = pBlockIdx->offset; + int64_t size = pBlockIdx->size; + + // alloc + code = tRealloc(&pReader->aBuf[0], size); + if (code) goto _err; + + // read + code = tsdbReadFile(pReader->pHeadFD, offset, pReader->aBuf[0], size); + if (code) goto _err; + + // decode + int64_t n = tGetMapData(pReader->aBuf[0], mDataBlk); + if (n < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + ASSERT(n == size); + + return code; + +_err: + tsdbError("vgId:%d, read block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbReadBlockSma(SDataFReader *pReader, SDataBlk *pDataBlk, SArray *aColumnDataAgg) { + int32_t code = 0; + SSmaInfo *pSmaInfo = &pDataBlk->smaInfo; + + ASSERT(pSmaInfo->size > 0); + + taosArrayClear(aColumnDataAgg); + + // alloc + code = tRealloc(&pReader->aBuf[0], pSmaInfo->size); + if (code) goto _err; + + // read + code = tsdbReadFile(pReader->pSmaFD, pSmaInfo->offset, pReader->aBuf[0], pSmaInfo->size); + if (code) goto _err; + + // decode + int32_t n = 0; + while (n < pSmaInfo->size) { + SColumnDataAgg sma; + n += tGetColumnDataAgg(pReader->aBuf[0] + n, &sma); + + if (taosArrayPush(aColumnDataAgg, &sma) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + ASSERT(n == pSmaInfo->size); + return code; + +_err: + tsdbError("vgId:%d tsdb read block sma failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo, SBlockData *pBlockData) { + int32_t code = 0; + + tBlockDataClear(pBlockData); + + STsdbFD *pFD = pReader->pDataFD; + + // uid + version + tskey + code = tRealloc(&pReader->aBuf[0], pBlkInfo->szKey); + if (code) goto _err; + + code = tsdbReadFile(pFD, pBlkInfo->offset, pReader->aBuf[0], pBlkInfo->szKey); + if (code) goto _err; + + SDiskDataHdr hdr; + uint8_t *p = pReader->aBuf[0] + tGetDiskDataHdr(pReader->aBuf[0], &hdr); ASSERT(hdr.delimiter == TSDB_FILE_DLMT); ASSERT(pBlockData->suid == hdr.suid); @@ -782,14 +972,18 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo if (code) goto _err; p += hdr.szKey; - ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey - sizeof(TSCKSUM)); + ASSERT(p - pReader->aBuf[0] == pBlkInfo->szKey); // read and decode columns if (taosArrayGetSize(pBlockData->aIdx) == 0) goto _exit; if (hdr.szBlkCol > 0) { int64_t offset = pBlkInfo->offset + pBlkInfo->szKey; - code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[0], hdr.szBlkCol + sizeof(TSCKSUM), 1); + + code = tRealloc(&pReader->aBuf[0], hdr.szBlkCol); + if (code) goto _err; + + code = tsdbReadFile(pFD, offset, pReader->aBuf[0], hdr.szBlkCol); if (code) goto _err; } @@ -827,10 +1021,13 @@ static int32_t tsdbReadBlockDataImpl(SDataFReader *pReader, SBlockInfo *pBlkInfo } } else { // decode from binary - int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + sizeof(TSCKSUM) + pBlockCol->offset; - int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); + int64_t offset = pBlkInfo->offset + pBlkInfo->szKey + hdr.szBlkCol + pBlockCol->offset; + int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue; + + code = tRealloc(&pReader->aBuf[1], size); + if (code) goto _err; - code = tsdbReadAndCheck(pFD, offset, &pReader->aBuf[1], size, 0); + code = tsdbReadFile(pFD, offset, pReader->aBuf[1], size); if (code) goto _err; code = tsdbDecmprColData(pReader->aBuf[1], pBlockCol, hdr.cmprAlg, hdr.nRow, pColData, &pReader->aBuf[2]); @@ -847,13 +1044,36 @@ _err: return code; } -int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBlockData) { +int32_t tsdbReadDataBlockEx(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { + int32_t code = 0; + SBlockInfo *pBlockInfo = &pDataBlk->aSubBlock[0]; + + // alloc + code = tRealloc(&pReader->aBuf[0], pBlockInfo->szBlock); + if (code) goto _err; + + // read + code = tsdbReadFile(pReader->pDataFD, pBlockInfo->offset, pReader->aBuf[0], pBlockInfo->szBlock); + if (code) goto _err; + + // decmpr + code = tDecmprBlockData(pReader->aBuf[0], pBlockInfo->szBlock, pBlockData, &pReader->aBuf[1]); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d tsdb read data block ex failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + return code; +} + +int32_t tsdbReadDataBlock(SDataFReader *pReader, SDataBlk *pDataBlk, SBlockData *pBlockData) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[0], 0, pBlockData); + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[0], pBlockData); if (code) goto _err; - if (pBlock->nSubBlock > 1) { + if (pDataBlk->nSubBlock > 1) { SBlockData bData1; SBlockData bData2; @@ -867,8 +1087,8 @@ int32_t tsdbReadDataBlock(SDataFReader *pReader, SBlock *pBlock, SBlockData *pBl tBlockDataInitEx(&bData1, pBlockData); tBlockDataInitEx(&bData2, pBlockData); - for (int32_t iSubBlock = 1; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - code = tsdbReadBlockDataImpl(pReader, &pBlock->aSubBlock[iSubBlock], 0, &bData1); + for (int32_t iSubBlock = 1; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { + code = tsdbReadBlockDataImpl(pReader, &pDataBlk->aSubBlock[iSubBlock], &bData1); if (code) { tBlockDataDestroy(&bData1, 1); tBlockDataDestroy(&bData2, 1); @@ -901,325 +1121,142 @@ _err: return code; } -int32_t tsdbReadLastBlock(SDataFReader *pReader, SBlockL *pBlockL, SBlockData *pBlockData) { +int32_t tsdbReadSttBlock(SDataFReader *pReader, int32_t iStt, SSttBlk *pSttBlk, SBlockData *pBlockData) { int32_t code = 0; - code = tsdbReadBlockDataImpl(pReader, &pBlockL->bInfo, 1, pBlockData); + // alloc + code = tRealloc(&pReader->aBuf[0], pSttBlk->bInfo.szBlock); + if (code) goto _err; + + // read + code = tsdbReadFile(pReader->aSttFD[iStt], pSttBlk->bInfo.offset, pReader->aBuf[0], pSttBlk->bInfo.szBlock); + if (code) goto _err; + + // decmpr + code = tDecmprBlockData(pReader->aBuf[0], pSttBlk->bInfo.szBlock, pBlockData, &pReader->aBuf[1]); if (code) goto _err; return code; _err: - tsdbError("vgId:%d tsdb read last block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d tsdb read stt block failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } -// SDataFWriter ==================================================== -int32_t tsdbDataFWriterOpen(SDataFWriter **ppWriter, STsdb *pTsdb, SDFileSet *pSet) { - int32_t code = 0; - int32_t flag; - int64_t n; - SDataFWriter *pWriter = NULL; - char fname[TSDB_FILENAME_LEN]; - char hdr[TSDB_FHDR_SIZE] = {0}; +// SDelFWriter ==================================================== +int32_t tsdbDelFWriterOpen(SDelFWriter **ppWriter, SDelFile *pFile, STsdb *pTsdb) { + int32_t code = 0; + char fname[TSDB_FILENAME_LEN]; + uint8_t hdr[TSDB_FHDR_SIZE] = {0}; + SDelFWriter *pDelFWriter; + int64_t n; // alloc - pWriter = taosMemoryCalloc(1, sizeof(*pWriter)); - if (pWriter == NULL) { + pDelFWriter = (SDelFWriter *)taosMemoryCalloc(1, sizeof(*pDelFWriter)); + if (pDelFWriter == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - if (code) goto _err; - pWriter->pTsdb = pTsdb; - pWriter->wSet = (SDFileSet){.diskId = pSet->diskId, - .fid = pSet->fid, - .pHeadF = &pWriter->fHead, - .pDataF = &pWriter->fData, - .pLastF = &pWriter->fLast, - .pSmaF = &pWriter->fSma}; - pWriter->fHead = *pSet->pHeadF; - pWriter->fData = *pSet->pDataF; - pWriter->fLast = *pSet->pLastF; - pWriter->fSma = *pSet->pSmaF; - - // head - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - tsdbHeadFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fHead, fname); - pWriter->pHeadFD = taosOpenFile(fname, flag); - if (pWriter->pHeadFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - ASSERT(n == TSDB_FHDR_SIZE); - - pWriter->fHead.size += TSDB_FHDR_SIZE; + pDelFWriter->pTsdb = pTsdb; + pDelFWriter->fDel = *pFile; - // data - if (pWriter->fData.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_WRITE; - } - tsdbDataFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fData, fname); - pWriter->pDataFD = taosOpenFile(fname, flag); - if (pWriter->pDataFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (pWriter->fData.size == 0) { - n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + tsdbDelFileName(pTsdb, pFile, fname); + code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ | TD_FILE_WRITE | TD_FILE_CREATE, + &pDelFWriter->pWriteH); + if (code) goto _err; - pWriter->fData.size += TSDB_FHDR_SIZE; - } else { - n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // update header + code = tsdbWriteFile(pDelFWriter->pWriteH, 0, hdr, TSDB_FHDR_SIZE); + if (code) goto _err; - ASSERT(n == pWriter->fData.size); - } + pDelFWriter->fDel.size = TSDB_FHDR_SIZE; + pDelFWriter->fDel.offset = 0; - // last - if (pWriter->fLast.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_WRITE; - } - tsdbLastFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fLast, fname); - pWriter->pLastFD = taosOpenFile(fname, flag); - if (pWriter->pLastFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - if (pWriter->fLast.size == 0) { - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + *ppWriter = pDelFWriter; + return code; - pWriter->fLast.size += TSDB_FHDR_SIZE; - } else { - n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } +_err: + tsdbError("vgId:%d, failed to open del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppWriter = NULL; + return code; +} - ASSERT(n == pWriter->fLast.size); - } +int32_t tsdbDelFWriterClose(SDelFWriter **ppWriter, int8_t sync) { + int32_t code = 0; + SDelFWriter *pWriter = *ppWriter; + STsdb *pTsdb = pWriter->pTsdb; - // sma - if (pWriter->fSma.size == 0) { - flag = TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC; - } else { - flag = TD_FILE_WRITE; - } - tsdbSmaFileName(pTsdb, pWriter->wSet.diskId, pWriter->wSet.fid, &pWriter->fSma, fname); - pWriter->pSmaFD = taosOpenFile(fname, flag); - if (pWriter->pSmaFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + // sync + if (sync) { + code = tsdbFsyncFile(pWriter->pWriteH); + if (code) goto _err; } - if (pWriter->fSma.size == 0) { - n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - pWriter->fSma.size += TSDB_FHDR_SIZE; - } else { - n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_END); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // close + tsdbCloseFile(&pWriter->pWriteH); - ASSERT(n == pWriter->fSma.size); + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pWriter->aBuf[iBuf]); } + taosMemoryFree(pWriter); - *ppWriter = pWriter; + *ppWriter = NULL; return code; _err: - tsdbError("vgId:%d, tsdb data file writer open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - *ppWriter = NULL; + tsdbError("vgId:%d, failed to close del file writer since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbDataFWriterClose(SDataFWriter **ppWriter, int8_t sync) { +int32_t tsdbWriteDelData(SDelFWriter *pWriter, SArray *aDelData, SDelIdx *pDelIdx) { int32_t code = 0; - STsdb *pTsdb = NULL; + int64_t size; + int64_t n; - if (*ppWriter == NULL) goto _exit; + // prepare + size = 0; + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { + size += tPutDelData(NULL, taosArrayGet(aDelData, iDelData)); + } - pTsdb = (*ppWriter)->pTsdb; - if (sync) { - if (taosFsyncFile((*ppWriter)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // alloc + code = tRealloc(&pWriter->aBuf[0], size); + if (code) goto _err; - if (taosFsyncFile((*ppWriter)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosFsyncFile((*ppWriter)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosFsyncFile((*ppWriter)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } - - if (taosCloseFile(&(*ppWriter)->pHeadFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pDataFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pLastFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - if (taosCloseFile(&(*ppWriter)->pSmaFD) < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - for (int32_t iBuf = 0; iBuf < sizeof((*ppWriter)->aBuf) / sizeof(uint8_t *); iBuf++) { - tFree((*ppWriter)->aBuf[iBuf]); - } - taosMemoryFree(*ppWriter); -_exit: - *ppWriter = NULL; - return code; - -_err: - tsdbError("vgId:%d, data file writer close failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbUpdateDFileSetHeader(SDataFWriter *pWriter) { - int32_t code = 0; - int64_t n; - char hdr[TSDB_FHDR_SIZE]; - - // head ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutHeadFile(hdr, &pWriter->fHead); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pHeadFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pHeadFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // data ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutDataFile(hdr, &pWriter->fData); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pDataFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pDataFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // last ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutLastFile(hdr, &pWriter->fLast); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pLastFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosWriteFile(pWriter->pLastFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; + // build + n = 0; + for (int32_t iDelData = 0; iDelData < taosArrayGetSize(aDelData); iDelData++) { + n += tPutDelData(pWriter->aBuf[0] + n, taosArrayGet(aDelData, iDelData)); } + ASSERT(n == size); - // sma ============== - memset(hdr, 0, TSDB_FHDR_SIZE); - tPutSmaFile(hdr, &pWriter->fSma); - taosCalcChecksumAppend(0, hdr, TSDB_FHDR_SIZE); - - n = taosLSeekFile(pWriter->pSmaFD, 0, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // write + code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); + if (code) goto _err; - n = taosWriteFile(pWriter->pSmaFD, hdr, TSDB_FHDR_SIZE); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // update + pDelIdx->offset = pWriter->fDel.size; + pDelIdx->size = size; + pWriter->fDel.size += size; return code; _err: - tsdbError("vgId:%d, update DFileSet header failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, failed to write del data since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size = 0; - int64_t n; - - // check - if (taosArrayGetSize(aBlockIdx) == 0) { - pHeadFile->offset = pHeadFile->size; - goto _exit; - } +int32_t tsdbWriteDelIdx(SDelFWriter *pWriter, SArray *aDelIdx) { + int32_t code = 0; + int64_t size; + int64_t n; + SDelIdx *pDelIdx; // prepare - size = sizeof(uint32_t); - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - size += tPutBlockIdx(NULL, taosArrayGet(aBlockIdx, iBlockIdx)); + size = 0; + for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { + size += tPutDelIdx(NULL, taosArrayGet(aDelIdx, iDelIdx)); } - size += sizeof(TSCKSUM); // alloc code = tRealloc(&pWriter->aBuf[0], size); @@ -1227,383 +1264,170 @@ int32_t tsdbWriteBlockIdx(SDataFWriter *pWriter, SArray *aBlockIdx) { // build n = 0; - n = tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iBlockIdx = 0; iBlockIdx < taosArrayGetSize(aBlockIdx); iBlockIdx++) { - n += tPutBlockIdx(pWriter->aBuf[0] + n, taosArrayGet(aBlockIdx, iBlockIdx)); + for (int32_t iDelIdx = 0; iDelIdx < taosArrayGetSize(aDelIdx); iDelIdx++) { + n += tPutDelIdx(pWriter->aBuf[0] + n, taosArrayGet(aDelIdx, iDelIdx)); } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); + ASSERT(n == size); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + code = tsdbWriteFile(pWriter->pWriteH, pWriter->fDel.size, pWriter->aBuf[0], size); + if (code) goto _err; // update - pHeadFile->offset = pHeadFile->size; - pHeadFile->size += size; + pWriter->fDel.offset = pWriter->fDel.size; + pWriter->fDel.size += size; -_exit: - tsdbTrace("vgId:%d write block idx, offset:%" PRId64 " size:%" PRId64 " nBlockIdx:%d", TD_VID(pWriter->pTsdb->pVnode), - pHeadFile->offset, size, taosArrayGetSize(aBlockIdx)); return code; _err: - tsdbError("vgId:%d, write block idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, write del idx failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbWriteBlock(SDataFWriter *pWriter, SMapData *mBlock, SBlockIdx *pBlockIdx) { - int32_t code = 0; - SHeadFile *pHeadFile = &pWriter->fHead; - int64_t size; - int64_t n; - - ASSERT(mBlock->nItem > 0); - - // alloc - size = sizeof(uint32_t) + tPutMapData(NULL, mBlock) + sizeof(TSCKSUM); - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; +int32_t tsdbUpdateDelFileHdr(SDelFWriter *pWriter) { + int32_t code = 0; + char hdr[TSDB_FHDR_SIZE] = {0}; + int64_t size = TSDB_FHDR_SIZE; + int64_t n; // build - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - n += tPutMapData(pWriter->aBuf[0] + n, mBlock); - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); + tPutDelFile(hdr, &pWriter->fDel); // write - n = taosWriteFile(pWriter->pHeadFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - // update - pBlockIdx->offset = pHeadFile->size; - pBlockIdx->size = size; - pHeadFile->size += size; + code = tsdbWriteFile(pWriter->pWriteH, 0, hdr, size); + if (code) goto _err; - tsdbTrace("vgId:%d, write block, file ID:%d commit ID:%d suid:%" PRId64 " uid:%" PRId64 " offset:%" PRId64 - " size:%" PRId64 " nItem:%d", - TD_VID(pWriter->pTsdb->pVnode), pWriter->wSet.fid, pHeadFile->commitID, pBlockIdx->suid, pBlockIdx->uid, - pBlockIdx->offset, pBlockIdx->size, mBlock->nItem); return code; _err: - tsdbError("vgId:%d, write block failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, update del file hdr failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); return code; } +// SDelFReader ==================================================== +struct SDelFReader { + STsdb *pTsdb; + SDelFile fDel; + STsdbFD *pReadH; + uint8_t *aBuf[1]; +}; -int32_t tsdbWriteBlockL(SDataFWriter *pWriter, SArray *aBlockL) { - int32_t code = 0; - SLastFile *pLastFile = &pWriter->fLast; - int64_t size; - int64_t n; - - // check - if (taosArrayGetSize(aBlockL) == 0) { - pLastFile->offset = pLastFile->size; - goto _exit; - } - - // size - size = sizeof(uint32_t); // TSDB_FILE_DLMT - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - size += tPutBlockL(NULL, taosArrayGet(aBlockL, iBlockL)); - } - size += sizeof(TSCKSUM); +int32_t tsdbDelFReaderOpen(SDelFReader **ppReader, SDelFile *pFile, STsdb *pTsdb) { + int32_t code = 0; + char fname[TSDB_FILENAME_LEN]; + SDelFReader *pDelFReader; + int64_t n; // alloc - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; - - // encode - n = 0; - n += tPutU32(pWriter->aBuf[0] + n, TSDB_FILE_DLMT); - for (int32_t iBlockL = 0; iBlockL < taosArrayGetSize(aBlockL); iBlockL++) { - n += tPutBlockL(pWriter->aBuf[0] + n, taosArrayGet(aBlockL, iBlockL)); - } - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); - - ASSERT(n + sizeof(TSCKSUM) == size); - - // write - n = taosWriteFile(pWriter->pLastFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); + pDelFReader = (SDelFReader *)taosMemoryCalloc(1, sizeof(*pDelFReader)); + if (pDelFReader == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - // update - pLastFile->offset = pLastFile->size; - pLastFile->size += size; + // open impl + pDelFReader->pTsdb = pTsdb; + pDelFReader->fDel = *pFile; -_exit: - tsdbTrace("vgId:%d tsdb write blockl, loffset:%" PRId64 " size:%" PRId64, TD_VID(pWriter->pTsdb->pVnode), - pLastFile->offset, size); + tsdbDelFileName(pTsdb, pFile, fname); + code = tsdbOpenFile(fname, pTsdb->pVnode->config.tsdbPageSize, TD_FILE_READ, &pDelFReader->pReadH); + if (code) goto _err; + + *ppReader = pDelFReader; return code; _err: - tsdbError("vgId:%d tsdb write blockl failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, del file reader open failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + *ppReader = NULL; return code; } -static void tsdbUpdateBlockInfo(SBlockData *pBlockData, SBlock *pBlock) { - for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { - TSDBKEY key = {.ts = pBlockData->aTSKEY[iRow], .version = pBlockData->aVersion[iRow]}; - - if (iRow == 0) { - if (tsdbKeyCmprFn(&pBlock->minKey, &key) > 0) { - pBlock->minKey = key; - } - } else { - if (pBlockData->aTSKEY[iRow] == pBlockData->aTSKEY[iRow - 1]) { - pBlock->hasDup = 1; - } - } +int32_t tsdbDelFReaderClose(SDelFReader **ppReader) { + int32_t code = 0; + SDelFReader *pReader = *ppReader; - if (iRow == pBlockData->nRow - 1 && tsdbKeyCmprFn(&pBlock->maxKey, &key) < 0) { - pBlock->maxKey = key; + if (pReader) { + tsdbCloseFile(&pReader->pReadH); + for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(uint8_t *); iBuf++) { + tFree(pReader->aBuf[iBuf]); } - - pBlock->minVer = TMIN(pBlock->minVer, key.version); - pBlock->maxVer = TMAX(pBlock->maxVer, key.version); + taosMemoryFree(pReader); } - pBlock->nRow += pBlockData->nRow; + *ppReader = NULL; + +_exit: + return code; } -static int32_t tsdbWriteBlockSma(SDataFWriter *pWriter, SBlockData *pBlockData, SSmaInfo *pSmaInfo) { +int32_t tsdbReadDelData(SDelFReader *pReader, SDelIdx *pDelIdx, SArray *aDelData) { int32_t code = 0; + int64_t offset = pDelIdx->offset; + int64_t size = pDelIdx->size; + int64_t n; - pSmaInfo->offset = 0; - pSmaInfo->size = 0; - - // encode - for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { - SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - - if ((!pColData->smaOn) || IS_VAR_DATA_TYPE(pColData->type)) continue; - - SColumnDataAgg sma; - tsdbCalcColDataSMA(pColData, &sma); - - code = tRealloc(&pWriter->aBuf[0], pSmaInfo->size + tPutColumnDataAgg(NULL, &sma)); - if (code) goto _err; - pSmaInfo->size += tPutColumnDataAgg(pWriter->aBuf[0] + pSmaInfo->size, &sma); - } + taosArrayClear(aDelData); - // write - if (pSmaInfo->size) { - int32_t size = pSmaInfo->size + sizeof(TSCKSUM); + // alloc + code = tRealloc(&pReader->aBuf[0], size); + if (code) goto _err; - code = tRealloc(&pWriter->aBuf[0], size); - if (code) goto _err; + // read + code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size); + if (code) goto _err; - taosCalcChecksumAppend(0, pWriter->aBuf[0], size); + // // decode + n = 0; + while (n < size) { + SDelData delData; + n += tGetDelData(pReader->aBuf[0] + n, &delData); - int64_t n = taosWriteFile(pWriter->pSmaFD, pWriter->aBuf[0], size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); + if (taosArrayPush(aDelData, &delData) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - - pSmaInfo->offset = pWriter->fSma.size; - pWriter->fSma.size += size; } + ASSERT(n == size); return code; _err: - tsdbError("vgId:%d tsdb write block sma failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, read del data failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; } -int32_t tsdbWriteBlockData(SDataFWriter *pWriter, SBlockData *pBlockData, SBlockInfo *pBlkInfo, SSmaInfo *pSmaInfo, - int8_t cmprAlg, int8_t toLast) { +int32_t tsdbReadDelIdx(SDelFReader *pReader, SArray *aDelIdx) { int32_t code = 0; + int32_t n; + int64_t offset = pReader->fDel.offset; + int64_t size = pReader->fDel.size - offset; - ASSERT(pBlockData->nRow > 0); - - pBlkInfo->offset = toLast ? pWriter->fLast.size : pWriter->fData.size; - pBlkInfo->szBlock = 0; - pBlkInfo->szKey = 0; + taosArrayClear(aDelIdx); - int32_t aBufN[4] = {0}; - code = tCmprBlockData(pBlockData, cmprAlg, NULL, NULL, pWriter->aBuf, aBufN); + // alloc + code = tRealloc(&pReader->aBuf[0], size); if (code) goto _err; - // write ================= - TdFilePtr pFD = toLast ? pWriter->pLastFD : pWriter->pDataFD; - - pBlkInfo->szKey = aBufN[3] + aBufN[2]; - pBlkInfo->szBlock = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; - - int64_t n = taosWriteFile(pFD, pWriter->aBuf[3], aBufN[3]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // read + code = tsdbReadFile(pReader->pReadH, offset, pReader->aBuf[0], size); + if (code) goto _err; - n = taosWriteFile(pFD, pWriter->aBuf[2], aBufN[2]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } + // decode + n = 0; + while (n < size) { + SDelIdx delIdx; - if (aBufN[1]) { - n = taosWriteFile(pFD, pWriter->aBuf[1], aBufN[1]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - } + n += tGetDelIdx(pReader->aBuf[0] + n, &delIdx); - if (aBufN[0]) { - n = taosWriteFile(pFD, pWriter->aBuf[0], aBufN[0]); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); + if (taosArrayPush(aDelIdx, &delIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } } - // update info - if (toLast) { - pWriter->fLast.size += pBlkInfo->szBlock; - } else { - pWriter->fData.size += pBlkInfo->szBlock; - } - - // ================= SMA ==================== - if (pSmaInfo) { - code = tsdbWriteBlockSma(pWriter, pBlockData, pSmaInfo); - if (code) goto _err; - } - -_exit: - tsdbTrace("vgId:%d tsdb write block data, suid:%" PRId64 " uid:%" PRId64 " nRow:%d, offset:%" PRId64 " size:%d", - TD_VID(pWriter->pTsdb->pVnode), pBlockData->suid, pBlockData->uid, pBlockData->nRow, pBlkInfo->offset, - pBlkInfo->szBlock); - return code; - -_err: - tsdbError("vgId:%d tsdb write block data failed since %s", TD_VID(pWriter->pTsdb->pVnode), tstrerror(code)); - return code; -} - -int32_t tsdbDFileSetCopy(STsdb *pTsdb, SDFileSet *pSetFrom, SDFileSet *pSetTo) { - int32_t code = 0; - int64_t n; - int64_t size; - TdFilePtr pOutFD = NULL; // TODO - TdFilePtr PInFD = NULL; // TODO - char fNameFrom[TSDB_FILENAME_LEN]; - char fNameTo[TSDB_FILENAME_LEN]; - - // head - tsdbHeadFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pHeadF, fNameFrom); - tsdbHeadFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pHeadF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pHeadF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // data - tsdbDataFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pDataF, fNameFrom); - tsdbDataFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pDataF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pDataF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // last - tsdbLastFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pLastF, fNameFrom); - tsdbLastFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pLastF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pLastF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); - - // sma - tsdbSmaFileName(pTsdb, pSetFrom->diskId, pSetFrom->fid, pSetFrom->pSmaF, fNameFrom); - tsdbSmaFileName(pTsdb, pSetTo->diskId, pSetTo->fid, pSetTo->pSmaF, fNameTo); - - pOutFD = taosOpenFile(fNameTo, TD_FILE_WRITE | TD_FILE_CREATE | TD_FILE_TRUNC); - if (pOutFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - PInFD = taosOpenFile(fNameFrom, TD_FILE_READ); - if (PInFD == NULL) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - - n = taosFSendFile(pOutFD, PInFD, 0, pSetFrom->pSmaF->size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _err; - } - taosCloseFile(&pOutFD); - taosCloseFile(&PInFD); + ASSERT(n == size); return code; _err: - tsdbError("vgId:%d, tsdb DFileSet copy failed since %s", TD_VID(pTsdb->pVnode), tstrerror(code)); + tsdbError("vgId:%d, read del idx failed since %s", TD_VID(pReader->pTsdb->pVnode), tstrerror(code)); return code; -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbRetention.c b/source/dnode/vnode/src/tsdb/tsdbRetention.c index a30b9154ab07084adc31c65089d223ac728445ae..eb4151079b9ccf4bae1f1e03b4e240e9e6f18214 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRetention.c +++ b/source/dnode/vnode/src/tsdb/tsdbRetention.c @@ -16,9 +16,19 @@ #include "tsdb.h" static bool tsdbShouldDoRetention(STsdb *pTsdb, int64_t now) { + STsdbKeepCfg *keepCfg = &pTsdb->keepCfg; + + if ((keepCfg->keep0 == keepCfg->keep1) && (keepCfg->keep1 == keepCfg->keep2)) { + return false; + } + + if (tfsGetLevel(pTsdb->pVnode->pTfs) <= 1) { + return false; + } + for (int32_t iSet = 0; iSet < taosArrayGetSize(pTsdb->fs.aDFileSet); iSet++) { SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); - int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); + int32_t expLevel = tsdbFidLevel(pSet->fid, keepCfg, now); SDiskID did; if (expLevel == pSet->diskId.level) continue; @@ -53,18 +63,19 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { if (code) goto _err; for (int32_t iSet = 0; iSet < taosArrayGetSize(fs.aDFileSet); iSet++) { - SDFileSet *pSet = (SDFileSet *)taosArrayGet(pTsdb->fs.aDFileSet, iSet); + SDFileSet *pSet = (SDFileSet *)taosArrayGet(fs.aDFileSet, iSet); int32_t expLevel = tsdbFidLevel(pSet->fid, &pTsdb->keepCfg, now); SDiskID did; if (expLevel < 0) { taosMemoryFree(pSet->pHeadF); taosMemoryFree(pSet->pDataF); - taosMemoryFree(pSet->pLastF); + taosMemoryFree(pSet->aSttF[0]); taosMemoryFree(pSet->pSmaF); taosArrayRemove(fs.aDFileSet, iSet); iSet--; } else { + if (expLevel == 0) continue; if (tfsAllocDisk(pTsdb->pVnode->pTfs, expLevel, &did) < 0) { code = terrno; goto _exit; @@ -82,8 +93,6 @@ int32_t tsdbDoRetention(STsdb *pTsdb, int64_t now) { code = tsdbFSUpsertFSet(&fs, &fSet); if (code) goto _err; } - - /* code */ } // do change fs diff --git a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c index ab2b2b617a3d36dbc2c86c2a2207cffac8f087f6..99e88a442c16e77f2db2ca752eb54f4e120532f8 100644 --- a/source/dnode/vnode/src/tsdb/tsdbSnapshot.c +++ b/source/dnode/vnode/src/tsdb/tsdbSnapshot.c @@ -16,6 +16,29 @@ #include "tsdb.h" // STsdbSnapReader ======================================== +typedef enum { SNAP_DATA_FILE_ITER = 0, SNAP_STT_FILE_ITER } EFIterT; +typedef struct { + SRBTreeNode n; + SRowInfo rInfo; + EFIterT type; + union { + struct { + SArray* aBlockIdx; + int32_t iBlockIdx; + SBlockIdx* pBlockIdx; + SMapData mBlock; + int32_t iBlock; + }; // .data file + struct { + int32_t iStt; + SArray* aSttBlk; + int32_t iSttBlk; + }; // .stt file + }; + SBlockData bData; + int32_t iRow; +} SFDataIter; + struct STsdbSnapReader { STsdb* pTsdb; int64_t sver; @@ -26,146 +49,301 @@ struct STsdbSnapReader { int8_t dataDone; int32_t fid; SDataFReader* pDataFReader; - SArray* aBlockIdx; // SArray - SArray* aBlockL; // SArray - SBlockIdx* pBlockIdx; - SBlockL* pBlockL; - - int32_t iBlockIdx; - int32_t iBlockL; - SMapData mBlock; // SMapData - int32_t iBlock; - SBlockData oBlockData; - SBlockData nBlockData; + SFDataIter* pIter; + SRBTree rbt; + SFDataIter aFDataIter[TSDB_MAX_STT_TRIGGER + 1]; + SBlockData bData; + SSkmInfo skmTable; // for del file int8_t delDone; SDelFReader* pDelFReader; SArray* aDelIdx; // SArray int32_t iDelIdx; SArray* aDelData; // SArray + uint8_t* aBuf[5]; }; -static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { +extern int32_t tRowInfoCmprFn(const void* p1, const void* p2); +extern int32_t tsdbReadDataBlockEx(SDataFReader* pReader, SDataBlk* pDataBlk, SBlockData* pBlockData); +extern int32_t tsdbUpdateTableSchema(SMeta* pMeta, int64_t suid, int64_t uid, SSkmInfo* pSkmInfo); + +static int32_t tsdbSnapReadOpenFile(STsdbSnapReader* pReader) { int32_t code = 0; - STsdb* pTsdb = pReader->pTsdb; - while (true) { - if (pReader->pDataFReader == NULL) { - // next - SDFileSet dFileSet = {.fid = pReader->fid}; - SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); - if (pSet == NULL) goto _exit; - pReader->fid = pSet->fid; - - // load - code = tsdbDataFReaderOpen(&pReader->pDataFReader, pTsdb, pSet); - if (code) goto _err; + SDFileSet dFileSet = {.fid = pReader->fid}; + SDFileSet* pSet = taosArraySearch(pReader->fs.aDFileSet, &dFileSet, tDFileSetCmprFn, TD_GT); + if (pSet == NULL) return code; - code = tsdbReadBlockIdx(pReader->pDataFReader, pReader->aBlockIdx); - if (code) goto _err; + pReader->fid = pSet->fid; + code = tsdbDataFReaderOpen(&pReader->pDataFReader, pReader->pTsdb, pSet); + if (code) goto _err; + + pReader->pIter = NULL; + tRBTreeCreate(&pReader->rbt, tRowInfoCmprFn); + + // .data file + SFDataIter* pIter = &pReader->aFDataIter[0]; + pIter->type = SNAP_DATA_FILE_ITER; + + code = tsdbReadBlockIdx(pReader->pDataFReader, pIter->aBlockIdx); + if (code) goto _err; - code = tsdbReadBlockL(pReader->pDataFReader, pReader->aBlockL); + for (pIter->iBlockIdx = 0; pIter->iBlockIdx < taosArrayGetSize(pIter->aBlockIdx); pIter->iBlockIdx++) { + pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx); + + code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock); + if (code) goto _err; + + for (pIter->iBlock = 0; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk); + + if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue; + + code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData); if (code) goto _err; - // init - pReader->iBlockIdx = 0; - if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { - pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + ASSERT(pIter->pBlockIdx->suid == pIter->bData.suid); + ASSERT(pIter->pBlockIdx->uid == pIter->bData.uid); - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); - if (code) goto _err; + for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { + int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; - pReader->iBlock = 0; - } else { - pReader->pBlockIdx = NULL; + if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.suid = pIter->pBlockIdx->suid; + pIter->rInfo.uid = pIter->pBlockIdx->uid; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + goto _add_iter_and_break; + } } + } - pReader->iBlockL = 0; - while (true) { - if (pReader->iBlockL >= taosArrayGetSize(pReader->aBlockL)) { - pReader->pBlockL = NULL; - break; - } + continue; - pReader->pBlockL = (SBlockL*)taosArrayGet(pReader->aBlockL, pReader->iBlockL); - if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { - // TODO - break; - } + _add_iter_and_break: + tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter); + break; + } - pReader->iBlockL++; - } + // .stt file + pIter = &pReader->aFDataIter[1]; + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + pIter->type = SNAP_STT_FILE_ITER; + pIter->iStt = iStt; - tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pTsdb->pVnode), pTsdb->path, - pReader->fid); + code = tsdbReadSttBlk(pReader->pDataFReader, iStt, pIter->aSttBlk); + if (code) goto _err; + + for (pIter->iSttBlk = 0; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { + SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); + + if (pSttBlk->minVer > pReader->ever) continue; + if (pSttBlk->maxVer < pReader->sver) continue; + + code = tsdbReadSttBlock(pReader->pDataFReader, iStt, pSttBlk, &pIter->bData); + if (code) goto _err; + + for (pIter->iRow = 0; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { + int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; + + if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.suid = pIter->bData.suid; + pIter->rInfo.uid = pIter->bData.uid; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + goto _add_iter; + } + } } - while (true) { - if (pReader->pBlockIdx && pReader->pBlockL) { - TABLEID id = {.suid = pReader->pBlockL->suid, .uid = pReader->pBlockL->minUid}; + continue; - ASSERT(0); + _add_iter: + tRBTreePut(&pReader->rbt, (SRBTreeNode*)pIter); + pIter++; + } - // if (tTABLEIDCmprFn(pReader->pBlockIdx, &minId) < 0) { - // // TODO - // } else if (tTABLEIDCmprFn(pReader->pBlockIdx, &maxId) < 0) { - // // TODO - // } else { - // // TODO - // } - } else if (pReader->pBlockIdx) { - while (pReader->iBlock < pReader->mBlock.nItem) { - SBlock block; - tMapDataGetItemByIdx(&pReader->mBlock, pReader->iBlock, &block, tGetBlock); - - if (block.minVer <= pReader->ever && block.maxVer >= pReader->sver) { - // load data (todo) - } + tsdbInfo("vgId:%d, vnode snapshot tsdb open data file to read for %s, fid:%d", TD_VID(pReader->pTsdb->pVnode), + pReader->pTsdb->path, pReader->fid); + return code; - // next - pReader->iBlock++; - if (*ppData) break; +_err: + tsdbError("vgId:%d vnode snapshot tsdb snap read open file failed since %s", TD_VID(pReader->pTsdb->pVnode), + tstrerror(code)); + return code; +} + +static SRowInfo* tsdbSnapGetRow(STsdbSnapReader* pReader) { return pReader->pIter ? &pReader->pIter->rInfo : NULL; } + +static int32_t tsdbSnapNextRow(STsdbSnapReader* pReader) { + int32_t code = 0; + + if (pReader->pIter) { + SFDataIter* pIter = pReader->pIter; + + while (true) { + _find_row: + for (pIter->iRow++; pIter->iRow < pIter->bData.nRow; pIter->iRow++) { + int64_t rowVer = pIter->bData.aVersion[pIter->iRow]; + + if (rowVer >= pReader->sver && rowVer <= pReader->ever) { + pIter->rInfo.uid = pIter->bData.uid ? pIter->bData.uid : pIter->bData.aUid[pIter->iRow]; + pIter->rInfo.row = tsdbRowFromBlockData(&pIter->bData, pIter->iRow); + goto _out; } + } - if (pReader->iBlock >= pReader->mBlock.nItem) { - pReader->iBlockIdx++; - if (pReader->iBlockIdx < taosArrayGetSize(pReader->aBlockIdx)) { - pReader->pBlockIdx = (SBlockIdx*)taosArrayGet(pReader->aBlockIdx, pReader->iBlockIdx); + if (pIter->type == SNAP_DATA_FILE_ITER) { + while (true) { + for (pIter->iBlock++; pIter->iBlock < pIter->mBlock.nItem; pIter->iBlock++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pIter->mBlock, pIter->iBlock, &dataBlk, tGetDataBlk); - code = tsdbReadBlock(pReader->pDataFReader, pReader->pBlockIdx, &pReader->mBlock); + if (dataBlk.minVer > pReader->ever || dataBlk.maxVer < pReader->sver) continue; + + code = tsdbReadDataBlockEx(pReader->pDataFReader, &dataBlk, &pIter->bData); if (code) goto _err; - pReader->iBlock = 0; - } else { - pReader->pBlockIdx = NULL; + pIter->iRow = -1; + goto _find_row; } + + pIter->iBlockIdx++; + if (pIter->iBlockIdx >= taosArrayGetSize(pIter->aBlockIdx)) break; + + pIter->pBlockIdx = (SBlockIdx*)taosArrayGet(pIter->aBlockIdx, pIter->iBlockIdx); + code = tsdbReadDataBlk(pReader->pDataFReader, pIter->pBlockIdx, &pIter->mBlock); + if (code) goto _err; + pIter->iBlock = -1; } - if (*ppData) goto _exit; - } else if (pReader->pBlockL) { - while (pReader->pBlockL) { - if (pReader->pBlockL->minVer <= pReader->ever && pReader->pBlockL->maxVer >= pReader->sver) { - // load data (todo) - } + pReader->pIter = NULL; + } else if (pIter->type == SNAP_STT_FILE_ITER) { + for (pIter->iSttBlk++; pIter->iSttBlk < taosArrayGetSize(pIter->aSttBlk); pIter->iSttBlk++) { + SSttBlk* pSttBlk = (SSttBlk*)taosArrayGet(pIter->aSttBlk, pIter->iSttBlk); - // next - pReader->iBlockL++; - if (pReader->iBlockL < taosArrayGetSize(pReader->aBlockL)) { - pReader->pBlockL = (SBlockL*)taosArrayGetSize(pReader->aBlockL); - } else { - pReader->pBlockL = NULL; - } + if (pSttBlk->minVer > pReader->ever || pSttBlk->maxVer < pReader->sver) continue; - if (*ppData) goto _exit; + code = tsdbReadSttBlock(pReader->pDataFReader, pIter->iStt, pSttBlk, &pIter->bData); + if (code) goto _err; + + pIter->iRow = -1; + goto _find_row; } + + pReader->pIter = NULL; + } else { + ASSERT(0); + } + } + + _out: + pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt); + if (pReader->pIter && pIter) { + int32_t c = tRowInfoCmprFn(&pReader->pIter->rInfo, &pIter->rInfo); + if (c > 0) { + tRBTreePut(&pReader->rbt, (SRBTreeNode*)pReader->pIter); + pReader->pIter = NULL; } else { + ASSERT(c); + } + } + } + + if (pReader->pIter == NULL) { + pReader->pIter = (SFDataIter*)tRBTreeMin(&pReader->rbt); + if (pReader->pIter) { + tRBTreeDrop(&pReader->rbt, (SRBTreeNode*)pReader->pIter); + } + } + + return code; + +_err: + return code; +} + +static int32_t tsdbSnapCmprData(STsdbSnapReader* pReader, uint8_t** ppData) { + int32_t code = 0; + + ASSERT(pReader->bData.nRow); + + int32_t aBufN[5] = {0}; + code = tCmprBlockData(&pReader->bData, TWO_STAGE_COMP, NULL, NULL, pReader->aBuf, aBufN); + if (code) goto _exit; + + int32_t size = aBufN[0] + aBufN[1] + aBufN[2] + aBufN[3]; + *ppData = taosMemoryMalloc(sizeof(SSnapDataHdr) + size); + if (*ppData == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + SSnapDataHdr* pHdr = (SSnapDataHdr*)*ppData; + pHdr->type = SNAP_DATA_TSDB; + pHdr->size = size; + + memcpy(pHdr->data, pReader->aBuf[3], aBufN[3]); + memcpy(pHdr->data + aBufN[3], pReader->aBuf[2], aBufN[2]); + if (aBufN[1]) { + memcpy(pHdr->data + aBufN[3] + aBufN[2], pReader->aBuf[1], aBufN[1]); + } + if (aBufN[0]) { + memcpy(pHdr->data + aBufN[3] + aBufN[2] + aBufN[1], pReader->aBuf[0], aBufN[0]); + } + +_exit: + return code; +} + +static int32_t tsdbSnapReadData(STsdbSnapReader* pReader, uint8_t** ppData) { + int32_t code = 0; + STsdb* pTsdb = pReader->pTsdb; + + while (true) { + if (pReader->pDataFReader == NULL) { + code = tsdbSnapReadOpenFile(pReader); + if (code) goto _err; + } + + if (pReader->pDataFReader == NULL) break; + + SRowInfo* pRowInfo = tsdbSnapGetRow(pReader); + if (pRowInfo == NULL) { + tsdbDataFReaderClose(&pReader->pDataFReader); + continue; + } + + TABLEID id = {.suid = pRowInfo->suid, .uid = pRowInfo->uid}; + SBlockData* pBlockData = &pReader->bData; + + code = tsdbUpdateTableSchema(pTsdb->pVnode->pMeta, id.suid, id.uid, &pReader->skmTable); + if (code) goto _err; + + code = tBlockDataInit(pBlockData, id.suid, id.uid, pReader->skmTable.pTSchema); + if (code) goto _err; + + while (pRowInfo->suid == id.suid && pRowInfo->uid == id.uid) { + code = tBlockDataAppendRow(pBlockData, &pRowInfo->row, NULL, pRowInfo->uid); + if (code) goto _err; + + code = tsdbSnapNextRow(pReader); + if (code) goto _err; + + pRowInfo = tsdbSnapGetRow(pReader); + if (pRowInfo == NULL) { tsdbDataFReaderClose(&pReader->pDataFReader); break; } + + if (pBlockData->nRow >= 4096) break; } + + code = tsdbSnapCmprData(pReader, ppData); + if (code) goto _err; + + break; } -_exit: return code; _err: @@ -216,7 +394,6 @@ static int32_t tsdbSnapReadDel(STsdbSnapReader* pReader, uint8_t** ppData) { size += tPutDelData(NULL, pDelData); } } - if (size == 0) continue; // org data @@ -292,23 +469,33 @@ int32_t tsdbSnapReaderOpen(STsdb* pTsdb, int64_t sver, int64_t ever, int8_t type goto _err; } + // data pReader->fid = INT32_MIN; - pReader->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pReader->aBlockIdx == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - pReader->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pReader->aBlockL == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) { + SFDataIter* pIter = &pReader->aFDataIter[iIter]; + + if (iIter == 0) { + pIter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pIter->aBlockIdx == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } else { + pIter->aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pIter->aSttBlk == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } + } + + code = tBlockDataCreate(&pIter->bData); + if (code) goto _err; } - pReader->mBlock = tMapDataInit(); - code = tBlockDataCreate(&pReader->oBlockData); - if (code) goto _err; - code = tBlockDataCreate(&pReader->nBlockData); + + code = tBlockDataCreate(&pReader->bData); if (code) goto _err; + // del pReader->aDelIdx = taosArrayInit(0, sizeof(SDelIdx)); if (pReader->aDelIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -335,18 +522,26 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { int32_t code = 0; STsdbSnapReader* pReader = *ppReader; - if (pReader->pDataFReader) { - tsdbDataFReaderClose(&pReader->pDataFReader); - } - taosArrayDestroy(pReader->aBlockL); - taosArrayDestroy(pReader->aBlockIdx); - tMapDataClear(&pReader->mBlock); - tBlockDataDestroy(&pReader->oBlockData, 1); - tBlockDataDestroy(&pReader->nBlockData, 1); - - if (pReader->pDelFReader) { - tsdbDelFReaderClose(&pReader->pDelFReader); + // data + if (pReader->pDataFReader) tsdbDataFReaderClose(&pReader->pDataFReader); + for (int32_t iIter = 0; iIter < sizeof(pReader->aFDataIter) / sizeof(pReader->aFDataIter[0]); iIter++) { + SFDataIter* pIter = &pReader->aFDataIter[iIter]; + + if (iIter == 0) { + taosArrayDestroy(pIter->aBlockIdx); + tMapDataClear(&pIter->mBlock); + } else { + taosArrayDestroy(pIter->aSttBlk); + } + + tBlockDataDestroy(&pIter->bData, 1); } + + tBlockDataDestroy(&pReader->bData, 1); + tTSchemaDestroy(pReader->skmTable.pTSchema); + + // del + if (pReader->pDelFReader) tsdbDelFReaderClose(&pReader->pDelFReader); taosArrayDestroy(pReader->aDelIdx); taosArrayDestroy(pReader->aDelData); @@ -354,6 +549,10 @@ int32_t tsdbSnapReaderClose(STsdbSnapReader** ppReader) { tsdbInfo("vgId:%d, vnode snapshot tsdb reader closed for %s", TD_VID(pReader->pTsdb->pVnode), pReader->pTsdb->path); + for (int32_t iBuf = 0; iBuf < sizeof(pReader->aBuf) / sizeof(pReader->aBuf[0]); iBuf++) { + tFree(pReader->aBuf[iBuf]); + } + taosMemoryFree(pReader); *ppReader = NULL; return code; @@ -410,40 +609,37 @@ struct STsdbSnapWriter { STsdbFS fs; // config - int32_t minutes; - int8_t precision; - int32_t minRow; - int32_t maxRow; - int8_t cmprAlg; - int64_t commitID; - + int32_t minutes; + int8_t precision; + int32_t minRow; + int32_t maxRow; + int8_t cmprAlg; + int64_t commitID; uint8_t* aBuf[5]; + // for data file SBlockData bData; - - int32_t fid; - SDataFReader* pDataFReader; - SArray* aBlockIdx; // SArray - int32_t iBlockIdx; - SBlockIdx* pBlockIdx; - SMapData mBlock; // SMapData - int32_t iBlock; - SBlockData* pBlockData; - int32_t iRow; - SBlockData bDataR; - SArray* aBlockL; // SArray - int32_t iBlockL; - SBlockData lDataR; - - SDataFWriter* pDataFWriter; - SBlockIdx* pBlockIdxW; // NULL when no committing table - SBlock blockW; - SBlockData bDataW; - SBlockIdx blockIdxW; - - SMapData mBlockW; // SMapData - SArray* aBlockIdxW; // SArray - SArray* aBlockLW; // SArray + int32_t fid; + TABLEID id; + SSkmInfo skmTable; + struct { + SDataFReader* pReader; + SArray* aBlockIdx; + int32_t iBlockIdx; + SBlockIdx* pBlockIdx; + SMapData mDataBlk; + int32_t iDataBlk; + SBlockData bData; + int32_t iRow; + } dReader; + struct { + SDataFWriter* pWriter; + SArray* aBlockIdx; + SMapData mDataBlk; + SArray* aSttBlk; + SBlockData bData; + SBlockData sData; + } dWriter; // for del file SDelFReader* pDelFReader; @@ -454,518 +650,447 @@ struct STsdbSnapWriter { SArray* aDelIdxW; }; -static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { +// SNAP_DATA_TSDB +extern int32_t tsdbWriteDataBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SMapData* mDataBlk, int8_t cmprAlg); +extern int32_t tsdbWriteSttBlock(SDataFWriter* pWriter, SBlockData* pBlockData, SArray* aSttBlk, int8_t cmprAlg); + +static int32_t tsdbSnapNextTableData(STsdbSnapWriter* pWriter) { int32_t code = 0; - ASSERT(pWriter->pDataFWriter); + ASSERT(pWriter->dReader.iRow >= pWriter->dReader.bData.nRow); - if (pWriter->pBlockIdxW == NULL) goto _exit; + if (pWriter->dReader.iBlockIdx < taosArrayGetSize(pWriter->dReader.aBlockIdx)) { + pWriter->dReader.pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->dReader.aBlockIdx, pWriter->dReader.iBlockIdx); - // consume remain rows - if (pWriter->pBlockData) { - ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); - while (pWriter->iRow < pWriter->pBlockData->nRow) { - code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), NULL, - 0); // todo - if (code) goto _err; + code = tsdbReadDataBlk(pWriter->dReader.pReader, pWriter->dReader.pBlockIdx, &pWriter->dReader.mDataBlk); + if (code) goto _exit; - if (pWriter->bDataW.nRow >= pWriter->maxRow * 4 / 5) { - // pWriter->blockW.last = 0; - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - if (code) goto _err; + pWriter->dReader.iBlockIdx++; + } else { + pWriter->dReader.pBlockIdx = NULL; + tMapDataReset(&pWriter->dReader.mDataBlk); + } + pWriter->dReader.iDataBlk = 0; // point to the next one + tBlockDataReset(&pWriter->dReader.bData); + pWriter->dReader.iRow = 0; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; +_exit: + return code; +} - tBlockReset(&pWriter->blockW); - tBlockDataClear(&pWriter->bDataW); - } +static int32_t tsdbSnapWriteCopyData(STsdbSnapWriter* pWriter, TABLEID* pId) { + int32_t code = 0; + + while (true) { + if (pWriter->dReader.pBlockIdx == NULL) break; + if (tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, pId) >= 0) break; + + SBlockIdx blkIdx = *pWriter->dReader.pBlockIdx; + code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dReader.mDataBlk, &blkIdx); + if (code) goto _exit; - pWriter->iRow++; + if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blkIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; } + + code = tsdbSnapNextTableData(pWriter); + if (code) goto _exit; } - // write remain data if has - if (pWriter->bDataW.nRow > 0) { - // pWriter->blockW.last = 0; - if (pWriter->bDataW.nRow < pWriter->minRow) { - if (pWriter->iBlock > pWriter->mBlock.nItem) { - // pWriter->blockW.last = 1; - } - } +_exit: + return code; +} - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - // if (code) goto _err; +static int32_t tsdbSnapWriteTableDataStart(STsdbSnapWriter* pWriter, TABLEID* pId) { + int32_t code = 0; - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; + code = tsdbSnapWriteCopyData(pWriter, pId); + if (code) goto _err; + + pWriter->id.suid = pId->suid; + pWriter->id.uid = pId->uid; + + code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pId->suid, pId->uid, &pWriter->skmTable); + if (code) goto _err; + + tMapDataReset(&pWriter->dWriter.mDataBlk); + code = tBlockDataInit(&pWriter->dWriter.bData, pId->suid, pId->uid, pWriter->skmTable.pTSchema); + if (code) goto _err; + + return code; + +_err: + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); + return code; +} + +static int32_t tsdbSnapWriteTableDataEnd(STsdbSnapWriter* pWriter) { + int32_t code = 0; + + if (pWriter->id.suid == 0 && pWriter->id.uid == 0) return code; + + int32_t c = 1; + if (pWriter->dReader.pBlockIdx) { + c = tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &pWriter->id); + ASSERT(c >= 0); } - while (true) { - if (pWriter->iBlock >= pWriter->mBlock.nItem) break; + if (c == 0) { + SBlockData* pBData = &pWriter->dWriter.bData; - SBlock block; - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) { + TSDBROW row = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow); - // if (block.last) { - // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - // if (code) goto _err; + code = tBlockDataAppendRow(pBData, &row, NULL, pWriter->id.uid); + if (code) goto _err; - // tBlockReset(&block); - // block.last = 1; - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pWriter->pBlockIdxW, &block, - // pWriter->cmprAlg); - // if (code) goto _err; - // } + if (pBData->nRow >= pWriter->maxRow) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg); + if (code) goto _err; + } + } - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, pBData, &pWriter->dWriter.mDataBlk, pWriter->cmprAlg); if (code) goto _err; - pWriter->iBlock++; + for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk); + + code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk); + if (code) goto _err; + } + + code = tsdbSnapNextTableData(pWriter); + if (code) goto _err; } - // SBlock - // code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, NULL, pWriter->pBlockIdxW); - // if (code) goto _err; + if (pWriter->dWriter.mDataBlk.nItem) { + SBlockIdx blockIdx = {.suid = pWriter->id.suid, .uid = pWriter->id.uid}; + code = tsdbWriteDataBlk(pWriter->dWriter.pWriter, &pWriter->dWriter.mDataBlk, &blockIdx); - // SBlockIdx - if (taosArrayPush(pWriter->aBlockIdxW, pWriter->pBlockIdxW) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + if (taosArrayPush(pWriter->dWriter.aBlockIdx, &blockIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _err; + } } -_exit: - tsdbInfo("vgId:%d, tsdb snapshot write table data end for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); + pWriter->id.suid = 0; + pWriter->id.uid = 0; + return code; _err: - tsdbError("vgId:%d, tsdb snapshot write table data end for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); return code; } -static int32_t tsdbSnapMoveWriteTableData(STsdbSnapWriter* pWriter, SBlockIdx* pBlockIdx) { +static int32_t tsdbSnapWriteOpenFile(STsdbSnapWriter* pWriter, int32_t fid) { int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; - code = tsdbReadBlock(pWriter->pDataFReader, pBlockIdx, &pWriter->mBlock); - if (code) goto _err; + ASSERT(pWriter->dWriter.pWriter == NULL); + + pWriter->fid = fid; + pWriter->id = (TABLEID){0}; + SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); - // SBlockData - SBlock block; - tMapDataReset(&pWriter->mBlockW); - for (int32_t iBlock = 0; iBlock < pWriter->mBlock.nItem; iBlock++) { - tMapDataGetItemByIdx(&pWriter->mBlock, iBlock, &block, tGetBlock); - - // if (block.last) { - // code = tsdbReadBlockData(pWriter->pDataFReader, pBlockIdx, &block, &pWriter->bDataR, NULL, NULL); - // if (code) goto _err; - - // tBlockReset(&block); - // block.last = 1; - // code = - // tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataR, NULL, NULL, pBlockIdx, &block, - // pWriter->cmprAlg); - // if (code) goto _err; - // } - - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); + // Reader + if (pSet) { + code = tsdbDataFReaderOpen(&pWriter->dReader.pReader, pWriter->pTsdb, pSet); if (code) goto _err; - } - // SBlock - SBlockIdx blockIdx = {.suid = pBlockIdx->suid, .uid = pBlockIdx->uid}; - code = tsdbWriteBlock(pWriter->pDataFWriter, &pWriter->mBlockW, &blockIdx); + code = tsdbReadBlockIdx(pWriter->dReader.pReader, pWriter->dReader.aBlockIdx); + if (code) goto _err; + } else { + ASSERT(pWriter->dReader.pReader == NULL); + taosArrayClear(pWriter->dReader.aBlockIdx); + } + pWriter->dReader.iBlockIdx = 0; // point to the next one + code = tsdbSnapNextTableData(pWriter); if (code) goto _err; - // SBlockIdx - if (taosArrayPush(pWriter->aBlockIdxW, &blockIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; + // Writer + SHeadFile fHead = {.commitID = pWriter->commitID}; + SDataFile fData = {.commitID = pWriter->commitID}; + SSmaFile fSma = {.commitID = pWriter->commitID}; + SSttFile fStt = {.commitID = pWriter->commitID}; + SDFileSet wSet = {.fid = pWriter->fid, .pHeadF = &fHead, .pDataF = &fData, .pSmaF = &fSma}; + if (pSet) { + wSet.diskId = pSet->diskId; + fData = *pSet->pDataF; + fSma = *pSet->pSmaF; + for (int32_t iStt = 0; iStt < pSet->nSttF; iStt++) { + wSet.aSttF[iStt] = pSet->aSttF[iStt]; + } + wSet.nSttF = pSet->nSttF + 1; // TODO: fix pSet->nSttF == pTsdb->maxFile + } else { + SDiskID did = {0}; + tfsAllocDisk(pTsdb->pVnode->pTfs, 0, &did); + tfsMkdirRecurAt(pTsdb->pVnode->pTfs, pTsdb->path, did); + wSet.diskId = did; + wSet.nSttF = 1; } + wSet.aSttF[wSet.nSttF - 1] = &fStt; + + code = tsdbDataFWriterOpen(&pWriter->dWriter.pWriter, pWriter->pTsdb, &wSet); + if (code) goto _err; + taosArrayClear(pWriter->dWriter.aBlockIdx); + tMapDataReset(&pWriter->dWriter.mDataBlk); + taosArrayClear(pWriter->dWriter.aSttBlk); + tBlockDataReset(&pWriter->dWriter.bData); + tBlockDataReset(&pWriter->dWriter.sData); -_exit: return code; _err: - tsdbError("vgId:%d, tsdb snapshot move write table data for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); return code; } -static int32_t tsdbSnapWriteTableDataImpl(STsdbSnapWriter* pWriter) { - int32_t code = 0; - SBlockData* pBlockData = &pWriter->bData; - int32_t iRow = 0; - TSDBROW row; - TSDBROW* pRow = &row; +static int32_t tsdbSnapWriteCloseFile(STsdbSnapWriter* pWriter) { + int32_t code = 0; - // // correct schema - // code = tBlockDataCorrectSchema(&pWriter->bDataW, pBlockData); - // if (code) goto _err; + ASSERT(pWriter->dWriter.pWriter); - // loop to merge - *pRow = tsdbRowFromBlockData(pBlockData, iRow); - while (true) { - if (pRow == NULL) break; + code = tsdbSnapWriteTableDataEnd(pWriter); + if (code) goto _err; - if (pWriter->pBlockData) { - ASSERT(pWriter->iRow < pWriter->pBlockData->nRow); + // copy remain table data + TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX}; + code = tsdbSnapWriteCopyData(pWriter, &id); + if (code) goto _err; - int32_t c = tsdbRowCmprFn(pRow, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow)); + code = + tsdbWriteSttBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.sData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg); + if (code) goto _err; - ASSERT(c); + // Indices + code = tsdbWriteBlockIdx(pWriter->dWriter.pWriter, pWriter->dWriter.aBlockIdx); + if (code) goto _err; - if (c < 0) { - // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - // if (code) goto _err; - - iRow++; - if (iRow < pWriter->pBlockData->nRow) { - *pRow = tsdbRowFromBlockData(pBlockData, iRow); - } else { - pRow = NULL; - } - } else if (c > 0) { - // code = tBlockDataAppendRow(&pWriter->bDataW, &tsdbRowFromBlockData(pWriter->pBlockData, pWriter->iRow), - // NULL); if (code) goto _err; + code = tsdbWriteSttBlk(pWriter->dWriter.pWriter, pWriter->dWriter.aSttBlk); + if (code) goto _err; - pWriter->iRow++; - if (pWriter->iRow >= pWriter->pBlockData->nRow) { - pWriter->pBlockData = NULL; - } - } - } else { - TSDBKEY key = TSDBROW_KEY(pRow); + code = tsdbUpdateDFileSetHeader(pWriter->dWriter.pWriter); + if (code) goto _err; - while (true) { - if (pWriter->iBlock >= pWriter->mBlock.nItem) break; + code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->dWriter.pWriter->wSet); + if (code) goto _err; - SBlock block; - int32_t c; + code = tsdbDataFWriterClose(&pWriter->dWriter.pWriter, 1); + if (code) goto _err; - tMapDataGetItemByIdx(&pWriter->mBlock, pWriter->iBlock, &block, tGetBlock); + if (pWriter->dReader.pReader) { + code = tsdbDataFReaderClose(&pWriter->dReader.pReader); + if (code) goto _err; + } - // if (block.last) { - // pWriter->pBlockData = &pWriter->bDataR; +_exit: + return code; - // code = tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, - // NULL); if (code) goto _err; pWriter->iRow = 0; +_err: + return code; +} - // pWriter->iBlock++; - // break; - // } +static int32_t tsdbSnapWriteToDataFile(STsdbSnapWriter* pWriter, int32_t iRow, int8_t* done) { + int32_t code = 0; - c = tsdbKeyCmprFn(&block.maxKey, &key); + SBlockData* pBData = &pWriter->bData; + TABLEID id = {.suid = pBData->suid, .uid = pBData->uid ? pBData->uid : pBData->aUid[iRow]}; + TSDBROW row = tsdbRowFromBlockData(pBData, iRow); + TSDBKEY key = TSDBROW_KEY(&row); - ASSERT(c); + *done = 0; + while (pWriter->dReader.iRow < pWriter->dReader.bData.nRow || + pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem) { + // Merge row by row + for (; pWriter->dReader.iRow < pWriter->dReader.bData.nRow; pWriter->dReader.iRow++) { + TSDBROW trow = tsdbRowFromBlockData(&pWriter->dReader.bData, pWriter->dReader.iRow); + TSDBKEY tKey = TSDBROW_KEY(&trow); - if (c < 0) { - if (pWriter->bDataW.nRow) { - // pWriter->blockW.last = 0; - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - // if (code) goto _err; + ASSERT(pWriter->dReader.bData.suid == id.suid && pWriter->dReader.bData.uid == id.uid); - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; + int32_t c = tsdbKeyCmprFn(&key, &tKey); + if (c < 0) { + code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid); + if (code) goto _err; + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->dWriter.bData, &trow, NULL, id.uid); + if (code) goto _err; + } else { + ASSERT(0); + } - tBlockReset(&pWriter->blockW); - tBlockDataClear(&pWriter->bDataW); - } + if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk, + pWriter->cmprAlg); + if (code) goto _err; + } - code = tMapDataPutItem(&pWriter->mBlockW, &block, tPutBlock); - if (code) goto _err; + if (c < 0) { + *done = 1; + goto _exit; + } + } - pWriter->iBlock++; - } else { - c = tsdbKeyCmprFn(&tBlockDataLastKey(pBlockData), &block.minKey); + // Merge row by block + SDataBlk tDataBlk = {.minKey = key, .maxKey = key}; + for (; pWriter->dReader.iDataBlk < pWriter->dReader.mDataBlk.nItem; pWriter->dReader.iDataBlk++) { + SDataBlk dataBlk; + tMapDataGetItemByIdx(&pWriter->dReader.mDataBlk, pWriter->dReader.iDataBlk, &dataBlk, tGetDataBlk); - ASSERT(c); + int32_t c = tDataBlkCmprFn(&dataBlk, &tDataBlk); + if (c < 0) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk, + pWriter->cmprAlg); + if (code) goto _err; - if (c > 0) { - pWriter->pBlockData = &pWriter->bDataR; - // code = - // tsdbReadBlockData(pWriter->pDataFReader, pWriter->pBlockIdx, &block, pWriter->pBlockData, NULL, - // NULL); - // if (code) goto _err; - pWriter->iRow = 0; + code = tMapDataPutItem(&pWriter->dWriter.mDataBlk, &dataBlk, tPutDataBlk); + if (code) goto _err; + } else if (c > 0) { + code = tBlockDataAppendRow(&pWriter->dWriter.bData, &row, NULL, id.uid); + if (code) goto _err; - pWriter->iBlock++; - } - break; + if (pWriter->dWriter.bData.nRow >= pWriter->maxRow) { + code = tsdbWriteDataBlock(pWriter->dWriter.pWriter, &pWriter->dWriter.bData, &pWriter->dWriter.mDataBlk, + pWriter->cmprAlg); + if (code) goto _err; } - } - - if (pWriter->pBlockData) continue; - - // code = tBlockDataAppendRow(&pWriter->bDataW, pRow, NULL); - // if (code) goto _err; - iRow++; - if (iRow < pBlockData->nRow) { - *pRow = tsdbRowFromBlockData(pBlockData, iRow); + *done = 1; + goto _exit; } else { - pRow = NULL; + code = tsdbReadDataBlockEx(pWriter->dReader.pReader, &dataBlk, &pWriter->dReader.bData); + if (code) goto _err; + pWriter->dReader.iRow = 0; + + pWriter->dReader.iDataBlk++; + break; } } - - _check_write: - if (pWriter->bDataW.nRow < pWriter->maxRow * 4 / 5) continue; - - _write_block: - // code = tsdbWriteBlockData(pWriter->pDataFWriter, &pWriter->bDataW, NULL, NULL, pWriter->pBlockIdxW, - // &pWriter->blockW, pWriter->cmprAlg); - // if (code) goto _err; - - code = tMapDataPutItem(&pWriter->mBlockW, &pWriter->blockW, tPutBlock); - if (code) goto _err; - - tBlockReset(&pWriter->blockW); - tBlockDataClear(&pWriter->bDataW); } +_exit: return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb write table data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); return code; } -static int32_t tsdbSnapWriteTableData(STsdbSnapWriter* pWriter, TABLEID id) { - int32_t code = 0; - SBlockData* pBlockData = &pWriter->bData; - TSDBKEY keyFirst = tBlockDataFirstKey(pBlockData); - TSDBKEY keyLast = tBlockDataLastKey(pBlockData); - - // end last table write if should - if (pWriter->pBlockIdxW) { - int32_t c = tTABLEIDCmprFn(pWriter->pBlockIdxW, &id); - if (c < 0) { - // end - code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; - - // reset - pWriter->pBlockIdxW = NULL; - } else if (c > 0) { - ASSERT(0); - } - } - - // start new table data write if need - if (pWriter->pBlockIdxW == NULL) { - // write table data ahead - while (true) { - if (pWriter->iBlockIdx >= taosArrayGetSize(pWriter->aBlockIdx)) break; - - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); +static int32_t tsdbSnapWriteToSttFile(STsdbSnapWriter* pWriter, int32_t iRow) { + int32_t code = 0; - if (c >= 0) break; + TABLEID id = {.suid = pWriter->bData.suid, + .uid = pWriter->bData.uid ? pWriter->bData.uid : pWriter->bData.aUid[iRow]}; + TSDBROW row = tsdbRowFromBlockData(&pWriter->bData, iRow); + SBlockData* pBData = &pWriter->dWriter.sData; - code = tsdbSnapMoveWriteTableData(pWriter, pBlockIdx); + if (pBData->suid || pBData->uid) { + if (!TABLE_SAME_SCHEMA(pBData->suid, pBData->uid, id.suid, id.uid)) { + code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg); if (code) goto _err; - pWriter->iBlockIdx++; + pBData->suid = 0; + pBData->uid = 0; } + } - // reader - pWriter->pBlockIdx = NULL; - if (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - ASSERT(pWriter->pDataFReader); - - SBlockIdx* pBlockIdx = (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx); - int32_t c = tTABLEIDCmprFn(pBlockIdx, &id); - - ASSERT(c >= 0); - - if (c == 0) { - pWriter->pBlockIdx = pBlockIdx; - pWriter->iBlockIdx++; - } - } - - if (pWriter->pBlockIdx) { - code = tsdbReadBlock(pWriter->pDataFReader, pWriter->pBlockIdx, &pWriter->mBlock); - if (code) goto _err; - } else { - tMapDataReset(&pWriter->mBlock); - } - pWriter->iBlock = 0; - pWriter->pBlockData = NULL; - pWriter->iRow = 0; - - // writer - pWriter->pBlockIdxW = &pWriter->blockIdxW; - pWriter->pBlockIdxW->suid = id.suid; - pWriter->pBlockIdxW->uid = id.uid; + if (pBData->suid == 0 && pBData->uid == 0) { + code = tsdbUpdateTableSchema(pWriter->pTsdb->pVnode->pMeta, pWriter->id.suid, pWriter->id.uid, &pWriter->skmTable); + if (code) goto _err; - tBlockReset(&pWriter->blockW); - tBlockDataReset(&pWriter->bDataW); - tMapDataReset(&pWriter->mBlockW); + code = tBlockDataInit(pBData, pWriter->id.suid, pWriter->id.suid ? 0 : pWriter->id.uid, pWriter->skmTable.pTSchema); + if (code) goto _err; } - ASSERT(pWriter->pBlockIdxW && pWriter->pBlockIdxW->suid == id.suid && pWriter->pBlockIdxW->uid == id.uid); - ASSERT(pWriter->pBlockIdx == NULL || (pWriter->pBlockIdx->suid == id.suid && pWriter->pBlockIdx->uid == id.uid)); - - code = tsdbSnapWriteTableDataImpl(pWriter); + code = tBlockDataAppendRow(pBData, &row, NULL, id.uid); if (code) goto _err; + if (pBData->nRow >= pWriter->maxRow) { + code = tsdbWriteSttBlock(pWriter->dWriter.pWriter, pBData, pWriter->dWriter.aSttBlk, pWriter->cmprAlg); + if (code) goto _err; + } + _exit: - tsdbDebug("vgId:%d, vnode snapshot tsdb write data impl for %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path); return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb write data impl for %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), - pWriter->pTsdb->path, tstrerror(code)); return code; } -static int32_t tsdbSnapWriteDataEnd(STsdbSnapWriter* pWriter) { +static int32_t tsdbSnapWriteRowData(STsdbSnapWriter* pWriter, int32_t iRow) { int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - - if (pWriter->pDataFWriter == NULL) goto _exit; - // finish current table - code = tsdbSnapWriteTableDataEnd(pWriter); - if (code) goto _err; + SBlockData* pBlockData = &pWriter->bData; + TABLEID id = {.suid = pBlockData->suid, .uid = pBlockData->uid ? pBlockData->uid : pBlockData->aUid[iRow]}; - // move remain table - while (pWriter->iBlockIdx < taosArrayGetSize(pWriter->aBlockIdx)) { - code = tsdbSnapMoveWriteTableData(pWriter, (SBlockIdx*)taosArrayGet(pWriter->aBlockIdx, pWriter->iBlockIdx)); + // End last table data write if need + if (tTABLEIDCmprFn(&pWriter->id, &id) != 0) { + code = tsdbSnapWriteTableDataEnd(pWriter); if (code) goto _err; - - pWriter->iBlockIdx++; } - // write remain stuff - if (taosArrayGetSize(pWriter->aBlockLW) > 0) { - code = tsdbWriteBlockL(pWriter->pDataFWriter, pWriter->aBlockIdxW); + // Start new table data write if need + if (pWriter->id.suid == 0 && pWriter->id.uid == 0) { + code = tsdbSnapWriteTableDataStart(pWriter, &id); if (code) goto _err; } - if (taosArrayGetSize(pWriter->aBlockIdx) > 0) { - code = tsdbWriteBlockIdx(pWriter->pDataFWriter, pWriter->aBlockIdxW); + // Merge with .data file data + int8_t done = 0; + if (pWriter->dReader.pBlockIdx && tTABLEIDCmprFn(pWriter->dReader.pBlockIdx, &id) == 0) { + code = tsdbSnapWriteToDataFile(pWriter, iRow, &done); if (code) goto _err; } - code = tsdbFSUpsertFSet(&pWriter->fs, &pWriter->pDataFWriter->wSet); - if (code) goto _err; - - code = tsdbDataFWriterClose(&pWriter->pDataFWriter, 1); - if (code) goto _err; - - if (pWriter->pDataFReader) { - code = tsdbDataFReaderClose(&pWriter->pDataFReader); + // Append to the .stt data block (todo: check if need to set/reload sst block) + if (!done) { + code = tsdbSnapWriteToSttFile(pWriter, iRow); if (code) goto _err; } _exit: - tsdbInfo("vgId:%d, vnode snapshot tsdb writer data end for %s", TD_VID(pTsdb->pVnode), pTsdb->path); return code; _err: - tsdbError("vgId:%d, vnode snapshot tsdb writer data end for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, - tstrerror(code)); + tsdbError("vgId:%d %s failed since %s", TD_VID(pWriter->pTsdb->pVnode), __func__, tstrerror(code)); return code; } static int32_t tsdbSnapWriteData(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { - int32_t code = 0; - STsdb* pTsdb = pWriter->pTsdb; - SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; - TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); - int64_t n; - - // decode + int32_t code = 0; + STsdb* pTsdb = pWriter->pTsdb; SBlockData* pBlockData = &pWriter->bData; - code = tDecmprBlockData(pData + sizeof(SSnapDataHdr) + sizeof(TABLEID), pHdr->size - sizeof(TABLEID), pBlockData, - pWriter->aBuf); - if (code) goto _err; - // open file - TSDBKEY keyFirst = {.version = pBlockData->aVersion[0], .ts = pBlockData->aTSKEY[0]}; - TSDBKEY keyLast = {.version = pBlockData->aVersion[pBlockData->nRow - 1], - .ts = pBlockData->aTSKEY[pBlockData->nRow - 1]}; + // Decode data + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + code = tDecmprBlockData(pHdr->data, pHdr->size, pBlockData, pWriter->aBuf); + if (code) goto _err; - int32_t fid = tsdbKeyFid(keyFirst.ts, pWriter->minutes, pWriter->precision); - ASSERT(fid == tsdbKeyFid(keyLast.ts, pWriter->minutes, pWriter->precision)); - if (pWriter->pDataFWriter == NULL || pWriter->fid != fid) { - // end last file data write if need - code = tsdbSnapWriteDataEnd(pWriter); - if (code) goto _err; + ASSERT(pBlockData->nRow > 0); - pWriter->fid = fid; + // Loop to handle each row + for (int32_t iRow = 0; iRow < pBlockData->nRow; iRow++) { + TSKEY ts = pBlockData->aTSKEY[iRow]; + int32_t fid = tsdbKeyFid(ts, pWriter->minutes, pWriter->precision); - // read - SDFileSet* pSet = taosArraySearch(pWriter->fs.aDFileSet, &(SDFileSet){.fid = fid}, tDFileSetCmprFn, TD_EQ); - if (pSet) { - code = tsdbDataFReaderOpen(&pWriter->pDataFReader, pTsdb, pSet); - if (code) goto _err; + if (pWriter->dWriter.pWriter == NULL || pWriter->fid != fid) { + if (pWriter->dWriter.pWriter) { + ASSERT(fid > pWriter->fid); - code = tsdbReadBlockIdx(pWriter->pDataFReader, pWriter->aBlockIdx); - if (code) goto _err; + code = tsdbSnapWriteCloseFile(pWriter); + if (code) goto _err; + } - code = tsdbReadBlockL(pWriter->pDataFReader, pWriter->aBlockL); + code = tsdbSnapWriteOpenFile(pWriter, fid); if (code) goto _err; - } else { - ASSERT(pWriter->pDataFReader == NULL); - taosArrayClear(pWriter->aBlockIdx); - taosArrayClear(pWriter->aBlockL); - } - pWriter->iBlockIdx = 0; - pWriter->pBlockIdx = NULL; - tMapDataReset(&pWriter->mBlock); - pWriter->iBlock = 0; - pWriter->pBlockData = NULL; - pWriter->iRow = 0; - pWriter->iBlockL = 0; - tBlockDataReset(&pWriter->bDataR); - tBlockDataReset(&pWriter->lDataR); - - // write - SHeadFile fHead; - SDataFile fData; - SLastFile fLast; - SSmaFile fSma; - SDFileSet wSet = {.pHeadF = &fHead, .pDataF = &fData, .pLastF = &fLast, .pSmaF = &fSma}; - - if (pSet) { - wSet.diskId = pSet->diskId; - wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; - fData = *pSet->pDataF; - fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0}; - fSma = *pSet->pSmaF; - } else { - wSet.diskId = (SDiskID){.level = 0, .id = 0}; - wSet.fid = fid; - fHead = (SHeadFile){.commitID = pWriter->commitID, .offset = 0, .size = 0}; - fData = (SDataFile){.commitID = pWriter->commitID, .size = 0}; - fLast = (SLastFile){.commitID = pWriter->commitID, .size = 0, .offset = 0}; - fSma = (SSmaFile){.commitID = pWriter->commitID, .size = 0}; } - code = tsdbDataFWriterOpen(&pWriter->pDataFWriter, pTsdb, &wSet); + code = tsdbSnapWriteRowData(pWriter, iRow); if (code) goto _err; - - taosArrayClear(pWriter->aBlockIdxW); - taosArrayClear(pWriter->aBlockLW); - tMapDataReset(&pWriter->mBlockW); - pWriter->pBlockIdxW = NULL; - tBlockDataReset(&pWriter->bDataW); } - code = tsdbSnapWriteTableData(pWriter, id); - if (code) goto _err; - - tsdbInfo("vgId:%d, vnode snapshot tsdb write data for %s, fid:%d suid:%" PRId64 " uid:%" PRId64 " nRow:%d", - TD_VID(pTsdb->pVnode), pTsdb->path, fid, id.suid, id.suid, pBlockData->nRow); return code; _err: @@ -974,10 +1099,41 @@ _err: return code; } +// SNAP_DATA_DEL +static int32_t tsdbSnapMoveWriteDelData(STsdbSnapWriter* pWriter, TABLEID* pId) { + int32_t code = 0; + + while (true) { + if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break; + + SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); + + if (tTABLEIDCmprFn(pDelIdx, pId) >= 0) break; + + code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); + if (code) goto _exit; + + SDelIdx delIdx = *pDelIdx; + code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); + if (code) goto _exit; + + if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + + pWriter->iDelIdx++; + } + +_exit: + return code; +} + static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; + // Open del file if not opened yet if (pWriter->pDelFWriter == NULL) { SDelFile* pDelFile = pWriter->fs.pDelFile; @@ -988,38 +1144,28 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 code = tsdbReadDelIdx(pWriter->pDelFReader, pWriter->aDelIdxR); if (code) goto _err; + } else { + taosArrayClear(pWriter->aDelIdxR); } + pWriter->iDelIdx = 0; // writer - SDelFile delFile = {.commitID = pWriter->commitID, .offset = 0, .size = 0}; + SDelFile delFile = {.commitID = pWriter->commitID}; code = tsdbDelFWriterOpen(&pWriter->pDelFWriter, &delFile, pTsdb); if (code) goto _err; + taosArrayClear(pWriter->aDelIdxW); } - // process the del data - TABLEID id = *(TABLEID*)(pData + sizeof(SSnapDataHdr)); - - while (true) { - if (pWriter->iDelIdx >= taosArrayGetSize(pWriter->aDelIdxR)) break; - if (tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) >= 0) break; - - SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - if (code) goto _err; - - SDelIdx delIdx = *pDelIdx; - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); - if (code) goto _err; + SSnapDataHdr* pHdr = (SSnapDataHdr*)pData; + TABLEID id = *(TABLEID*)pHdr->data; - if (taosArrayPush(pWriter->aDelIdxW, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } + ASSERT(pHdr->size + sizeof(SSnapDataHdr) == nData); - pWriter->iDelIdx++; - } + // Move write data < id + code = tsdbSnapMoveWriteDelData(pWriter, &id); + if (code) goto _err; + // Merge incoming data with current if (pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR) && tTABLEIDCmprFn(taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx), &id) == 0) { SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); @@ -1053,7 +1199,6 @@ static int32_t tsdbSnapWriteDel(STsdbSnapWriter* pWriter, uint8_t* pData, uint32 goto _err; } -_exit: return code; _err: @@ -1066,23 +1211,14 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { int32_t code = 0; STsdb* pTsdb = pWriter->pTsdb; - if (pWriter->pDelFWriter == NULL) goto _exit; - - for (; pWriter->iDelIdx < taosArrayGetSize(pWriter->aDelIdxR); pWriter->iDelIdx++) { - SDelIdx* pDelIdx = (SDelIdx*)taosArrayGet(pWriter->aDelIdxR, pWriter->iDelIdx); - - code = tsdbReadDelData(pWriter->pDelFReader, pDelIdx, pWriter->aDelData); - if (code) goto _err; + if (pWriter->pDelFWriter == NULL) return code; - SDelIdx delIdx = *pDelIdx; - code = tsdbWriteDelData(pWriter->pDelFWriter, pWriter->aDelData, &delIdx); - if (code) goto _err; + TABLEID id = {.suid = INT64_MAX, .uid = INT64_MAX}; + code = tsdbSnapMoveWriteDelData(pWriter, &id); + if (code) goto _err; - if (taosArrayPush(pWriter->aDelIdxR, &delIdx) == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - } + code = tsdbWriteDelIdx(pWriter->pDelFWriter, pWriter->aDelIdxW); + if (code) goto _err; code = tsdbUpdateDelFileHdr(pWriter->pDelFWriter); if (code) goto _err; @@ -1098,7 +1234,6 @@ static int32_t tsdbSnapWriteDelEnd(STsdbSnapWriter* pWriter) { if (code) goto _err; } -_exit: tsdbInfo("vgId:%d, vnode snapshot tsdb write del for %s end", TD_VID(pTsdb->pVnode), pTsdb->path); return code; @@ -1108,6 +1243,7 @@ _err: return code; } +// APIs int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWriter** ppWriter) { int32_t code = 0; STsdbSnapWriter* pWriter = NULL; @@ -1133,39 +1269,38 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr pWriter->cmprAlg = pTsdb->pVnode->config.tsdbCfg.compression; pWriter->commitID = pTsdb->pVnode->state.commitID; - // for data file + // SNAP_DATA_TSDB code = tBlockDataCreate(&pWriter->bData); - if (code) goto _err; - pWriter->aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); - if (pWriter->aBlockIdx == NULL) { + + pWriter->fid = INT32_MIN; + pWriter->id = (TABLEID){0}; + // Reader + pWriter->dReader.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->dReader.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tBlockDataCreate(&pWriter->bDataR); + code = tBlockDataCreate(&pWriter->dReader.bData); if (code) goto _err; - pWriter->aBlockL = taosArrayInit(0, sizeof(SBlockL)); - if (pWriter->aBlockL == NULL) { + // Writer + pWriter->dWriter.aBlockIdx = taosArrayInit(0, sizeof(SBlockIdx)); + if (pWriter->dWriter.aBlockIdx == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - - pWriter->aBlockIdxW = taosArrayInit(0, sizeof(SBlockIdx)); - if (pWriter->aBlockIdxW == NULL) { + pWriter->dWriter.aSttBlk = taosArrayInit(0, sizeof(SSttBlk)); + if (pWriter->dWriter.aSttBlk == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; goto _err; } - code = tBlockDataCreate(&pWriter->bDataW); + code = tBlockDataCreate(&pWriter->dWriter.bData); + if (code) goto _err; + code = tBlockDataCreate(&pWriter->dWriter.sData); if (code) goto _err; - pWriter->aBlockLW = taosArrayInit(0, sizeof(SBlockL)); - if (pWriter->aBlockLW == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _err; - } - - // for del file + // SNAP_DATA_DEL pWriter->aDelIdxR = taosArrayInit(0, sizeof(SDelIdx)); if (pWriter->aDelIdxR == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; @@ -1186,6 +1321,7 @@ int32_t tsdbSnapWriterOpen(STsdb* pTsdb, int64_t sver, int64_t ever, STsdbSnapWr tsdbInfo("vgId:%d, tsdb snapshot writer open for %s succeed", TD_VID(pTsdb->pVnode), pTsdb->path); return code; + _err: tsdbError("vgId:%d, tsdb snapshot writer open for %s failed since %s", TD_VID(pTsdb->pVnode), pTsdb->path, tstrerror(code)); @@ -1196,14 +1332,17 @@ _err: int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { int32_t code = 0; STsdbSnapWriter* pWriter = *ppWriter; + STsdb* pTsdb = pWriter->pTsdb; if (rollback) { ASSERT(0); // code = tsdbFSRollback(pWriter->pTsdb->pFS); // if (code) goto _err; } else { - code = tsdbSnapWriteDataEnd(pWriter); - if (code) goto _err; + if (pWriter->dWriter.pWriter) { + code = tsdbSnapWriteCloseFile(pWriter); + if (code) goto _err; + } code = tsdbSnapWriteDelEnd(pWriter); if (code) goto _err; @@ -1211,14 +1350,44 @@ int32_t tsdbSnapWriterClose(STsdbSnapWriter** ppWriter, int8_t rollback) { code = tsdbFSCommit1(pWriter->pTsdb, &pWriter->fs); if (code) goto _err; + // lock + taosThreadRwlockWrlock(&pTsdb->rwLock); + code = tsdbFSCommit2(pWriter->pTsdb, &pWriter->fs); - if (code) goto _err; + if (code) { + taosThreadRwlockUnlock(&pTsdb->rwLock); + goto _err; + } + + // unlock + taosThreadRwlockUnlock(&pTsdb->rwLock); } + // SNAP_DATA_DEL + taosArrayDestroy(pWriter->aDelIdxW); + taosArrayDestroy(pWriter->aDelData); + taosArrayDestroy(pWriter->aDelIdxR); + + // SNAP_DATA_TSDB + + // Writer + tBlockDataDestroy(&pWriter->dWriter.sData, 1); + tBlockDataDestroy(&pWriter->dWriter.bData, 1); + taosArrayDestroy(pWriter->dWriter.aSttBlk); + tMapDataClear(&pWriter->dWriter.mDataBlk); + taosArrayDestroy(pWriter->dWriter.aBlockIdx); + + // Reader + tBlockDataDestroy(&pWriter->dReader.bData, 1); + tMapDataClear(&pWriter->dReader.mDataBlk); + taosArrayDestroy(pWriter->dReader.aBlockIdx); + + tBlockDataDestroy(&pWriter->bData, 1); + tTSchemaDestroy(pWriter->skmTable.pTSchema); + for (int32_t iBuf = 0; iBuf < sizeof(pWriter->aBuf) / sizeof(uint8_t*); iBuf++) { tFree(pWriter->aBuf[iBuf]); } - tsdbInfo("vgId:%d, vnode snapshot tsdb writer close for %s", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); taosMemoryFree(pWriter); *ppWriter = NULL; @@ -1243,8 +1412,8 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) goto _exit; } else { - if (pWriter->pDataFWriter) { - code = tsdbSnapWriteDataEnd(pWriter); + if (pWriter->dWriter.pWriter) { + code = tsdbSnapWriteCloseFile(pWriter); if (code) goto _err; } } @@ -1257,7 +1426,6 @@ int32_t tsdbSnapWrite(STsdbSnapWriter* pWriter, uint8_t* pData, uint32_t nData) _exit: tsdbDebug("vgId:%d, tsdb snapshot write for %s succeed", TD_VID(pWriter->pTsdb->pVnode), pWriter->pTsdb->path); - return code; _err: diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index 6db9d5e6f40c5d35e52d90dd86b28f4cb7a94676..64c150484b4f873bd15fb69c2b238bd840e1ea1f 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -51,6 +51,22 @@ _exit: return code; } +int32_t tMapDataCopy(SMapData *pFrom, SMapData *pTo) { + int32_t code = 0; + + pTo->nItem = pFrom->nItem; + pTo->nData = pFrom->nData; + code = tRealloc((uint8_t **)&pTo->aOffset, sizeof(int32_t) * pFrom->nItem); + if (code) goto _exit; + code = tRealloc(&pTo->pData, pFrom->nData); + if (code) goto _exit; + memcpy(pTo->aOffset, pFrom->aOffset, sizeof(int32_t) * pFrom->nItem); + memcpy(pTo->pData, pFrom->pData, pFrom->nData); + +_exit: + return code; +} + int32_t tMapDataSearch(SMapData *pMapData, void *pSearchItem, int32_t (*tGetItemFn)(uint8_t *, void *), int32_t (*tItemCmprFn)(const void *, const void *), void *pItem) { int32_t code = 0; @@ -198,7 +214,7 @@ int32_t tCmprBlockIdx(void const *lhs, void const *rhs) { int32_t tCmprBlockL(void const *lhs, void const *rhs) { SBlockIdx *lBlockIdx = (SBlockIdx *)lhs; - SBlockL *rBlockL = (SBlockL *)rhs; + SSttBlk *rBlockL = (SSttBlk *)rhs; if (lBlockIdx->suid < rBlockL->suid) { return -1; @@ -215,69 +231,69 @@ int32_t tCmprBlockL(void const *lhs, void const *rhs) { return 0; } -// SBlock ====================================================== -void tBlockReset(SBlock *pBlock) { - *pBlock = (SBlock){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; +// SDataBlk ====================================================== +void tDataBlkReset(SDataBlk *pDataBlk) { + *pDataBlk = (SDataBlk){.minKey = TSDBKEY_MAX, .maxKey = TSDBKEY_MIN, .minVer = VERSION_MAX, .maxVer = VERSION_MIN}; } -int32_t tPutBlock(uint8_t *p, void *ph) { - int32_t n = 0; - SBlock *pBlock = (SBlock *)ph; - - n += tPutI64v(p ? p + n : p, pBlock->minKey.version); - n += tPutI64v(p ? p + n : p, pBlock->minKey.ts); - n += tPutI64v(p ? p + n : p, pBlock->maxKey.version); - n += tPutI64v(p ? p + n : p, pBlock->maxKey.ts); - n += tPutI64v(p ? p + n : p, pBlock->minVer); - n += tPutI64v(p ? p + n : p, pBlock->maxVer); - n += tPutI32v(p ? p + n : p, pBlock->nRow); - n += tPutI8(p ? p + n : p, pBlock->hasDup); - n += tPutI8(p ? p + n : p, pBlock->nSubBlock); - for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tPutI64v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].offset); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szBlock); - n += tPutI32v(p ? p + n : p, pBlock->aSubBlock[iSubBlock].szKey); - } - if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { - n += tPutI64v(p ? p + n : p, pBlock->smaInfo.offset); - n += tPutI32v(p ? p + n : p, pBlock->smaInfo.size); +int32_t tPutDataBlk(uint8_t *p, void *ph) { + int32_t n = 0; + SDataBlk *pDataBlk = (SDataBlk *)ph; + + n += tPutI64v(p ? p + n : p, pDataBlk->minKey.version); + n += tPutI64v(p ? p + n : p, pDataBlk->minKey.ts); + n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.version); + n += tPutI64v(p ? p + n : p, pDataBlk->maxKey.ts); + n += tPutI64v(p ? p + n : p, pDataBlk->minVer); + n += tPutI64v(p ? p + n : p, pDataBlk->maxVer); + n += tPutI32v(p ? p + n : p, pDataBlk->nRow); + n += tPutI8(p ? p + n : p, pDataBlk->hasDup); + n += tPutI8(p ? p + n : p, pDataBlk->nSubBlock); + for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { + n += tPutI64v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].offset); + n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szBlock); + n += tPutI32v(p ? p + n : p, pDataBlk->aSubBlock[iSubBlock].szKey); + } + if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { + n += tPutI64v(p ? p + n : p, pDataBlk->smaInfo.offset); + n += tPutI32v(p ? p + n : p, pDataBlk->smaInfo.size); } return n; } -int32_t tGetBlock(uint8_t *p, void *ph) { - int32_t n = 0; - SBlock *pBlock = (SBlock *)ph; - - n += tGetI64v(p + n, &pBlock->minKey.version); - n += tGetI64v(p + n, &pBlock->minKey.ts); - n += tGetI64v(p + n, &pBlock->maxKey.version); - n += tGetI64v(p + n, &pBlock->maxKey.ts); - n += tGetI64v(p + n, &pBlock->minVer); - n += tGetI64v(p + n, &pBlock->maxVer); - n += tGetI32v(p + n, &pBlock->nRow); - n += tGetI8(p + n, &pBlock->hasDup); - n += tGetI8(p + n, &pBlock->nSubBlock); - for (int8_t iSubBlock = 0; iSubBlock < pBlock->nSubBlock; iSubBlock++) { - n += tGetI64v(p + n, &pBlock->aSubBlock[iSubBlock].offset); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szBlock); - n += tGetI32v(p + n, &pBlock->aSubBlock[iSubBlock].szKey); - } - if (pBlock->nSubBlock == 1 && !pBlock->hasDup) { - n += tGetI64v(p + n, &pBlock->smaInfo.offset); - n += tGetI32v(p + n, &pBlock->smaInfo.size); +int32_t tGetDataBlk(uint8_t *p, void *ph) { + int32_t n = 0; + SDataBlk *pDataBlk = (SDataBlk *)ph; + + n += tGetI64v(p + n, &pDataBlk->minKey.version); + n += tGetI64v(p + n, &pDataBlk->minKey.ts); + n += tGetI64v(p + n, &pDataBlk->maxKey.version); + n += tGetI64v(p + n, &pDataBlk->maxKey.ts); + n += tGetI64v(p + n, &pDataBlk->minVer); + n += tGetI64v(p + n, &pDataBlk->maxVer); + n += tGetI32v(p + n, &pDataBlk->nRow); + n += tGetI8(p + n, &pDataBlk->hasDup); + n += tGetI8(p + n, &pDataBlk->nSubBlock); + for (int8_t iSubBlock = 0; iSubBlock < pDataBlk->nSubBlock; iSubBlock++) { + n += tGetI64v(p + n, &pDataBlk->aSubBlock[iSubBlock].offset); + n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szBlock); + n += tGetI32v(p + n, &pDataBlk->aSubBlock[iSubBlock].szKey); + } + if (pDataBlk->nSubBlock == 1 && !pDataBlk->hasDup) { + n += tGetI64v(p + n, &pDataBlk->smaInfo.offset); + n += tGetI32v(p + n, &pDataBlk->smaInfo.size); } else { - pBlock->smaInfo.offset = 0; - pBlock->smaInfo.size = 0; + pDataBlk->smaInfo.offset = 0; + pDataBlk->smaInfo.size = 0; } return n; } -int32_t tBlockCmprFn(const void *p1, const void *p2) { - SBlock *pBlock1 = (SBlock *)p1; - SBlock *pBlock2 = (SBlock *)p2; +int32_t tDataBlkCmprFn(const void *p1, const void *p2) { + SDataBlk *pBlock1 = (SDataBlk *)p1; + SDataBlk *pBlock2 = (SDataBlk *)p2; if (tsdbKeyCmprFn(&pBlock1->maxKey, &pBlock2->minKey) < 0) { return -1; @@ -288,48 +304,48 @@ int32_t tBlockCmprFn(const void *p1, const void *p2) { return 0; } -bool tBlockHasSma(SBlock *pBlock) { - if (pBlock->nSubBlock > 1) return false; - if (pBlock->hasDup) return false; +bool tDataBlkHasSma(SDataBlk *pDataBlk) { + if (pDataBlk->nSubBlock > 1) return false; + if (pDataBlk->hasDup) return false; - return pBlock->smaInfo.size > 0; + return pDataBlk->smaInfo.size > 0; } -// SBlockL ====================================================== -int32_t tPutBlockL(uint8_t *p, void *ph) { +// SSttBlk ====================================================== +int32_t tPutSttBlk(uint8_t *p, void *ph) { int32_t n = 0; - SBlockL *pBlockL = (SBlockL *)ph; - - n += tPutI64(p ? p + n : p, pBlockL->suid); - n += tPutI64(p ? p + n : p, pBlockL->minUid); - n += tPutI64(p ? p + n : p, pBlockL->maxUid); - n += tPutI64v(p ? p + n : p, pBlockL->minKey); - n += tPutI64v(p ? p + n : p, pBlockL->maxKey); - n += tPutI64v(p ? p + n : p, pBlockL->minVer); - n += tPutI64v(p ? p + n : p, pBlockL->maxVer); - n += tPutI32v(p ? p + n : p, pBlockL->nRow); - n += tPutI64v(p ? p + n : p, pBlockL->bInfo.offset); - n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szBlock); - n += tPutI32v(p ? p + n : p, pBlockL->bInfo.szKey); + SSttBlk *pSttBlk = (SSttBlk *)ph; + + n += tPutI64(p ? p + n : p, pSttBlk->suid); + n += tPutI64(p ? p + n : p, pSttBlk->minUid); + n += tPutI64(p ? p + n : p, pSttBlk->maxUid); + n += tPutI64v(p ? p + n : p, pSttBlk->minKey); + n += tPutI64v(p ? p + n : p, pSttBlk->maxKey); + n += tPutI64v(p ? p + n : p, pSttBlk->minVer); + n += tPutI64v(p ? p + n : p, pSttBlk->maxVer); + n += tPutI32v(p ? p + n : p, pSttBlk->nRow); + n += tPutI64v(p ? p + n : p, pSttBlk->bInfo.offset); + n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szBlock); + n += tPutI32v(p ? p + n : p, pSttBlk->bInfo.szKey); return n; } -int32_t tGetBlockL(uint8_t *p, void *ph) { +int32_t tGetSttBlk(uint8_t *p, void *ph) { int32_t n = 0; - SBlockL *pBlockL = (SBlockL *)ph; - - n += tGetI64(p + n, &pBlockL->suid); - n += tGetI64(p + n, &pBlockL->minUid); - n += tGetI64(p + n, &pBlockL->maxUid); - n += tGetI64v(p + n, &pBlockL->minKey); - n += tGetI64v(p + n, &pBlockL->maxKey); - n += tGetI64v(p + n, &pBlockL->minVer); - n += tGetI64v(p + n, &pBlockL->maxVer); - n += tGetI32v(p + n, &pBlockL->nRow); - n += tGetI64v(p + n, &pBlockL->bInfo.offset); - n += tGetI32v(p + n, &pBlockL->bInfo.szBlock); - n += tGetI32v(p + n, &pBlockL->bInfo.szKey); + SSttBlk *pSttBlk = (SSttBlk *)ph; + + n += tGetI64(p + n, &pSttBlk->suid); + n += tGetI64(p + n, &pSttBlk->minUid); + n += tGetI64(p + n, &pSttBlk->maxUid); + n += tGetI64v(p + n, &pSttBlk->minKey); + n += tGetI64v(p + n, &pSttBlk->maxKey); + n += tGetI64v(p + n, &pSttBlk->minVer); + n += tGetI64v(p + n, &pSttBlk->maxVer); + n += tGetI32v(p + n, &pSttBlk->nRow); + n += tGetI64v(p + n, &pSttBlk->bInfo.offset); + n += tGetI32v(p + n, &pSttBlk->bInfo.szBlock); + n += tGetI32v(p + n, &pSttBlk->bInfo.szKey); return n; } @@ -893,248 +909,6 @@ int32_t tsdbBuildDeleteSkyline(SArray *aDelData, int32_t sidx, int32_t eidx, SAr return code; } -// SColData ======================================== -void tColDataInit(SColData *pColData, int16_t cid, int8_t type, int8_t smaOn) { - pColData->cid = cid; - pColData->type = type; - pColData->smaOn = smaOn; - tColDataReset(pColData); -} - -void tColDataReset(SColData *pColData) { - pColData->nVal = 0; - pColData->flag = 0; - pColData->nData = 0; -} - -void tColDataClear(void *ph) { - SColData *pColData = (SColData *)ph; - - tFree(pColData->pBitMap); - tFree((uint8_t *)pColData->aOffset); - tFree(pColData->pData); -} - -int32_t tColDataAppendValue(SColData *pColData, SColVal *pColVal) { - int32_t code = 0; - int64_t size; - SValue value = {0}; - SValue *pValue = &value; - - ASSERT(pColVal->cid == pColData->cid); - ASSERT(pColVal->type == pColData->type); - - // realloc bitmap - size = BIT2_SIZE(pColData->nVal + 1); - code = tRealloc(&pColData->pBitMap, size); - if (code) goto _exit; - if ((pColData->nVal & 3) == 0) { - pColData->pBitMap[pColData->nVal >> 2] = 0; - } - - // put value - if (pColVal->isNone) { - pColData->flag |= HAS_NONE; - SET_BIT2(pColData->pBitMap, pColData->nVal, 0); - } else if (pColVal->isNull) { - pColData->flag |= HAS_NULL; - SET_BIT2(pColData->pBitMap, pColData->nVal, 1); - } else { - pColData->flag |= HAS_VALUE; - SET_BIT2(pColData->pBitMap, pColData->nVal, 2); - pValue = &pColVal->value; - } - - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - code = tRealloc((uint8_t **)&pColData->aOffset, sizeof(int32_t) * (pColData->nVal + 1)); - if (code) goto _exit; - pColData->aOffset[pColData->nVal] = pColData->nData; - - // value - if ((!pColVal->isNone) && (!pColVal->isNull)) { - code = tRealloc(&pColData->pData, pColData->nData + pColVal->value.nData); - if (code) goto _exit; - memcpy(pColData->pData + pColData->nData, pColVal->value.pData, pColVal->value.nData); - pColData->nData += pColVal->value.nData; - } - } else { - code = tRealloc(&pColData->pData, pColData->nData + tPutValue(NULL, pValue, pColVal->type)); - if (code) goto _exit; - pColData->nData += tPutValue(pColData->pData + pColData->nData, pValue, pColVal->type); - } - - pColData->nVal++; - -_exit: - return code; -} - -int32_t tColDataCopy(SColData *pColDataSrc, SColData *pColDataDest) { - int32_t code = 0; - int32_t size; - - ASSERT(pColDataSrc->nVal > 0); - ASSERT(pColDataDest->cid = pColDataSrc->cid); - ASSERT(pColDataDest->type = pColDataSrc->type); - - pColDataDest->smaOn = pColDataSrc->smaOn; - pColDataDest->nVal = pColDataSrc->nVal; - pColDataDest->flag = pColDataSrc->flag; - - // bitmap - if (pColDataSrc->flag != HAS_NONE && pColDataSrc->flag != HAS_NULL && pColDataSrc->flag != HAS_VALUE) { - size = BIT2_SIZE(pColDataSrc->nVal); - code = tRealloc(&pColDataDest->pBitMap, size); - if (code) goto _exit; - memcpy(pColDataDest->pBitMap, pColDataSrc->pBitMap, size); - } - - // offset - if (IS_VAR_DATA_TYPE(pColDataDest->type)) { - size = sizeof(int32_t) * pColDataSrc->nVal; - - code = tRealloc((uint8_t **)&pColDataDest->aOffset, size); - if (code) goto _exit; - - memcpy(pColDataDest->aOffset, pColDataSrc->aOffset, size); - } - - // value - pColDataDest->nData = pColDataSrc->nData; - code = tRealloc(&pColDataDest->pData, pColDataSrc->nData); - if (code) goto _exit; - memcpy(pColDataDest->pData, pColDataSrc->pData, pColDataDest->nData); - -_exit: - return code; -} - -int32_t tColDataGetValue(SColData *pColData, int32_t iVal, SColVal *pColVal) { - int32_t code = 0; - - ASSERT(iVal < pColData->nVal); - ASSERT(pColData->flag); - - if (pColData->flag == HAS_NONE) { - *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); - goto _exit; - } else if (pColData->flag == HAS_NULL) { - *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); - goto _exit; - } else if (pColData->flag != HAS_VALUE) { - uint8_t v = GET_BIT2(pColData->pBitMap, iVal); - if (v == 0) { - *pColVal = COL_VAL_NONE(pColData->cid, pColData->type); - goto _exit; - } else if (v == 1) { - *pColVal = COL_VAL_NULL(pColData->cid, pColData->type); - goto _exit; - } - } - - // get value - SValue value; - if (IS_VAR_DATA_TYPE(pColData->type)) { - if (iVal + 1 < pColData->nVal) { - value.nData = pColData->aOffset[iVal + 1] - pColData->aOffset[iVal]; - } else { - value.nData = pColData->nData - pColData->aOffset[iVal]; - } - - value.pData = pColData->pData + pColData->aOffset[iVal]; - } else { - tGetValue(pColData->pData + tDataTypes[pColData->type].bytes * iVal, &value, pColData->type); - } - *pColVal = COL_VAL_VALUE(pColData->cid, pColData->type, value); - -_exit: - return code; -} - -int32_t tPutColData(uint8_t *p, SColData *pColData) { - int32_t n = 0; - - n += tPutI16v(p ? p + n : p, pColData->cid); - n += tPutI8(p ? p + n : p, pColData->type); - n += tPutI8(p ? p + n : p, pColData->smaOn); - n += tPutI32v(p ? p + n : p, pColData->nVal); - n += tPutU8(p ? p + n : p, pColData->flag); - - if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; - if (pColData->flag != HAS_VALUE) { - // bitmap - - int32_t size = BIT2_SIZE(pColData->nVal); - if (p) { - memcpy(p + n, pColData->pBitMap, size); - } - n += size; - } - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - - int32_t size = sizeof(int32_t) * pColData->nVal; - if (p) { - memcpy(p + n, pColData->aOffset, size); - } - n += size; - } - n += tPutI32v(p ? p + n : p, pColData->nData); - if (p) { - memcpy(p + n, pColData->pData, pColData->nData); - } - n += pColData->nData; - -_exit: - return n; -} - -int32_t tGetColData(uint8_t *p, SColData *pColData) { - int32_t n = 0; - - n += tGetI16v(p + n, &pColData->cid); - n += tGetI8(p + n, &pColData->type); - n += tGetI8(p + n, &pColData->smaOn); - n += tGetI32v(p + n, &pColData->nVal); - n += tGetU8(p + n, &pColData->flag); - - if (pColData->flag == HAS_NONE || pColData->flag == HAS_NULL) goto _exit; - if (pColData->flag != HAS_VALUE) { - // bitmap - - int32_t size = BIT2_SIZE(pColData->nVal); - pColData->pBitMap = p + n; - n += size; - } - if (IS_VAR_DATA_TYPE(pColData->type)) { - // offset - - int32_t size = sizeof(int32_t) * pColData->nVal; - pColData->aOffset = (int32_t *)(p + n); - n += size; - } - n += tGetI32v(p + n, &pColData->nData); - pColData->pData = p + n; - n += pColData->nData; - -_exit: - return n; -} - -static FORCE_INLINE int32_t tColDataCmprFn(const void *p1, const void *p2) { - SColData *pColData1 = (SColData *)p1; - SColData *pColData2 = (SColData *)p2; - - if (pColData1->cid < pColData2->cid) { - return -1; - } else if (pColData1->cid > pColData2->cid) { - return 1; - } - - return 0; -} - // SBlockData ====================================================== int32_t tBlockDataCreate(SBlockData *pBlockData) { int32_t code = 0; @@ -1166,7 +940,7 @@ void tBlockDataDestroy(SBlockData *pBlockData, int8_t deepClear) { tFree((uint8_t *)pBlockData->aVersion); tFree((uint8_t *)pBlockData->aTSKEY); taosArrayDestroy(pBlockData->aIdx); - taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataClear : NULL); + taosArrayDestroyEx(pBlockData->aColData, deepClear ? tColDataDestroy : NULL); pBlockData->aUid = NULL; pBlockData->aVersion = NULL; pBlockData->aTSKEY = NULL; @@ -1235,7 +1009,7 @@ void tBlockDataClear(SBlockData *pBlockData) { pBlockData->nRow = 0; for (int32_t iColData = 0; iColData < taosArrayGetSize(pBlockData->aIdx); iColData++) { SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, iColData); - tColDataReset(pColData); + tColDataClear(pColData); } } @@ -1485,7 +1259,7 @@ void tBlockDataGetColData(SBlockData *pBlockData, int16_t cid, SColData **ppColD while (lidx <= ridx) { int32_t midx = (lidx + ridx) / 2; SColData *pColData = tBlockDataGetColDataByIdx(pBlockData, midx); - int32_t c = tColDataCmprFn(pColData, &(SColData){.cid = cid}); + int32_t c = (pColData->cid == cid) ? 0 : ((pColData->cid > cid) ? 1 : -1); if (c == 0) { *ppColData = pColData; @@ -1532,7 +1306,7 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, if (code) goto _exit; blockCol.offset = aBufN[0]; - aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue + sizeof(TSCKSUM); + aBufN[0] = aBufN[0] + blockCol.szBitmap + blockCol.szOffset + blockCol.szValue; } code = tRealloc(&aBuf[1], hdr.szBlkCol + tPutBlockCol(NULL, &blockCol)); @@ -1540,15 +1314,8 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, hdr.szBlkCol += tPutBlockCol(aBuf[1] + hdr.szBlkCol, &blockCol); } - aBufN[1] = 0; - if (hdr.szBlkCol > 0) { - aBufN[1] = hdr.szBlkCol + sizeof(TSCKSUM); - - code = tRealloc(&aBuf[1], aBufN[1]); - if (code) goto _exit; - - taosCalcChecksumAppend(0, aBuf[1], aBufN[1]); - } + // SBlockCol + aBufN[1] = hdr.szBlkCol; // uid + version + tskey aBufN[2] = 0; @@ -1569,16 +1336,11 @@ int32_t tCmprBlockData(SBlockData *pBlockData, int8_t cmprAlg, uint8_t **ppOut, if (code) goto _exit; aBufN[2] += hdr.szKey; - aBufN[2] += sizeof(TSCKSUM); - code = tRealloc(&aBuf[2], aBufN[2]); - if (code) goto _exit; - // hdr aBufN[3] = tPutDiskDataHdr(NULL, &hdr); code = tRealloc(&aBuf[3], aBufN[3]); if (code) goto _exit; tPutDiskDataHdr(aBuf[3], &hdr); - taosCalcChecksumAppend(taosCalcChecksum(0, aBuf[3], aBufN[3]), aBuf[2], aBufN[2]); // aggragate if (ppOut) { @@ -1603,17 +1365,13 @@ _exit: int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uint8_t *aBuf[]) { int32_t code = 0; - tBlockDataClear(pBlockData); + tBlockDataReset(pBlockData); int32_t n = 0; SDiskDataHdr hdr = {0}; // SDiskDataHdr n += tGetDiskDataHdr(pIn + n, &hdr); - if (!taosCheckChecksumWhole(pIn, n + hdr.szUid + hdr.szVer + hdr.szKey + sizeof(TSCKSUM))) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } ASSERT(hdr.delimiter == TSDB_FILE_DLMT); pBlockData->suid = hdr.suid; @@ -1641,7 +1399,7 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin code = tsdbDecmprData(pIn + n, hdr.szKey, TSDB_DATA_TYPE_TIMESTAMP, hdr.cmprAlg, (uint8_t **)&pBlockData->aTSKEY, sizeof(TSKEY) * hdr.nRow, &aBuf[0]); if (code) goto _exit; - n = n + hdr.szKey + sizeof(TSCKSUM); + n += hdr.szKey; // loop to decode each column data if (hdr.szBlkCol == 0) goto _exit; @@ -1663,8 +1421,8 @@ int32_t tDecmprBlockData(uint8_t *pIn, int32_t szIn, SBlockData *pBlockData, uin if (code) goto _exit; } } else { - code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + sizeof(TSCKSUM) + blockCol.offset, &blockCol, hdr.cmprAlg, - hdr.nRow, pColData, &aBuf[0]); + code = tsdbDecmprColData(pIn + n + hdr.szBlkCol + blockCol.offset, &blockCol, hdr.cmprAlg, hdr.nRow, pColData, + &aBuf[0]); if (code) goto _exit; } } @@ -1986,47 +1744,16 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol int32_t size = 0; // bitmap if (pColData->flag != HAS_VALUE) { - uint8_t *pBitMap = pColData->pBitMap; - int32_t szBitMap = BIT2_SIZE(pColData->nVal); - - // BIT2 to BIT1 - if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { + int32_t szBitMap; + if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + szBitMap = BIT2_SIZE(pColData->nVal); + } else { szBitMap = BIT1_SIZE(pColData->nVal); - pBitMap = taosMemoryCalloc(1, szBitMap); - if (pBitMap == NULL) { - code = TSDB_CODE_OUT_OF_MEMORY; - goto _exit; - } - - for (int32_t iVal = 0; iVal < pColData->nVal; iVal++) { - uint8_t v = GET_BIT2(pColData->pBitMap, iVal); - switch (pColData->flag) { - case (HAS_NULL | HAS_NONE): - SET_BIT1(pBitMap, iVal, v); - break; - case (HAS_VALUE | HAS_NONE): - if (v) { - SET_BIT1(pBitMap, iVal, 1); - } else { - SET_BIT1(pBitMap, iVal, 0); - } - break; - case (HAS_VALUE | HAS_NULL): - SET_BIT1(pBitMap, iVal, v - 1); - break; - default: - ASSERT(0); - } - } } - code = tsdbCmprData(pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, &pBlockCol->szBitmap, - ppBuf); + code = tsdbCmprData(pColData->pBitMap, szBitMap, TSDB_DATA_TYPE_TINYINT, cmprAlg, ppOut, nOut + size, + &pBlockCol->szBitmap, ppBuf); if (code) goto _exit; - - if (pColData->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { - taosMemoryFree(pBitMap); - } } size += pBlockCol->szBitmap; @@ -2039,19 +1766,13 @@ int32_t tsdbCmprColData(SColData *pColData, int8_t cmprAlg, SBlockCol *pBlockCol size += pBlockCol->szOffset; // value - if (pColData->flag != (HAS_NULL | HAS_NONE)) { + if ((pColData->flag != (HAS_NULL | HAS_NONE)) && pColData->nData) { code = tsdbCmprData((uint8_t *)pColData->pData, pColData->nData, pColData->type, cmprAlg, ppOut, nOut + size, &pBlockCol->szValue, ppBuf); if (code) goto _exit; } size += pBlockCol->szValue; - // checksum - size += sizeof(TSCKSUM); - code = tRealloc(ppOut, nOut + size); - if (code) goto _exit; - taosCalcChecksumAppend(0, *ppOut + nOut, size); - _exit: return code; } @@ -2060,12 +1781,6 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t **ppBuf) { int32_t code = 0; - int32_t size = pBlockCol->szBitmap + pBlockCol->szOffset + pBlockCol->szValue + sizeof(TSCKSUM); - if (!taosCheckChecksumWhole(pIn, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - ASSERT(pColData->cid == pBlockCol->cid); ASSERT(pColData->type == pBlockCol->type); pColData->smaOn = pBlockCol->smaOn; @@ -2076,46 +1791,15 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in uint8_t *p = pIn; // bitmap if (pBlockCol->szBitmap) { - if (pBlockCol->flag != (HAS_VALUE | HAS_NULL | HAS_NONE)) { - uint8_t *pBitMap = NULL; - code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pBitMap, - BIT1_SIZE(pColData->nVal), ppBuf); - if (code) goto _exit; - - code = tRealloc(&pColData->pBitMap, BIT2_SIZE(pColData->nVal)); - if (code) { - tFree(pBitMap); - goto _exit; - } - - // BIT1 to BIT2 - for (int32_t iVal = 0; iVal < nVal; iVal++) { - uint8_t v = GET_BIT1(pBitMap, iVal); - switch (pBlockCol->flag) { - case (HAS_NULL | HAS_NONE): - SET_BIT2(pColData->pBitMap, iVal, v); - break; - case (HAS_VALUE | HAS_NONE): - if (v) { - SET_BIT2(pColData->pBitMap, iVal, 2); - } else { - SET_BIT2(pColData->pBitMap, iVal, 0); - } - break; - case (HAS_VALUE | HAS_NULL): - SET_BIT2(pColData->pBitMap, iVal, v + 1); - break; - default: - ASSERT(0); - } - } - - tFree(pBitMap); + int32_t szBitMap; + if (pColData->flag == (HAS_VALUE | HAS_NULL | HAS_NONE)) { + szBitMap = BIT2_SIZE(pColData->nVal); } else { - code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, - BIT2_SIZE(pColData->nVal), ppBuf); - if (code) goto _exit; + szBitMap = BIT1_SIZE(pColData->nVal); } + + code = tsdbDecmprData(p, pBlockCol->szBitmap, TSDB_DATA_TYPE_TINYINT, cmprAlg, &pColData->pBitMap, szBitMap, ppBuf); + if (code) goto _exit; } p += pBlockCol->szBitmap; @@ -2137,37 +1821,3 @@ int32_t tsdbDecmprColData(uint8_t *pIn, SBlockCol *pBlockCol, int8_t cmprAlg, in _exit: return code; } - -int32_t tsdbReadAndCheck(TdFilePtr pFD, int64_t offset, uint8_t **ppOut, int32_t size, int8_t toCheck) { - int32_t code = 0; - - // alloc - code = tRealloc(ppOut, size); - if (code) goto _exit; - - // seek - int64_t n = taosLSeekFile(pFD, offset, SEEK_SET); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } - - // read - n = taosReadFile(pFD, *ppOut, size); - if (n < 0) { - code = TAOS_SYSTEM_ERROR(errno); - goto _exit; - } else if (n < size) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - - // check - if (toCheck && !taosCheckChecksumWhole(*ppOut, size)) { - code = TSDB_CODE_FILE_CORRUPTED; - goto _exit; - } - -_exit: - return code; -} diff --git a/source/dnode/vnode/src/tsdb/tsdbWrite.c b/source/dnode/vnode/src/tsdb/tsdbWrite.c index 383652531e211504983444d9d783ddf9189f5161..0a9fbf92a4bf62326aa9755b827b83d0d510d2f7 100644 --- a/source/dnode/vnode/src/tsdb/tsdbWrite.c +++ b/source/dnode/vnode/src/tsdb/tsdbWrite.c @@ -39,7 +39,7 @@ int tsdbInsertData(STsdb *pTsdb, int64_t version, SSubmitReq *pMsg, SSubmitRsp * SSubmitBlkRsp r = {0}; tGetSubmitMsgNext(&msgIter, &pBlock); if (pBlock == NULL) break; - if (tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r) < 0) { + if ((terrno = tsdbInsertTableData(pTsdb, version, &msgIter, pBlock, &r)) < 0) { return -1; } diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index 580ab8bc93cac3a5057821f238cf85fc1011fa38..5adb2eb3592e7bcba7803b5e2972f0bdd3689adb 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -13,6 +13,7 @@ * along with this program. If not, see . */ +#include "tutil.h" #include "vnd.h" const SVnodeCfg vnodeCfgDefault = {.vgId = -1, @@ -47,7 +48,9 @@ const SVnodeCfg vnodeCfgDefault = {.vgId = -1, }, .hashBegin = 0, .hashEnd = 0, - .hashMethod = 0}; + .hashMethod = 0, + .sttTrigger = TSDB_DEFAULT_STT_FILE, + .tsdbPageSize = TSDB_DEFAULT_PAGE_SIZE}; int vnodeCheckCfg(const SVnodeCfg *pCfg) { // TODO @@ -106,9 +109,13 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddIntegerToObject(pJson, "wal.retentionSize", pCfg->walCfg.retentionSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.segSize", pCfg->walCfg.segSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "wal.level", pCfg->walCfg.level) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "sstTrigger", pCfg->sttTrigger) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashBegin", pCfg->hashBegin) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashEnd", pCfg->hashEnd) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "hashMethod", pCfg->hashMethod) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashPrefix", pCfg->hashPrefix) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "hashSuffix", pCfg->hashSuffix) < 0) return -1; + if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum) < 0) return -1; if (tjsonAddIntegerToObject(pJson, "syncCfg.myIndex", pCfg->syncCfg.myIndex) < 0) return -1; @@ -128,6 +135,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { tjsonAddItemToArray(pNodeInfoArr, pNodeInfo); } + // add tsdb page size config + if (tjsonAddIntegerToObject(pJson, "tsdbPageSize", pCfg->tsdbPageSize) < 0) return -1; + return 0; } @@ -205,12 +215,18 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(pJson, "wal.level", pCfg->walCfg.level, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "sstTrigger", pCfg->sttTrigger, code); + if (code < 0) pCfg->sttTrigger = TSDB_DEFAULT_SST_TRIGGER; tjsonGetNumberValue(pJson, "hashBegin", pCfg->hashBegin, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "hashEnd", pCfg->hashEnd, code); if (code < 0) return -1; tjsonGetNumberValue(pJson, "hashMethod", pCfg->hashMethod, code); if (code < 0) return -1; + tjsonGetNumberValue(pJson, "hashPrefix", pCfg->hashPrefix, code); + if (code < 0) pCfg->hashPrefix = TSDB_DEFAULT_HASH_PREFIX; + tjsonGetNumberValue(pJson, "hashSuffix", pCfg->hashSuffix, code); + if (code < 0) pCfg->hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; tjsonGetNumberValue(pJson, "syncCfg.replicaNum", pCfg->syncCfg.replicaNum, code); if (code < 0) return -1; @@ -239,6 +255,9 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { tjsonGetStringValue(pNodeInfo, "nodeFqdn", (pCfg->syncCfg.nodeInfo)[i].nodeFqdn); } + tjsonGetNumberValue(pJson, "tsdbPageSize", pCfg->tsdbPageSize, code); + if (code < 0) pCfg->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE * 1024; + return 0; } @@ -247,7 +266,8 @@ int vnodeValidateTableHash(SVnode *pVnode, char *tableFName) { switch (pVnode->config.hashMethod) { default: - hashValue = MurmurHash3_32(tableFName, strlen(tableFName)); + hashValue = taosGetTbHashVal(tableFName, strlen(tableFName), pVnode->config.hashMethod, pVnode->config.hashPrefix, + pVnode->config.hashSuffix); break; } diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index a4fd984fb762a934b48489da254b3af1aa4dc908..4ccfea40510a6148ceec57893629c693d9d7a1cd 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -60,6 +60,8 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { snprintf(dir, TSDB_FILENAME_LEN, "%s%s%s", tfsGetPrimaryPath(pTfs), TD_DIRSEP, path); + info.config = vnodeCfgDefault; + // load vnode info ret = vnodeLoadInfo(dir, &info); if (ret < 0) { diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 1461d371810a0328b87d40102b602ab9aee8bb26..825bf8a0a7c0adbf5a19c08c2e2418da3b6e571a 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -368,6 +368,7 @@ _exit: int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); pLoad->syncState = syncGetMyRole(pVnode->sync); + pLoad->cacheUsage = tsdbCacheGetUsage(pVnode); pLoad->numOfTables = metaGetTbNum(pVnode->pMeta); pLoad->numOfTimeSeries = metaGetTimeSeriesNum(pVnode->pMeta); pLoad->totalStorage = (int64_t)3 * 1073741824; @@ -424,8 +425,8 @@ int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list) { return TSDB_CODE_SUCCESS; } -int32_t vnodeGetStbIdList(SVnode* pVnode, int64_t suid, SArray* list) { - SMStbCursor* pCur = metaOpenStbCursor(pVnode->pMeta, suid); +int32_t vnodeGetStbIdList(SVnode *pVnode, int64_t suid, SArray *list) { + SMStbCursor *pCur = metaOpenStbCursor(pVnode->pMeta, suid); if (!pCur) { return TSDB_CODE_FAILED; } @@ -467,9 +468,13 @@ static int32_t vnodeGetStbColumnNum(SVnode *pVnode, tb_uid_t suid, int *num) { STSchema *pTSchema = metaGetTbTSchema(pVnode->pMeta, suid, -1); // metaGetTbTSchemaEx(pVnode->pMeta, suid, suid, -1, &pTSchema); - *num = pTSchema->numOfCols; + if (pTSchema) { + *num = pTSchema->numOfCols; - taosMemoryFree(pTSchema); + taosMemoryFree(pTSchema); + } else { + *num = 2; + } return TSDB_CODE_SUCCESS; } diff --git a/source/dnode/vnode/src/vnd/vnodeSnapshot.c b/source/dnode/vnode/src/vnd/vnodeSnapshot.c index 5a81f9191920ffac0ae055e140e7e59448dfd406..08c3a34699b6f9f83b366e42f77e42a6094f09ed 100644 --- a/source/dnode/vnode/src/vnd/vnodeSnapshot.c +++ b/source/dnode/vnode/src/vnd/vnodeSnapshot.c @@ -39,7 +39,7 @@ struct SVSnapReader { SStreamStateReader *pStreamStateReader; // rsma int8_t rsmaDone; - SRsmaSnapReader *pRsmaReader; + SRSmaSnapReader *pRsmaReader; }; int32_t vnodeSnapReaderOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapReader **ppReader) { @@ -241,7 +241,7 @@ struct SVSnapWriter { SStreamTaskWriter *pStreamTaskWriter; SStreamStateWriter *pStreamStateWriter; // rsma - SRsmaSnapWriter *pRsmaSnapWriter; + SRSmaSnapWriter *pRsmaSnapWriter; }; int32_t vnodeSnapWriterOpen(SVnode *pVnode, int64_t sver, int64_t ever, SVSnapWriter **ppWriter) { @@ -354,7 +354,8 @@ int32_t vnodeSnapWrite(SVSnapWriter *pWriter, uint8_t *pData, uint32_t nData) { code = metaSnapWrite(pWriter->pMetaSnapWriter, pData, nData); if (code) goto _err; } break; - case SNAP_DATA_TSDB: { + case SNAP_DATA_TSDB: + case SNAP_DATA_DEL: { // tsdb if (pWriter->pTsdbSnapWriter == NULL) { code = tsdbSnapWriterOpen(pVnode->pTsdb, pWriter->sver, pWriter->ever, &pWriter->pTsdbSnapWriter); diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 85feecff1a0247790eec14b503b7dda6247a6c87..51d83d8eedf6845e67a5ee81ff96debe52275bd1 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -371,7 +371,7 @@ void vnodeUpdateMetaRsp(SVnode *pVnode, STableMetaRsp *pMetaRsp) { if (NULL == pMetaRsp) { return; } - + strcpy(pMetaRsp->dbFName, pVnode->config.dbname); pMetaRsp->dbId = pVnode->config.dbId; pMetaRsp->vgId = TD_VID(pVnode); @@ -496,6 +496,7 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR // loop to create table for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { pCreateReq = req.pReqs + iReq; + memset(&cRsp, 0, sizeof(cRsp)); if ((terrno = grantCheck(TSDB_GRANT_TIMESERIES)) < 0) { rcode = -1; @@ -526,14 +527,14 @@ static int32_t vnodeProcessCreateTbReq(SVnode *pVnode, int64_t version, void *pR cRsp.code = TSDB_CODE_SUCCESS; tdFetchTbUidList(pVnode->pSma, &pStore, pCreateReq->ctb.suid, pCreateReq->uid); taosArrayPush(tbUids, &pCreateReq->uid); - vnodeUpdateMetaRsp(pVnode, cRsp.pMeta); + vnodeUpdateMetaRsp(pVnode, cRsp.pMeta); } taosArrayPush(rsp.pArray, &cRsp); } tqUpdateTbUidList(pVnode->pTq, tbUids, true); - if (tdUpdateTbUidList(pVnode->pSma, pStore) < 0) { + if (tdUpdateTbUidList(pVnode->pSma, pStore, true) < 0) { goto _exit; } tdUidStoreFree(pStore); @@ -691,6 +692,7 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq SEncoder encoder = {0}; int32_t ret; SArray *tbUids = NULL; + STbUidStore *pStore = NULL; pRsp->msgType = TDMT_VND_DROP_TABLE_RSP; pRsp->pCont = NULL; @@ -714,9 +716,10 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq for (int32_t iReq = 0; iReq < req.nReqs; iReq++) { SVDropTbReq *pDropTbReq = req.pReqs + iReq; SVDropTbRsp dropTbRsp = {0}; + tb_uid_t tbUid = 0; /* code */ - ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids); + ret = metaDropTable(pVnode->pMeta, version, pDropTbReq, tbUids, &tbUid); if (ret < 0) { if (pDropTbReq->igNotExists && terrno == TSDB_CODE_VND_TABLE_NOT_EXIST) { dropTbRsp.code = TSDB_CODE_SUCCESS; @@ -725,15 +728,18 @@ static int32_t vnodeProcessDropTbReq(SVnode *pVnode, int64_t version, void *pReq } } else { dropTbRsp.code = TSDB_CODE_SUCCESS; + if (tbUid > 0) tdFetchTbUidList(pVnode->pSma, &pStore, pDropTbReq->suid, tbUid); } taosArrayPush(rsp.pArray, &dropTbRsp); } tqUpdateTbUidList(pVnode->pTq, tbUids, false); + tdUpdateTbUidList(pVnode->pSma, pStore, false); _exit: taosArrayDestroy(tbUids); + tdUidStoreFree(pStore); tDecoderClear(&decoder); tEncodeSize(tEncodeSVDropTbBatchRsp, &rsp, pRsp->contLen, ret); pRsp->pCont = rpcMallocCont(pRsp->contLen); @@ -1106,6 +1112,7 @@ static int32_t vnodeProcessDeleteReq(SVnode *pVnode, int64_t version, void *pReq tDecoderInit(pCoder, pReq, len); tDecodeDeleteRes(pCoder, pRes); + ASSERT(taosArrayGetSize(pRes->uidList) == 0 || (pRes->skey != 0 && pRes->ekey != 0)); for (int32_t iUid = 0; iUid < taosArrayGetSize(pRes->uidList); iUid++) { code = tsdbDeleteTableData(pVnode->pTsdb, version, pRes->suid, *(uint64_t *)taosArrayGet(pRes->uidList, iUid), diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 6935489ff4000aa37aed912769120713102ed4c1..3a774afdcce2fc21334686a39a65fcb83606bab2 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -13,96 +13,46 @@ * along with this program. If not, see . */ -#include "trpc.h" -#include "query.h" -#include "tname.h" #include "catalogInt.h" +#include "query.h" #include "systable.h" +#include "tname.h" +#include "trpc.h" -SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = { - { - CTG_OP_UPDATE_VGROUP, - "update vgInfo", - ctgOpUpdateVgroup - }, - { - CTG_OP_UPDATE_TB_META, - "update tbMeta", - ctgOpUpdateTbMeta - }, - { - CTG_OP_DROP_DB_CACHE, - "drop DB", - ctgOpDropDbCache - }, - { - CTG_OP_DROP_DB_VGROUP, - "drop DBVgroup", - ctgOpDropDbVgroup - }, - { - CTG_OP_DROP_STB_META, - "drop stbMeta", - ctgOpDropStbMeta - }, - { - CTG_OP_DROP_TB_META, - "drop tbMeta", - ctgOpDropTbMeta - }, - { - CTG_OP_UPDATE_USER, - "update user", - ctgOpUpdateUser - }, - { - CTG_OP_UPDATE_VG_EPSET, - "update epset", - ctgOpUpdateEpset - }, - { - CTG_OP_UPDATE_TB_INDEX, - "update tbIndex", - ctgOpUpdateTbIndex - }, - { - CTG_OP_DROP_TB_INDEX, - "drop tbIndex", - ctgOpDropTbIndex - }, - { - CTG_OP_CLEAR_CACHE, - "clear cache", - ctgOpClearCache - } -}; - - - +SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update vgInfo", ctgOpUpdateVgroup}, + {CTG_OP_UPDATE_TB_META, "update tbMeta", ctgOpUpdateTbMeta}, + {CTG_OP_DROP_DB_CACHE, "drop DB", ctgOpDropDbCache}, + {CTG_OP_DROP_DB_VGROUP, "drop DBVgroup", ctgOpDropDbVgroup}, + {CTG_OP_DROP_STB_META, "drop stbMeta", ctgOpDropStbMeta}, + {CTG_OP_DROP_TB_META, "drop tbMeta", ctgOpDropTbMeta}, + {CTG_OP_UPDATE_USER, "update user", ctgOpUpdateUser}, + {CTG_OP_UPDATE_VG_EPSET, "update epset", ctgOpUpdateEpset}, + {CTG_OP_UPDATE_TB_INDEX, "update tbIndex", ctgOpUpdateTbIndex}, + {CTG_OP_DROP_TB_INDEX, "drop tbIndex", ctgOpDropTbIndex}, + {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); - + if (dbCache->deleted) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); - ctgDebug("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); - + ctgDebug("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); + *inCache = false; return TSDB_CODE_SUCCESS; } - if (NULL == dbCache->vgCache.vgInfo) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); *inCache = false; - ctgDebug("db vgInfo is empty, dbId:0x%"PRIx64, dbCache->dbId); + ctgDebug("db vgInfo is empty, dbId:0x%" PRIx64, dbCache->dbId); return TSDB_CODE_SUCCESS; } *inCache = true; - + return TSDB_CODE_SUCCESS; } @@ -110,7 +60,7 @@ int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_LOCK(CTG_WRITE, &dbCache->vgCache.vgLock); if (dbCache->deleted) { - ctgDebug("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); + ctgDebug("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } @@ -118,19 +68,13 @@ int32_t ctgWLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache) { return TSDB_CODE_SUCCESS; } -void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); -} +void ctgRUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->vgCache.vgLock); } -void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); -} +void ctgWUnlockVgInfo(SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_WRITE, &dbCache->vgCache.vgLock); } -void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { - CTG_UNLOCK(CTG_READ, &dbCache->dbLock); -} +void ctgReleaseDBCache(SCatalog *pCtg, SCtgDBCache *dbCache) { CTG_UNLOCK(CTG_READ, &dbCache->dbLock); } -int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { +int32_t ctgAcquireDBCacheImpl(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache, bool acquire) { char *p = strchr(dbFName, '.'); if (p && IS_SYS_DBNAME(p + 1)) { dbFName = p + 1; @@ -150,35 +94,35 @@ int32_t ctgAcquireDBCacheImpl(SCatalog* pCtg, const char *dbFName, SCtgDBCache * if (dbCache->deleted) { if (acquire) { ctgReleaseDBCache(pCtg, dbCache); - } - + } + *pCache = NULL; ctgDebug("db is removing from cache, dbFName:%s", dbFName); return TSDB_CODE_SUCCESS; } *pCache = dbCache; - + return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgAcquireDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, true)); } -int32_t ctgGetDBCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgGetDBCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { CTG_RET(ctgAcquireDBCacheImpl(pCtg, dbFName, pCache, false)); } -void ctgReleaseVgInfoToCache(SCatalog* pCtg, SCtgDBCache *dbCache) { +void ctgReleaseVgInfoToCache(SCatalog *pCtg, SCtgDBCache *dbCache) { ctgRUnlockVgInfo(dbCache); ctgReleaseDBCache(pCtg, dbCache); } -void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { +void ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache) { if (pCache) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); + taosHashRelease(dbCache->tbCache, pCache); } if (dbCache) { @@ -186,10 +130,10 @@ void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* } } -void ctgReleaseTbIndexToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* pCache) { +void ctgReleaseTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache) { if (pCache) { CTG_UNLOCK(CTG_READ, &pCache->indexLock); - taosHashRelease(dbCache->tbCache, pCache); + taosHashRelease(dbCache->tbCache, pCache); } if (dbCache) { @@ -197,10 +141,10 @@ void ctgReleaseTbIndexToCache(SCatalog* pCtg, SCtgDBCache *dbCache, SCtgTbCache* } } -int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCache **pCache) { +int32_t ctgAcquireVgInfoFromCache(SCatalog *pCtg, const char *dbFName, SCtgDBCache **pCache) { SCtgDBCache *dbCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); - if (NULL == dbCache) { + if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } @@ -217,7 +161,7 @@ int32_t ctgAcquireVgInfoFromCache(SCatalog* pCtg, const char *dbFName, SCtgDBCac CTG_CACHE_STAT_INC(numOfVgHit, 1); ctgDebug("Got db vgInfo from cache, dbFName:%s", dbFName); - + return TSDB_CODE_SUCCESS; _return: @@ -229,19 +173,19 @@ _return: *pCache = NULL; CTG_CACHE_STAT_INC(numOfVgMiss, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { +int32_t ctgAcquireTbMetaFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; - SCtgTbCache* pCache = NULL; + SCtgTbCache *pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } - + pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); @@ -258,7 +202,7 @@ int32_t ctgAcquireTbMetaFromCache(SCatalog* pCtg, char *dbFName, char* tbName, S *pTb = pCache; ctgDebug("tb %s meta got in cache, dbFName:%s", tbName, dbFName); - + CTG_CACHE_STAT_INC(numOfMetaHit, 1); return TSDB_CODE_SUCCESS; @@ -268,20 +212,20 @@ _return: ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_CACHE_STAT_INC(numOfMetaMiss, 1); - + return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache** pTb) { - SCtgDBCache* dbCache = NULL; - SCtgTbCache* pCache = NULL; +int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) { + SCtgDBCache *dbCache = NULL; + SCtgTbCache *pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } - - char* stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + + char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); goto _return; @@ -304,7 +248,7 @@ int32_t ctgAcquireStbMetaFromCache(SCatalog* pCtg, char *dbFName, uint64_t suid, *pTb = pCache; ctgDebug("stb 0x%" PRIx64 " meta got in cache, dbFName:%s", suid, dbFName); - + CTG_CACHE_STAT_INC(numOfMetaHit, 1); return TSDB_CODE_SUCCESS; @@ -317,20 +261,19 @@ _return: *pDb = NULL; *pTb = NULL; - + return TSDB_CODE_SUCCESS; } - -int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { +int32_t ctgAcquireTbIndexFromCache(SCatalog *pCtg, char *dbFName, char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; - SCtgTbCache* pCache = NULL; + SCtgTbCache *pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); goto _return; } - + int32_t sz = 0; pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { @@ -348,7 +291,7 @@ int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, *pTb = pCache; ctgDebug("tb %s index got in cache, dbFName:%s", tbName, dbFName); - + CTG_CACHE_STAT_INC(numOfIndexHit, 1); return TSDB_CODE_SUCCESS; @@ -358,32 +301,31 @@ _return: ctgReleaseTbIndexToCache(pCtg, dbCache, pCache); CTG_CACHE_STAT_INC(numOfIndexMiss, 1); - + return TSDB_CODE_SUCCESS; } - -int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char *dbFName, char* tbName, int32_t *exist) { +int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *exist) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - + *exist = 0; return TSDB_CODE_SUCCESS; } *exist = 1; ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - + return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { - int32_t code = 0; +int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { + int32_t code = 0; SCtgDBCache *dbCache = NULL; - SCtgTbCache *tbCache = NULL; + SCtgTbCache *tbCache = NULL; *pTableMeta = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; @@ -399,12 +341,12 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** return TSDB_CODE_SUCCESS; } - STableMeta* tbMeta = tbCache->pMeta; + STableMeta *tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; ctx->tbInfo.tbType = tbMeta->tableType; - + if (tbMeta->tableType != TSDB_CHILD_TABLE) { int32_t metaSize = CTG_META_SIZE(tbMeta); *pTableMeta = taosMemoryCalloc(1, metaSize); @@ -414,14 +356,14 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** } memcpy(*pTableMeta, tbMeta, metaSize); - + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", ctx->pName->tname, tbMeta->tableType, dbFName); return TSDB_CODE_SUCCESS; } // PROCESS FOR CHILD TABLE - + int32_t metaSize = sizeof(SCTableMeta); *pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == *pTableMeta) { @@ -429,10 +371,10 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** } memcpy(*pTableMeta, tbMeta, metaSize); - + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", - ctx->pName->tname, ctx->tbInfo.tbType, dbFName); + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, + ctx->tbInfo.tbType, dbFName); ctgAcquireStbMetaFromCache(pCtg, dbFName, ctx->tbInfo.suid, &dbCache, &tbCache); if (NULL == tbCache) { @@ -441,17 +383,17 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgDebug("stb 0x%" PRIx64 " meta not in cache", ctx->tbInfo.suid); return TSDB_CODE_SUCCESS; } - - STableMeta* stbMeta = tbCache->pMeta; - if (stbMeta->suid != ctx->tbInfo.suid) { + + STableMeta *stbMeta = tbCache->pMeta; + if (stbMeta->suid != ctx->tbInfo.suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, ctx->tbInfo.suid); + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%" PRIx64, stbMeta->suid, ctx->tbInfo.suid); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } metaSize = CTG_META_SIZE(stbMeta); *pTableMeta = taosMemoryRealloc(*pTableMeta, metaSize); - if (NULL == *pTableMeta) { + if (NULL == *pTableMeta) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -461,24 +403,24 @@ int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got tb %s meta from cache, dbFName:%s", ctx->pName->tname, dbFName); - + return TSDB_CODE_SUCCESS; _return: ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); taosMemoryFreeClear(*pTableMeta); - + CTG_RET(code); } -int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, uint64_t *suid, - char *stbName) { +int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, int32_t *tver, int32_t *tbType, + uint64_t *suid, char *stbName) { *sver = -1; *tver = -1; SCtgDBCache *dbCache = NULL; - SCtgTbCache *tbCache = NULL; + SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); @@ -488,7 +430,7 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, return TSDB_CODE_SUCCESS; } - STableMeta* tbMeta = tbCache->pMeta; + STableMeta *tbMeta = tbCache->pMeta; *tbType = tbMeta->tableType; *suid = tbMeta->suid; @@ -496,29 +438,29 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, *sver = tbMeta->sversion; *tver = tbMeta->tversion; - ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:0x%" PRIx64, - pTableName->tname, dbFName, *tbType, *sver, *tver, *suid); + ctgDebug("Got tb %s ver from cache, dbFName:%s, tbType:%d, sver:%d, tver:%d, suid:0x%" PRIx64, pTableName->tname, + dbFName, *tbType, *sver, *tver, *suid); ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; } // PROCESS FOR CHILD TABLE - + ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); - + ctgAcquireStbMetaFromCache(pCtg, dbFName, *suid, &dbCache, &tbCache); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } - - STableMeta* stbMeta = tbCache->pMeta; + + STableMeta *stbMeta = tbCache->pMeta; if (stbMeta->suid != *suid) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid:0x%" PRIx64 , stbMeta->suid, *suid); + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid:0x%" PRIx64, stbMeta->suid, *suid); CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -533,15 +475,15 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("Got tb %s sver %d tver %d from cache, type:%d, dbFName:%s", pTableName->tname, *sver, *tver, *tbType, dbFName); + ctgDebug("Got tb %s sver %d tver %d from cache, type:%d, dbFName:%s", pTableName->tname, *sver, *tver, *tbType, + dbFName); return TSDB_CODE_SUCCESS; } - -int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tbName, int32_t *tbType) { +int32_t ctgReadTbTypeFromCache(SCatalog *pCtg, char *dbFName, char *tbName, int32_t *tbType) { SCtgDBCache *dbCache = NULL; - SCtgTbCache *tbCache = NULL; + SCtgTbCache *tbCache = NULL; CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); @@ -551,15 +493,15 @@ int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tbName, int3 *tbType = tbCache->pMeta->tableType; ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tbName, *tbType, dbFName); - + ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tbName, *tbType, dbFName); + return TSDB_CODE_SUCCESS; } -int32_t ctgReadTbIndexFromCache(SCatalog* pCtg, SName* pTableName, SArray** pRes) { - int32_t code = 0; +int32_t ctgReadTbIndexFromCache(SCatalog *pCtg, SName *pTableName, SArray **pRes) { + int32_t code = 0; SCtgDBCache *dbCache = NULL; - SCtgTbCache *tbCache = NULL; + SCtgTbCache *tbCache = NULL; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); @@ -580,14 +522,14 @@ _return: CTG_RET(code); } -int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { +int32_t ctgChkAuthFromCache(SCatalog *pCtg, char *user, char *dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { char *p = strchr(dbFName, '.'); if (p) { ++p; } else { p = dbFName; } - + if (IS_SYS_DBNAME(p)) { *inCache = true; *pass = true; @@ -605,7 +547,7 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE ctgDebug("Got user from cache, user:%s", user); CTG_CACHE_STAT_INC(numOfUserHit, 1); - + if (pUser->superUser) { *pass = true; return TSDB_CODE_SUCCESS; @@ -617,54 +559,53 @@ int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE CTG_UNLOCK(CTG_READ, &pUser->lock); return TSDB_CODE_SUCCESS; } - + if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_READ) { *pass = true; } - + if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && type == AUTH_TYPE_WRITE) { *pass = true; } CTG_UNLOCK(CTG_READ, &pUser->lock); - + return TSDB_CODE_SUCCESS; _return: *inCache = false; CTG_CACHE_STAT_INC(numOfUserMiss, 1); - + return TSDB_CODE_SUCCESS; } void ctgDequeue(SCtgCacheOperation **op) { SCtgQNode *orig = gCtgMgmt.queue.head; - + SCtgQNode *node = gCtgMgmt.queue.head->next; gCtgMgmt.queue.head = gCtgMgmt.queue.head->next; CTG_QUEUE_DEC(); - + taosMemoryFreeClear(orig); *op = node->op; } - -int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) { +int32_t ctgEnqueue(SCatalog *pCtg, SCtgCacheOperation *operation) { SCtgQNode *node = taosMemoryCalloc(1, sizeof(SCtgQNode)); if (NULL == node) { qError("calloc %d failed", (int32_t)sizeof(SCtgQNode)); CTG_RET(TSDB_CODE_OUT_OF_MEMORY); } - bool syncOp = operation->syncOp; - char* opName = gCtgCacheOperation[operation->opId].name; + bool syncOp = operation->syncOp; + char *opName = gCtgCacheOperation[operation->opId].name; if (operation->syncOp) { tsem_init(&operation->rspSem, 0, 0); } - + node->op = operation; CTG_LOCK(CTG_WRITE, &gCtgMgmt.queue.qlock); @@ -699,12 +640,11 @@ int32_t ctgEnqueue(SCatalog* pCtg, SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - -int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId) { - int32_t code = 0; +int32_t ctgDropDbCacheEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_DROP_DB_CACHE; - + SCtgDropDBMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDBMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDBMsg)); @@ -732,12 +672,12 @@ _return: CTG_RET(code); } -int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char *dbFName, bool syncOp) { - int32_t code = 0; +int32_t ctgDropDbVgroupEnqueue(SCatalog *pCtg, const char *dbFName, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_DROP_DB_VGROUP; op->syncOp = syncOp; - + SCtgDropDbVgroupMsg *msg = taosMemoryMalloc(sizeof(SCtgDropDbVgroupMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropDbVgroupMsg)); @@ -764,14 +704,13 @@ _return: CTG_RET(code); } - - -int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, bool syncOp) { - int32_t code = 0; +int32_t ctgDropStbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *stbName, uint64_t suid, + bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_DROP_STB_META; op->syncOp = syncOp; - + SCtgDropStbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropStbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropStbMetaMsg)); @@ -796,14 +735,12 @@ _return: CTG_RET(code); } - - -int32_t ctgDropTbMetaEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { - int32_t code = 0; +int32_t ctgDropTbMetaEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, const char *tbName, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_DROP_TB_META; op->syncOp = syncOp; - + SCtgDropTblMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTblMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTblMetaMsg)); @@ -827,12 +764,12 @@ _return: CTG_RET(code); } -int32_t ctgUpdateVgroupEnqueue(SCatalog* pCtg, const char *dbFName, int64_t dbId, SDBVgInfo* dbInfo, bool syncOp) { - int32_t code = 0; +int32_t ctgUpdateVgroupEnqueue(SCatalog *pCtg, const char *dbFName, int64_t dbId, SDBVgInfo *dbInfo, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_VGROUP; op->syncOp = syncOp; - + SCtgUpdateVgMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateVgMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateVgMsg)); @@ -864,12 +801,12 @@ _return: CTG_RET(code); } -int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool syncOp) { - int32_t code = 0; +int32_t ctgUpdateTbMetaEnqueue(SCatalog *pCtg, STableMetaOutput *output, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_TB_META; op->syncOp = syncOp; - + SCtgUpdateTbMetaMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbMetaMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbMetaMsg)); @@ -889,7 +826,7 @@ int32_t ctgUpdateTbMetaEnqueue(SCatalog* pCtg, STableMetaOutput *output, bool sy CTG_ERR_JRET(ctgEnqueue(pCtg, op)); return TSDB_CODE_SUCCESS; - + _return: if (output) { @@ -898,15 +835,15 @@ _return: } taosMemoryFreeClear(msg); - + CTG_RET(code); } -int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEpSet* pEpSet) { - int32_t code = 0; +int32_t ctgUpdateVgEpsetEnqueue(SCatalog *pCtg, char *dbFName, int32_t vgId, SEpSet *pEpSet) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_VG_EPSET; - + SCtgUpdateEpsetMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateEpsetMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateEpsetMsg)); @@ -923,22 +860,20 @@ int32_t ctgUpdateVgEpsetEnqueue(SCatalog* pCtg, char *dbFName, int32_t vgId, SEp CTG_ERR_JRET(ctgEnqueue(pCtg, op)); return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } - - -int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncOp) { - int32_t code = 0; +int32_t ctgUpdateUserEnqueue(SCatalog *pCtg, SGetUserAuthRsp *pAuth, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_USER; op->syncOp = syncOp; - + SCtgUpdateUserMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateUserMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateUserMsg)); @@ -951,23 +886,23 @@ int32_t ctgUpdateUserEnqueue(SCatalog* pCtg, SGetUserAuthRsp *pAuth, bool syncOp op->data = msg; CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - + return TSDB_CODE_SUCCESS; - + _return: tFreeSGetUserAuthRsp(pAuth); taosMemoryFreeClear(msg); - + CTG_RET(code); } -int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncOp) { - int32_t code = 0; +int32_t ctgUpdateTbIndexEnqueue(SCatalog *pCtg, STableIndex **pIndex, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_UPDATE_TB_INDEX; op->syncOp = syncOp; - + SCtgUpdateTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgUpdateTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgUpdateTbIndexMsg)); @@ -983,22 +918,22 @@ int32_t ctgUpdateTbIndexEnqueue(SCatalog* pCtg, STableIndex **pIndex, bool syncO *pIndex = NULL; return TSDB_CODE_SUCCESS; - + _return: taosArrayDestroyEx((*pIndex)->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(*pIndex); taosMemoryFreeClear(msg); - + CTG_RET(code); } -int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { - int32_t code = 0; +int32_t ctgDropTbIndexEnqueue(SCatalog *pCtg, SName *pName, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_DROP_TB_INDEX; op->syncOp = syncOp; - + SCtgDropTbIndexMsg *msg = taosMemoryMalloc(sizeof(SCtgDropTbIndexMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgDropTbIndexMsg)); @@ -1012,25 +947,24 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp) { op->data = msg; CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - + return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } - -int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool freeCtg, bool stopQueue, bool syncOp) { - int32_t code = 0; +int32_t ctgClearCacheEnqueue(SCatalog *pCtg, bool freeCtg, bool stopQueue, bool syncOp) { + int32_t code = 0; SCtgCacheOperation *op = taosMemoryCalloc(1, sizeof(SCtgCacheOperation)); op->opId = CTG_OP_CLEAR_CACHE; op->syncOp = syncOp; op->stopQueue = stopQueue; op->unLocked = true; - + SCtgClearCacheMsg *msg = taosMemoryMalloc(sizeof(SCtgClearCacheMsg)); if (NULL == msg) { ctgError("malloc %d failed", (int32_t)sizeof(SCtgClearCacheMsg)); @@ -1042,24 +976,23 @@ int32_t ctgClearCacheEnqueue(SCatalog* pCtg, bool freeCtg, bool stopQueue, bool op->data = msg; CTG_ERR_JRET(ctgEnqueue(pCtg, op)); - + return TSDB_CODE_SUCCESS; - + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { mgmt->slotRIdx = 0; mgmt->slotNum = rentSec / CTG_RENT_SLOT_SECOND; mgmt->type = type; size_t msgSize = sizeof(SCtgRentSlot) * mgmt->slotNum; - + mgmt->slots = taosMemoryCalloc(1, msgSize); if (NULL == mgmt->slots) { qError("calloc %d failed", (int32_t)msgSize); @@ -1067,34 +1000,34 @@ int32_t ctgMetaRentInit(SCtgRentMgmt *mgmt, uint32_t rentSec, int8_t type) { } qDebug("meta rent initialized, type:%d, slotNum:%d", type, mgmt->slotNum); - + return TSDB_CODE_SUCCESS; } - int32_t ctgMetaRentAdd(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size) { int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { slot->meta = taosArrayInit(CTG_DEFAULT_RENT_SLOT_SIZE, size); if (NULL == slot->meta) { - qError("taosArrayInit %d failed, id:0x%"PRIx64", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, mgmt->type); + qError("taosArrayInit %d failed, id:0x%" PRIx64 ", slot idx:%d, type:%d", CTG_DEFAULT_RENT_SLOT_SIZE, id, widx, + mgmt->type); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } } if (NULL == taosArrayPush(slot->meta, meta)) { - qError("taosArrayPush meta to rent failed, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("taosArrayPush meta to rent failed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } slot->needSort = true; - qDebug("add meta to rent, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("add meta to rent, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1102,20 +1035,22 @@ _return: CTG_RET(code); } -int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, __compar_fn_t searchCompare) { +int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t size, __compar_fn_t sortCompare, + __compar_fn_t searchCompare) { int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; + int32_t code = 0; CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qDebug("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("empty meta slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } if (slot->needSort) { - qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta slot before sorte, slot idx:%d, type:%d, size:%d", widx, mgmt->type, + (int32_t)taosArrayGetSize(slot->meta)); taosArraySort(slot->meta, sortCompare); slot->needSort = false; qDebug("meta slot sorted, slot idx:%d, type:%d, size:%d", widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); @@ -1123,20 +1058,22 @@ int32_t ctgMetaRentUpdate(SCtgRentMgmt *mgmt, void *meta, int64_t id, int32_t si void *orig = taosArraySearch(slot->meta, &id, searchCompare, TD_EQ); if (NULL == orig) { - qDebug("meta not found in slot, id:0x%"PRIx64", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, (int32_t)taosArrayGetSize(slot->meta)); + qDebug("meta not found in slot, id:0x%" PRIx64 ", slot idx:%d, type:%d, size:%d", id, widx, mgmt->type, + (int32_t)taosArrayGetSize(slot->meta)); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } memcpy(orig, meta, size); - qDebug("meta in rent updated, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent updated, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: CTG_UNLOCK(CTG_WRITE, &slot->lock); if (code) { - qDebug("meta in rent update failed, will try to add it, code:%x, id:0x%"PRIx64", slot idx:%d, type:%d", code, id, widx, mgmt->type); + qDebug("meta in rent update failed, will try to add it, code:%x, id:0x%" PRIx64 ", slot idx:%d, type:%d", code, id, + widx, mgmt->type); CTG_RET(ctgMetaRentAdd(mgmt, meta, id, size)); } @@ -1147,11 +1084,11 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int16_t widx = abs((int)(id % mgmt->slotNum)); SCtgRentSlot *slot = &mgmt->slots[widx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_WRITE, &slot->lock); if (NULL == slot->meta) { - qError("empty meta slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("empty meta slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } @@ -1163,13 +1100,13 @@ int32_t ctgMetaRentRemove(SCtgRentMgmt *mgmt, int64_t id, __compar_fn_t sortComp int32_t idx = taosArraySearchIdx(slot->meta, &id, searchCompare, TD_EQ); if (idx < 0) { - qError("meta not found in slot, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qError("meta not found in slot, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } taosArrayRemove(slot->meta, idx); - qDebug("meta in rent removed, id:0x%"PRIx64", slot idx:%d, type:%d", id, widx, mgmt->type); + qDebug("meta in rent removed, id:0x%" PRIx64 ", slot idx:%d, type:%d", id, widx, mgmt->type); _return: @@ -1178,7 +1115,6 @@ _return: CTG_RET(code); } - int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_t size) { int16_t ridx = atomic_add_fetch_16(&mgmt->slotRIdx, 1); if (ridx >= mgmt->slotNum) { @@ -1187,8 +1123,8 @@ int32_t ctgMetaRentGetImpl(SCtgRentMgmt *mgmt, void **res, uint32_t *num, int32_ } SCtgRentSlot *slot = &mgmt->slots[ridx]; - int32_t code = 0; - + int32_t code = 0; + CTG_LOCK(CTG_READ, &slot->lock); if (NULL == slot->meta) { qDebug("empty meta in slot:%d, type:%d", ridx, mgmt->type); @@ -1254,13 +1190,15 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { SCtgDBCache newDBCache = {0}; newDBCache.dbId = dbId; - newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + newDBCache.tbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), + true, HASH_ENTRY_LOCK); if (NULL == newDBCache.tbCache) { ctgError("taosHashInit %d metaCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), true, HASH_ENTRY_LOCK); + newDBCache.stbCache = taosHashInit(gCtgMgmt.cfg.maxTblCacheNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_UBIGINT), + true, HASH_ENTRY_LOCK); if (NULL == newDBCache.stbCache) { ctgError("taosHashInit %d stbCache failed", gCtgMgmt.cfg.maxTblCacheNum); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); @@ -1272,21 +1210,21 @@ int32_t ctgAddNewDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId) { ctgDebug("db already in cache, dbFName:%s", dbFName); goto _return; } - + ctgError("taosHashPut db to cache failed, dbFName:%s", dbFName); CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } CTG_CACHE_STAT_INC(numOfDb, 1); - + SDbVgVersion vgVersion = {.dbId = newDBCache.dbId, .vgVersion = -1}; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - ctgDebug("db added to cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId); + ctgDebug("db added to cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); CTG_ERR_RET(ctgMetaRentAdd(&pCtg->dbRent, &vgVersion, dbId, sizeof(SDbVgVersion))); - ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%"PRIx64, dbFName, vgVersion.vgVersion, dbId); + ctgDebug("db added to rent, dbFName:%s, vgVersion:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, dbId); return TSDB_CODE_SUCCESS; @@ -1297,30 +1235,29 @@ _return: CTG_RET(code); } - -void ctgRemoveStbRent(SCatalog* pCtg, SCtgDBCache *dbCache) { +void ctgRemoveStbRent(SCatalog *pCtg, SCtgDBCache *dbCache) { if (NULL == dbCache->stbCache) { return; } - + void *pIter = taosHashIterate(dbCache->stbCache, NULL); while (pIter) { uint64_t *suid = NULL; suid = taosHashGetKey(pIter, NULL); - if (TSDB_CODE_SUCCESS == ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { - ctgDebug("stb removed from rent, suid:0x%"PRIx64, *suid); + if (TSDB_CODE_SUCCESS == + ctgMetaRentRemove(&pCtg->stbRent, *suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)) { + ctgDebug("stb removed from rent, suid:0x%" PRIx64, *suid); } - + pIter = taosHashIterate(dbCache->stbCache, pIter); } } - -int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* dbFName) { +int32_t ctgRemoveDBFromCache(SCatalog *pCtg, SCtgDBCache *dbCache, const char *dbFName) { uint64_t dbId = dbCache->dbId; - - ctgInfo("start to remove db from cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbCache->dbId); + + ctgInfo("start to remove db from cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbCache->dbId); CTG_LOCK(CTG_WRITE, &dbCache->dbLock); @@ -1331,7 +1268,7 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d CTG_UNLOCK(CTG_WRITE, &dbCache->dbLock); CTG_ERR_RET(ctgMetaRentRemove(&pCtg->dbRent, dbId, ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); - ctgDebug("db removed from rent, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId); + ctgDebug("db removed from rent, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); if (taosHashRemove(pCtg->dbCache, dbFName, strlen(dbFName))) { ctgInfo("taosHashRemove from dbCache failed, may be removed, dbFName:%s", dbFName); @@ -1339,19 +1276,18 @@ int32_t ctgRemoveDBFromCache(SCatalog* pCtg, SCtgDBCache *dbCache, const char* d } CTG_CACHE_STAT_DEC(numOfDb, 1); - ctgInfo("db removed from cache, dbFName:%s, dbId:0x%"PRIx64, dbFName, dbId); - + ctgInfo("db removed from cache, dbFName:%s, dbId:0x%" PRIx64, dbFName, dbId); + return TSDB_CODE_SUCCESS; } - -int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { - int32_t code = 0; +int32_t ctgGetAddDBCache(SCatalog *pCtg, const char *dbFName, uint64_t dbId, SCtgDBCache **pCache) { + int32_t code = 0; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, dbFName, &dbCache); - + if (dbCache) { - // TODO OPEN IT + // TODO OPEN IT #if 0 if (dbCache->dbId == dbId) { *pCache = dbCache; @@ -1368,7 +1304,7 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt *pCache = dbCache; return TSDB_CODE_SUCCESS; } - + if (dbCache->dbId == dbId) { *pCache = dbCache; return TSDB_CODE_SUCCESS; @@ -1376,7 +1312,7 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt #endif CTG_ERR_RET(ctgRemoveDBFromCache(pCtg, dbCache, dbFName)); } - + CTG_ERR_RET(ctgAddNewDBCache(pCtg, dbFName, dbId)); ctgGetDBCache(pCtg, dbFName, &dbCache); @@ -1386,7 +1322,8 @@ int32_t ctgGetAddDBCache(SCatalog* pCtg, const char *dbFName, uint64_t dbId, SCt return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uint64_t dbId, uint64_t suid, SCtgTbCache* pCache) { +int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char *dbFName, char *tbName, uint64_t dbId, uint64_t suid, + SCtgTbCache *pCache) { SSTableVersion metaRent = {.dbId = dbId, .suid = suid}; if (pCache->pMeta) { metaRent.sversion = pCache->pMeta->sversion; @@ -1396,49 +1333,51 @@ int32_t ctgUpdateRentStbVersion(SCatalog *pCtg, char* dbFName, char* tbName, uin if (pCache->pIndex) { metaRent.smaVer = pCache->pIndex->version; } - + strcpy(metaRent.dbFName, dbFName); strcpy(metaRent.stbName, tbName); - - CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - ctgDebug("db %s,0x%" PRIx64 " stb %s,0x%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", - dbFName, dbId, tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); + CTG_ERR_RET(ctgMetaRentUpdate(&pCtg->stbRent, &metaRent, metaRent.suid, sizeof(SSTableVersion), + ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - return TSDB_CODE_SUCCESS; -} + ctgDebug("db %s,0x%" PRIx64 " stb %s,0x%" PRIx64 " sver %d tver %d smaVer %d updated to stbRent", dbFName, dbId, + tbName, suid, metaRent.sversion, metaRent.tversion, metaRent.smaVer); + return TSDB_CODE_SUCCESS; +} -int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, STableMeta *meta, int32_t metaSize) { +int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, uint64_t dbId, char *tbName, + STableMeta *meta, int32_t metaSize) { if (NULL == dbCache->tbCache || NULL == dbCache->stbCache) { taosMemoryFree(meta); - ctgError("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); + ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - bool isStb = meta->tableType == TSDB_SUPER_TABLE; - SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); - STableMeta *orig = (pCache ? pCache->pMeta : NULL); - int8_t origType = 0; - uint64_t origSuid = 0; - + bool isStb = meta->tableType == TSDB_SUPER_TABLE; + SCtgTbCache *pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + STableMeta *orig = (pCache ? pCache->pMeta : NULL); + int8_t origType = 0; + uint64_t origSuid = 0; + if (orig) { origType = orig->tableType; - if (origType == meta->tableType && orig->uid == meta->uid && (origType == TSDB_CHILD_TABLE || (orig->sversion >= meta->sversion && orig->tversion >= meta->tversion))) { + if (origType == meta->tableType && orig->uid == meta->uid && + (origType == TSDB_CHILD_TABLE || (orig->sversion >= meta->sversion && orig->tversion >= meta->tversion))) { taosMemoryFree(meta); ctgDebug("ignore table %s meta update", tbName); return TSDB_CODE_SUCCESS; } - + if (origType == TSDB_SUPER_TABLE) { if (taosHashRemove(dbCache->stbCache, &orig->suid, sizeof(orig->suid))) { - ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%"PRIx64, dbFName, tbName, orig->suid); + ctgError("stb not exist in stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); } else { CTG_CACHE_STAT_DEC(numOfStb, 1); - ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%"PRIx64, dbFName, tbName, orig->suid); + ctgDebug("stb removed from stbCache, dbFName:%s, stb:%s, suid:0x%" PRIx64, dbFName, tbName, orig->suid); } - + origSuid = orig->suid; } } @@ -1451,7 +1390,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam ctgError("taosHashPut new tbCache failed, dbFName:%s, tbName:%s, tbType:%d", dbFName, tbName, meta->tableType); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); } else { taosMemoryFree(pCache->pMeta); @@ -1469,35 +1408,37 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam return TSDB_CODE_SUCCESS; } - if (origSuid != meta->suid && taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { - ctgError("taosHashPut to stable cache failed, suid:0x%"PRIx64, meta->suid); + if (origSuid != meta->suid && + taosHashPut(dbCache->stbCache, &meta->suid, sizeof(meta->suid), tbName, strlen(tbName) + 1) != 0) { + ctgError("taosHashPut to stable cache failed, suid:0x%" PRIx64, meta->suid); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } CTG_CACHE_STAT_INC(numOfStb, 1); - ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, meta->tableType); + ctgDebug("stb 0x%" PRIx64 " updated to cache, dbFName:%s, tbName:%s, tbType:%d", meta->suid, dbFName, tbName, + meta->tableType); CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbId, meta->suid, pCache)); - + return TSDB_CODE_SUCCESS; } -int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFName, char *tbName, STableIndex **index) { +int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFName, char *tbName, STableIndex **index) { if (NULL == dbCache->tbCache) { ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); - ctgError("db is dropping, dbId:0x%"PRIx64, dbCache->dbId); + ctgError("db is dropping, dbId:0x%" PRIx64, dbCache->dbId); CTG_ERR_RET(TSDB_CODE_CTG_DB_DROPPED); } - STableIndex* pIndex = *index; - uint64_t suid = pIndex->suid; - SCtgTbCache* pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); + STableIndex *pIndex = *index; + uint64_t suid = pIndex->suid; + SCtgTbCache *pCache = taosHashGet(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { SCtgTbCache cache = {0}; cache.pIndex = pIndex; - + if (taosHashPut(dbCache->tbCache, tbName, strlen(tbName), &cache, sizeof(cache)) != 0) { ctgFreeSTableIndex(*index); taosMemoryFreeClear(*index); @@ -1506,12 +1447,13 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa } *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, + (int32_t)taosArrayGetSize(pIndex->pIndex)); if (suid) { CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, pIndex->suid, &cache)); } - + return TSDB_CODE_SUCCESS; } @@ -1526,24 +1468,25 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char* dbFNa pCache->pIndex = pIndex; *index = NULL; - ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); + ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, + (int32_t)taosArrayGetSize(pIndex->pIndex)); if (suid) { CTG_ERR_RET(ctgUpdateRentStbVersion(pCtg, dbFName, tbName, dbCache->dbId, suid, pCache)); } - + return TSDB_CODE_SUCCESS; } -int32_t ctgUpdateTbMetaToCache(SCatalog* pCtg, STableMetaOutput* pOut, bool syncReq) { - STableMetaOutput* pOutput = NULL; - int32_t code = 0; - +int32_t ctgUpdateTbMetaToCache(SCatalog *pCtg, STableMetaOutput *pOut, bool syncReq) { + STableMetaOutput *pOutput = NULL; + int32_t code = 0; + CTG_ERR_RET(ctgCloneMetaOutput(pOut, &pOutput)); CTG_ERR_JRET(ctgUpdateTbMetaEnqueue(pCtg, pOutput, syncReq)); return TSDB_CODE_SUCCESS; - + _return: ctgFreeSTableMetaOutput(pOutput); @@ -1551,11 +1494,11 @@ _return: } void ctgClearAllInstance(void) { - SCatalog* pCtg = NULL; + SCatalog *pCtg = NULL; - void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { - pCtg = *(SCatalog**)pIter; + pCtg = *(SCatalog **)pIter; if (pCtg) { ctgClearHandle(pCtg); @@ -1566,11 +1509,11 @@ void ctgClearAllInstance(void) { } void ctgFreeAllInstance(void) { - SCatalog* pCtg = NULL; + SCatalog *pCtg = NULL; - void* pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); + void *pIter = taosHashIterate(gCtgMgmt.pCluster, NULL); while (pIter) { - pCtg = *(SCatalog**)pIter; + pCtg = *(SCatalog **)pIter; if (pCtg) { ctgFreeHandle(pCtg); @@ -1582,51 +1525,51 @@ void ctgFreeAllInstance(void) { taosHashClear(gCtgMgmt.pCluster); } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; - SDBVgInfo* dbInfo = msg->dbInfo; - char* dbFName = msg->dbFName; - SCatalog* pCtg = msg->pCtg; - + SDBVgInfo *dbInfo = msg->dbInfo; + char *dbFName = msg->dbFName; + SCatalog *pCtg = msg->pCtg; + if (NULL == dbInfo->vgHash) { goto _return; } - + if (dbInfo->vgVersion < 0 || taosHashGetSize(dbInfo->vgHash) <= 0) { - ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", - dbFName, dbInfo->vgHash, dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); + ctgError("invalid db vgInfo, dbFName:%s, vgHash:%p, vgVersion:%d, vgHashSize:%d", dbFName, dbInfo->vgHash, + dbInfo->vgVersion, taosHashGetSize(dbInfo->vgHash)); CTG_ERR_JRET(TSDB_CODE_APP_ERROR); } - bool newAdded = false; + bool newAdded = false; SDbVgVersion vgVersion = {.dbId = msg->dbId, .vgVersion = dbInfo->vgVersion, .numOfTable = dbInfo->numOfTable}; SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetAddDBCache(msg->pCtg, dbFName, msg->dbId, &dbCache)); if (NULL == dbCache) { - ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%"PRIx64, dbFName, msg->dbId); + ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, dbFName, msg->dbId); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); } SCtgVgCache *vgCache = &dbCache->vgCache; CTG_ERR_JRET(ctgWLockVgInfo(msg->pCtg, dbCache)); - + if (vgCache->vgInfo) { SDBVgInfo *vgInfo = vgCache->vgInfo; - + if (dbInfo->vgVersion < vgInfo->vgVersion) { ctgDebug("db vgVer is old, dbFName:%s, vgVer:%d, curVer:%d", dbFName, dbInfo->vgVersion, vgInfo->vgVersion); ctgWUnlockVgInfo(dbCache); - + goto _return; } if (dbInfo->vgVersion == vgInfo->vgVersion && dbInfo->numOfTable == vgInfo->numOfTable) { - ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, dbInfo->numOfTable); + ctgDebug("no new db vgVer or numOfTable, dbFName:%s, vgVer:%d, numOfTable:%d", dbFName, dbInfo->vgVersion, + dbInfo->numOfTable); ctgWUnlockVgInfo(dbCache); - + goto _return; } @@ -1636,61 +1579,63 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { vgCache->vgInfo = dbInfo; msg->dbInfo = NULL; - ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%"PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); + ctgDebug("db vgInfo updated, dbFName:%s, vgVer:%d, dbId:0x%" PRIx64, dbFName, vgVersion.vgVersion, vgVersion.dbId); ctgWUnlockVgInfo(dbCache); dbCache = NULL; strncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_RET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); _return: ctgFreeVgInfo(msg->dbInfo); taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgOpDropDbCache(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropDBMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { goto _return; } - + if (dbCache->dbId != msg->dbId) { - ctgInfo("dbId already updated, dbFName:%s, dbId:0x%"PRIx64 ", targetId:0x%"PRIx64, msg->dbFName, dbCache->dbId, msg->dbId); + ctgInfo("dbId already updated, dbFName:%s, dbId:0x%" PRIx64 ", targetId:0x%" PRIx64, msg->dbFName, dbCache->dbId, + msg->dbId); goto _return; } - + CTG_ERR_JRET(ctgRemoveDBFromCache(pCtg, dbCache, msg->dbFName)); _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropDbVgroupMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(msg->pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { goto _return; } - - CTG_ERR_RET(ctgWLockVgInfo(pCtg, dbCache)); - + + CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); + ctgFreeVgInfo(dbCache->vgCache.vgInfo); dbCache->vgCache.vgInfo = NULL; @@ -1701,17 +1646,16 @@ int32_t ctgOpDropDbVgroup(SCtgCacheOperation *operation) { _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateTbMetaMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; - STableMetaOutput* pMeta = msg->pMeta; - SCtgDBCache *dbCache = NULL; + SCatalog *pCtg = msg->pCtg; + STableMetaOutput *pMeta = msg->pMeta; + SCtgDBCache *dbCache = NULL; if ((!CTG_IS_META_CTABLE(pMeta->metaType)) && NULL == pMeta->tbMeta) { ctgError("no valid tbmeta got from meta rsp, dbFName:%s, tbName:%s", pMeta->dbFName, pMeta->tbName); @@ -1721,8 +1665,8 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { if (CTG_IS_META_BOTH(pMeta->metaType) && TSDB_SUPER_TABLE != pMeta->tbMeta->tableType) { ctgError("table type error, expected:%d, actual:%d", TSDB_SUPER_TABLE, pMeta->tbMeta->tableType); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); - } - + } + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pMeta->dbFName, pMeta->dbId, &dbCache)); if (NULL == dbCache) { ctgInfo("conflict db update, ignore this update, dbFName:%s, dbId:0x%" PRIx64, pMeta->dbFName, pMeta->dbId); @@ -1731,17 +1675,19 @@ int32_t ctgOpUpdateTbMeta(SCtgCacheOperation *operation) { if (CTG_IS_META_TABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { int32_t metaSize = CTG_META_SIZE(pMeta->tbMeta); - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); + CTG_ERR_JRET( + ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->tbName, pMeta->tbMeta, metaSize)); pMeta->tbMeta = NULL; } if (CTG_IS_META_CTABLE(pMeta->metaType) || CTG_IS_META_BOTH(pMeta->metaType)) { - SCTableMeta* ctbMeta = taosMemoryMalloc(sizeof(SCTableMeta)); + SCTableMeta *ctbMeta = taosMemoryMalloc(sizeof(SCTableMeta)); if (NULL == ctbMeta) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } memcpy(ctbMeta, &pMeta->ctbMeta, sizeof(SCTableMeta)); - CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, (STableMeta *)ctbMeta, sizeof(SCTableMeta))); + CTG_ERR_JRET(ctgWriteTbMetaToCache(pCtg, dbCache, pMeta->dbFName, pMeta->dbId, pMeta->ctbName, + (STableMeta *)ctbMeta, sizeof(SCTableMeta))); } _return: @@ -1750,37 +1696,37 @@ _return: taosMemoryFreeClear(pMeta->tbMeta); taosMemoryFreeClear(pMeta); } - + taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropStbMetaMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); if (NULL == dbCache) { - return TSDB_CODE_SUCCESS; + goto _return; } if (msg->dbId && (dbCache->dbId != msg->dbId)) { - ctgDebug("dbId already modified, dbFName:%s, current:0x%"PRIx64", dbId:0x%"PRIx64", stb:%s, suid:0x%"PRIx64, + ctgDebug("dbId already modified, dbFName:%s, current:0x%" PRIx64 ", dbId:0x%" PRIx64 ", stb:%s, suid:0x%" PRIx64, msg->dbFName, dbCache->dbId, msg->dbId, msg->stbName, msg->suid); - return TSDB_CODE_SUCCESS; + goto _return; } - + if (taosHashRemove(dbCache->stbCache, &msg->suid, sizeof(msg->suid))) { - ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + ctgDebug("stb not exist in stbCache, may be removed, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, + msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(numOfStb, 1); } - SCtgTbCache* pTbCache = taosHashGet(dbCache->tbCache, msg->stbName, strlen(msg->stbName)); + SCtgTbCache *pTbCache = taosHashGet(dbCache->tbCache, msg->stbName, strlen(msg->stbName)); if (NULL == pTbCache) { ctgDebug("stb %s already not in cache", msg->stbName); goto _return; @@ -1790,29 +1736,29 @@ int32_t ctgOpDropStbMeta(SCtgCacheOperation *operation) { ctgFreeTbCacheImpl(pTbCache); CTG_UNLOCK(CTG_WRITE, &pTbCache->metaLock); - if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { - ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + if (taosHashRemove(dbCache->tbCache, msg->stbName, strlen(msg->stbName))) { + ctgError("stb not exist in cache, dbFName:%s, stb:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); } else { CTG_CACHE_STAT_DEC(numOfTbl, 1); } - - ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); + + ctgInfo("stb removed from cache, dbFName:%s, stbName:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); CTG_ERR_JRET(ctgMetaRentRemove(&msg->pCtg->stbRent, msg->suid, ctgStbVersionSortCompare, ctgStbVersionSearchCompare)); - - ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:0x%"PRIx64, msg->dbFName, msg->stbName, msg->suid); - + + ctgDebug("stb removed from rent, dbFName:%s, stbName:%s, suid:0x%" PRIx64, msg->dbFName, msg->stbName, msg->suid); + _return: taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropTblMetaMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; SCtgDBCache *dbCache = NULL; ctgGetDBCache(pCtg, msg->dbFName, &dbCache); @@ -1821,11 +1767,12 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { } if (dbCache->dbId != msg->dbId) { - ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%"PRIx64", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, msg->dbFName, msg->tbName); + ctgDebug("dbId 0x%" PRIx64 " not match with curId 0x%" PRIx64 ", dbFName:%s, tbName:%s", msg->dbId, dbCache->dbId, + msg->dbFName, msg->tbName); goto _return; } - SCtgTbCache* pTbCache = taosHashGet(dbCache->tbCache, msg->tbName, strlen(msg->tbName)); + SCtgTbCache *pTbCache = taosHashGet(dbCache->tbCache, msg->tbName, strlen(msg->tbName)); if (NULL == pTbCache) { ctgDebug("tb %s already not in cache", msg->tbName); goto _return; @@ -1834,7 +1781,7 @@ int32_t ctgOpDropTbMeta(SCtgCacheOperation *operation) { CTG_LOCK(CTG_WRITE, &pTbCache->metaLock); ctgFreeTbCacheImpl(pTbCache); CTG_UNLOCK(CTG_WRITE, &pTbCache->metaLock); - + if (taosHashRemove(dbCache->tbCache, msg->tbName, strlen(msg->tbName))) { ctgError("tb %s not exist in cache, dbFName:%s", msg->tbName, msg->dbFName); CTG_ERR_JRET(TSDB_CODE_CTG_INTERNAL_ERROR); @@ -1852,10 +1799,10 @@ _return: } int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateUserMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; - + SCatalog *pCtg = msg->pCtg; + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user)); if (NULL == pUser) { SCtgUserAuth userAuth = {0}; @@ -1899,17 +1846,17 @@ _return: taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); - + taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateEpsetMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; - + SCatalog *pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { @@ -1919,23 +1866,23 @@ int32_t ctgOpUpdateEpset(SCtgCacheOperation *operation) { CTG_ERR_JRET(ctgWLockVgInfo(pCtg, dbCache)); - SDBVgInfo *vgInfo = dbCache->vgCache.vgInfo; + SDBVgInfo *vgInfo = dbCache->vgCache.vgInfo; if (NULL == vgInfo) { ctgDebug("vgroup in db %s not cached, ignore epset update", msg->dbFName); goto _return; } - - SVgroupInfo* pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); + + SVgroupInfo *pInfo = taosHashGet(vgInfo->vgHash, &msg->vgId, sizeof(msg->vgId)); if (NULL == pInfo) { ctgDebug("no vgroup %d in db %s, ignore epset update", msg->vgId, msg->dbFName); goto _return; } - SEp* pOrigEp = &pInfo->epSet.eps[pInfo->epSet.inUse]; - SEp* pNewEp = &msg->epSet.eps[msg->epSet.inUse]; - ctgDebug("vgroup %d epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d, dbFName:%s in ctg", - pInfo->vgId, pInfo->epSet.inUse, pInfo->epSet.numOfEps, pOrigEp->fqdn, pOrigEp->port, - msg->epSet.inUse, msg->epSet.numOfEps, pNewEp->fqdn, pNewEp->port, msg->dbFName); + SEp *pOrigEp = &pInfo->epSet.eps[pInfo->epSet.inUse]; + SEp *pNewEp = &msg->epSet.eps[msg->epSet.inUse]; + ctgDebug("vgroup %d epset updated from %d/%d=>%s:%d to %d/%d=>%s:%d, dbFName:%s in ctg", pInfo->vgId, + pInfo->epSet.inUse, pInfo->epSet.numOfEps, pOrigEp->fqdn, pOrigEp->port, msg->epSet.inUse, + msg->epSet.numOfEps, pNewEp->fqdn, pNewEp->port, msg->dbFName); pInfo->epSet = msg->epSet; @@ -1946,17 +1893,17 @@ _return: } taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgUpdateTbIndexMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; - STableIndex* pIndex = msg->pIndex; - SCtgDBCache *dbCache = NULL; - + SCatalog *pCtg = msg->pCtg; + STableIndex *pIndex = msg->pIndex; + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgGetAddDBCache(pCtg, pIndex->dbFName, 0, &dbCache)); CTG_ERR_JRET(ctgWriteTbIndexToCache(pCtg, dbCache, pIndex->dbFName, pIndex->tbName, &pIndex)); @@ -1967,24 +1914,24 @@ _return: taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pIndex); } - + taosMemoryFreeClear(msg); - + CTG_RET(code); } int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgDropTbIndexMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; - SCtgDBCache *dbCache = NULL; - + SCatalog *pCtg = msg->pCtg; + SCtgDBCache *dbCache = NULL; + CTG_ERR_JRET(ctgGetDBCache(pCtg, msg->dbFName, &dbCache)); if (NULL == dbCache) { return TSDB_CODE_SUCCESS; } - STableIndex* pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); + STableIndex *pIndex = taosMemoryCalloc(1, sizeof(STableIndex)); if (NULL == pIndex) { CTG_ERR_JRET(TSDB_CODE_OUT_OF_MEMORY); } @@ -2000,17 +1947,16 @@ _return: taosArrayDestroyEx(pIndex->pIndex, tFreeSTableIndexInfo); taosMemoryFreeClear(pIndex); } - + taosMemoryFreeClear(msg); - + CTG_RET(code); } - int32_t ctgOpClearCache(SCtgCacheOperation *operation) { - int32_t code = 0; + int32_t code = 0; SCtgClearCacheMsg *msg = operation->data; - SCatalog* pCtg = msg->pCtg; + SCatalog *pCtg = msg->pCtg; CTG_LOCK(CTG_WRITE, &gCtgMgmt.lock); @@ -2020,7 +1966,7 @@ int32_t ctgOpClearCache(SCtgCacheOperation *operation) { } else { ctgClearHandle(pCtg); } - + goto _return; } @@ -2033,17 +1979,17 @@ int32_t ctgOpClearCache(SCtgCacheOperation *operation) { _return: CTG_UNLOCK(CTG_WRITE, &gCtgMgmt.lock); - + taosMemoryFreeClear(msg); - + CTG_RET(code); } void ctgCleanupCacheQueue(void) { - SCtgQNode *node = NULL; - SCtgQNode *nodeNext = NULL; + SCtgQNode *node = NULL; + SCtgQNode *nodeNext = NULL; SCtgCacheOperation *op = NULL; - bool stopQueue = false; + bool stopQueue = false; while (true) { node = gCtgMgmt.queue.head->next; @@ -2055,12 +2001,12 @@ void ctgCleanupCacheQueue(void) { ctgDebug("process [%s] operation", gCtgCacheOperation[op->opId].name); (*gCtgCacheOperation[op->opId].func)(op); stopQueue = true; - CTG_RT_STAT_INC(numOfOpDequeue, 1); + CTG_RT_STAT_INC(numOfOpDequeue, 1); } else { taosMemoryFree(op->data); - CTG_RT_STAT_INC(numOfOpAbort, 1); + CTG_RT_STAT_INC(numOfOpAbort, 1); } - + if (op->syncOp) { tsem_post(&op->rspSem); } else { @@ -2070,7 +2016,7 @@ void ctgCleanupCacheQueue(void) { nodeNext = node->next; taosMemoryFree(node); - + node = nodeNext; } @@ -2085,7 +2031,7 @@ void ctgCleanupCacheQueue(void) { gCtgMgmt.queue.tail = NULL; } -void* ctgUpdateThreadFunc(void* param) { +void *ctgUpdateThreadFunc(void *param) { setThreadName("catalog"); qInfo("catalog update thread started"); @@ -2094,8 +2040,8 @@ void* ctgUpdateThreadFunc(void* param) { if (tsem_wait(&gCtgMgmt.queue.reqSem)) { qError("ctg tsem_wait failed, error:%s", tstrerror(TAOS_SYSTEM_ERROR(errno))); } - - if (atomic_load_8((int8_t*)&gCtgMgmt.exit)) { + + if (atomic_load_8((int8_t *)&gCtgMgmt.exit)) { ctgCleanupCacheQueue(); break; } @@ -2105,7 +2051,7 @@ void* ctgUpdateThreadFunc(void* param) { SCatalog *pCtg = ((SCtgUpdateMsgHeader *)operation->data)->pCtg; ctgDebug("process [%s] operation", gCtgCacheOperation[operation->opId].name); - + (*gCtgCacheOperation[operation->opId].func)(operation); if (operation->syncOp) { @@ -2114,18 +2060,17 @@ void* ctgUpdateThreadFunc(void* param) { taosMemoryFreeClear(operation); } - CTG_RT_STAT_INC(numOfOpDequeue, 1); + CTG_RT_STAT_INC(numOfOpDequeue, 1); ctgdShowCacheInfo(); ctgdShowClusterCache(pCtg); } qInfo("catalog update thread stopped"); - + return NULL; } - int32_t ctgStartUpdateThread() { TdThreadAttr thAttr; taosThreadAttrInit(&thAttr); @@ -2135,13 +2080,12 @@ int32_t ctgStartUpdateThread() { terrno = TAOS_SYSTEM_ERROR(errno); CTG_ERR_RET(terrno); } - + taosThreadAttrDestroy(&thAttr); return TSDB_CODE_SUCCESS; } - -int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { +int32_t ctgGetTbMetaFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { if (IS_SYS_DBNAME(ctx->pName->dbname)) { CTG_FLAG_SET_SYS_DB(ctx->flag); } @@ -2221,14 +2165,15 @@ int32_t ctgGetTbMetaBFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } #endif -int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx* ctx, int32_t dbIdx, int32_t *fetchIdx, int32_t baseResIdx, SArray* pList) { - int32_t tbNum = taosArrayGetSize(pList); - SName* pName = taosArrayGet(pList, 0); - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - int32_t flag = CTG_FLAG_UNKNOWN_STB; - uint64_t lastSuid = 0; - STableMeta* lastTableMeta = NULL; - +int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMetasCtx *ctx, int32_t dbIdx, + int32_t *fetchIdx, int32_t baseResIdx, SArray *pList) { + int32_t tbNum = taosArrayGetSize(pList); + SName *pName = taosArrayGet(pList, 0); + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + int32_t flag = CTG_FLAG_UNKNOWN_STB; + uint64_t lastSuid = 0; + STableMeta *lastTableMeta = NULL; + if (IS_SYS_DBNAME(pName->dbname)) { CTG_FLAG_SET_SYS_DB(flag); strcpy(dbFName, pName->dbname); @@ -2237,9 +2182,9 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } SCtgDBCache *dbCache = NULL; - SCtgTbCache* pCache = NULL; + SCtgTbCache *pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); - + if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); for (int32_t i = 0; i < tbNum; ++i) { @@ -2251,14 +2196,14 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } for (int32_t i = 0; i < tbNum; ++i) { - SName* pName = taosArrayGet(pList, i); + SName *pName = taosArrayGet(pList, i); pCache = taosHashAcquire(dbCache->tbCache, pName->tname, strlen(pName->tname)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", pName->tname, dbFName); ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); - + continue; } @@ -2267,11 +2212,11 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgDebug("tb %s meta not in cache, dbFName:%s", pName->tname, dbFName); ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); - + continue; } - STableMeta* tbMeta = pCache->pMeta; + STableMeta *tbMeta = pCache->pMeta; SCtgTbMetaCtx nctx = {0}; nctx.flag = flag; @@ -2280,8 +2225,8 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe nctx.tbInfo.suid = tbMeta->suid; nctx.tbInfo.tbType = tbMeta->tableType; - SMetaRes res = {0}; - STableMeta* pTableMeta = NULL; + SMetaRes res = {0}; + STableMeta *pTableMeta = NULL; if (tbMeta->tableType != TSDB_CHILD_TABLE) { int32_t metaSize = CTG_META_SIZE(tbMeta); pTableMeta = taosMemoryCalloc(1, metaSize); @@ -2289,20 +2234,20 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + memcpy(pTableMeta, tbMeta, metaSize); - + CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - + taosHashRelease(dbCache->tbCache, pCache); + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); - + res.pRes = pTableMeta; taosArrayPush(ctx->pResList, &res); continue; } - + // PROCESS FOR CHILD TABLE if (lastSuid && tbMeta->suid == lastSuid && lastTableMeta) { @@ -2310,32 +2255,32 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe memcpy(pTableMeta, tbMeta, sizeof(SCTableMeta)); CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - + taosHashRelease(dbCache->tbCache, pCache); + ctgDebug("Got tb %s meta from cache, type:%d, dbFName:%s", pName->tname, tbMeta->tableType, dbFName); - + res.pRes = pTableMeta; taosArrayPush(ctx->pResList, &res); - + continue; } - + int32_t metaSize = sizeof(SCTableMeta); pTableMeta = taosMemoryCalloc(1, metaSize); if (NULL == pTableMeta) { - ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); + ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + memcpy(pTableMeta, tbMeta, metaSize); - + CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - - ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", - pName->tname, nctx.tbInfo.tbType, dbFName); - - char* stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); + taosHashRelease(dbCache->tbCache, pCache); + + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", pName->tname, + nctx.tbInfo.tbType, dbFName); + + char *stName = taosHashAcquire(dbCache->stbCache, &pTableMeta->suid, sizeof(pTableMeta->suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", pTableMeta->suid, dbFName); ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); @@ -2349,11 +2294,11 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe if (NULL == pCache) { ctgDebug("stb 0x%" PRIx64 " name %s not in cache, dbFName:%s", pTableMeta->suid, stName, dbFName); taosHashRelease(dbCache->stbCache, stName); - + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); - taosMemoryFreeClear(pTableMeta); + taosMemoryFreeClear(pTableMeta); continue; } @@ -2363,8 +2308,8 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe if (NULL == pCache->pMeta) { ctgDebug("stb 0x%" PRIx64 " meta not in cache, dbFName:%s", pTableMeta->suid, dbFName); CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - + taosHashRelease(dbCache->tbCache, pCache); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); @@ -2372,14 +2317,15 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe continue; } - - STableMeta* stbMeta = pCache->pMeta; - if (stbMeta->suid != nctx.tbInfo.suid) { + + STableMeta *stbMeta = pCache->pMeta; + if (stbMeta->suid != nctx.tbInfo.suid) { CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - - ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%"PRIx64 , stbMeta->suid, nctx.tbInfo.suid); - + taosHashRelease(dbCache->tbCache, pCache); + + ctgError("stb suid 0x%" PRIx64 " in stbCache mis-match, expected suid 0x%" PRIx64, stbMeta->suid, + nctx.tbInfo.suid); + ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArraySetSize(ctx->pResList, taosArrayGetSize(ctx->pResList) + 1); @@ -2387,19 +2333,19 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe continue; } - + metaSize = CTG_META_SIZE(stbMeta); pTableMeta = taosMemoryRealloc(pTableMeta, metaSize); - if (NULL == pTableMeta) { + if (NULL == pTableMeta) { ctgReleaseTbMetaToCache(pCtg, dbCache, pCache); CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + memcpy(&pTableMeta->sversion, &stbMeta->sversion, metaSize - sizeof(SCTableMeta)); - + CTG_UNLOCK(CTG_READ, &pCache->metaLock); - taosHashRelease(dbCache->tbCache, pCache); - + taosHashRelease(dbCache->tbCache, pCache); + res.pRes = pTableMeta; taosArrayPush(ctx->pResList, &res); @@ -2408,14 +2354,13 @@ int32_t ctgGetTbMetasFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMe } ctgReleaseDBCache(pCtg, dbCache); - + return TSDB_CODE_SUCCESS; } - -int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { +int32_t ctgRemoveTbMetaFromCache(SCatalog *pCtg, SName *pTableName, bool syncReq) { int32_t code = 0; - STableMeta* tblMeta = NULL; + STableMeta *tblMeta = NULL; SCtgTbMetaCtx tbCtx = {0}; tbCtx.flag = CTG_FLAG_UNKNOWN_STB; tbCtx.pName = pTableName; @@ -2449,7 +2394,7 @@ int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVg CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); } - SCtgDBCache* dbCache = NULL; + SCtgDBCache *dbCache = NULL; int32_t code = 0; char dbFName[TSDB_DB_FNAME_LEN] = {0}; tNameGetFullDbName(pTableName, dbFName); @@ -2476,5 +2421,3 @@ _return: CTG_RET(code); } - - diff --git a/source/libs/catalog/src/ctgDbg.c b/source/libs/catalog/src/ctgDbg.c index bd3402dc394186b03d116c2c2ebe5e83838bdddb..d21524230736c47fb3c81de0c00e1402f09a585e 100644 --- a/source/libs/catalog/src/ctgDbg.c +++ b/source/libs/catalog/src/ctgDbg.c @@ -367,18 +367,22 @@ void ctgdShowDBCache(SCatalog* pCtg, SHashObj *dbHash) { int32_t stbNum = dbCache->stbCache ? taosHashGetSize(dbCache->stbCache) : 0; int32_t vgVersion = CTG_DEFAULT_INVALID_VERSION; int32_t hashMethod = -1; + int16_t hashPrefix = 0; + int16_t hashSuffix = 0; int32_t vgNum = 0; if (dbCache->vgCache.vgInfo) { vgVersion = dbCache->vgCache.vgInfo->vgVersion; hashMethod = dbCache->vgCache.vgInfo->hashMethod; + hashPrefix = dbCache->vgCache.vgInfo->hashPrefix; + hashSuffix = dbCache->vgCache.vgInfo->hashSuffix; if (dbCache->vgCache.vgInfo->vgHash) { vgNum = taosHashGetSize(dbCache->vgCache.vgInfo->vgHash); } } - ctgDebug("[%d] db [%.*s][0x%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, vgNum:%d", - i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, vgNum); + ctgDebug("[%d] db [%.*s][0x%"PRIx64"] %s: metaNum:%d, stbNum:%d, vgVersion:%d, hashMethod:%d, prefix:%d, suffix:%d, vgNum:%d", + i, (int32_t)len, dbFName, dbCache->dbId, dbCache->deleted?"deleted":"", metaNum, stbNum, vgVersion, hashMethod, hashPrefix, hashSuffix, vgNum); pIter = taosHashIterate(dbHash, pIter); } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index e28234ab7603248e7261829bcb59e44ba24491fe..97b174de1c7c70553011aae62b9dcf3e779c0efb 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -99,7 +99,16 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { } void ctgFreeQNode(SCtgQNode *node) { - //TODO + if (NULL == node) { + return; + } + + if (node->op) { + taosMemoryFree(node->op->data); + taosMemoryFree(node->op); + } + + taosMemoryFree(node); } void ctgFreeSTableIndex(void *info) { @@ -848,15 +857,11 @@ int32_t ctgGetVgInfoFromHashValue(SCatalog *pCtg, SDBVgInfo *dbInfo, const SName CTG_ERR_RET(TSDB_CODE_TSC_DB_NOT_SELECTED); } - tableNameHashFp fp = NULL; SVgroupInfo *vgInfo = NULL; - - CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); - char tbFullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pTableName, tbFullName); - uint32_t hashValue = (*fp)(tbFullName, (uint32_t)strlen(tbFullName)); + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); void *pIter = taosHashIterate(dbInfo->vgHash, NULL); while (pIter) { @@ -919,11 +924,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo CTG_ERR_RET(TSDB_CODE_CTG_INTERNAL_ERROR); } - tableNameHashFp fp = NULL; SVgroupInfo *vgInfo = NULL; - - CTG_ERR_RET(ctgGetHashFunction(dbInfo->hashMethod, &fp)); - int32_t tbNum = taosArrayGetSize(pNames); if (1 == vgNum) { @@ -975,7 +976,7 @@ int32_t ctgGetVgInfosFromHashValue(SCatalog *pCtg, SCtgTaskReq* tReq, SDBVgInfo tbNameLen = offset + strlen(pName->tname); strcpy(tbFullName + offset, pName->tname); - uint32_t hashValue = (*fp)(tbFullName, (uint32_t)tbNameLen); + uint32_t hashValue = taosGetTbHashVal(tbFullName, (uint32_t)strlen(tbFullName), dbInfo->hashMethod, dbInfo->hashPrefix, dbInfo->hashSuffix); SVgroupInfo **p = taosArraySearch(pVgList, &hashValue, ctgHashValueComp, TD_EQ); diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 0be85333dca130884516c214e324fbab82e33953..c01c269e643b2a1f2a44817a74807c20db718d4b 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -218,6 +218,8 @@ void ctgTestBuildDBVgroup(SDBVgInfo **pdbVgroup) { ctgTestCurrentVgVersion = dbVgroup->vgVersion; dbVgroup->hashMethod = 0; + dbVgroup->hashPrefix = 0; + dbVgroup->hashSuffix = 0; dbVgroup->vgHash = taosHashInit(ctgTestVgNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); vgNum = ctgTestGetVgNumFromVgVersion(dbVgroup->vgVersion); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 9e7fcc222788e16f60252727ba7ca7c911366e9b..baf7d447cc6f0bbb3e8b552acdefd38abafc98cc 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -87,9 +87,8 @@ struct SqlFunctionCtx; size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); - -void initResultRow(SResultRow* pResultRow); -void closeResultRow(SResultRow* pResultRow); +void closeResultRow(SResultRow* pResultRow); +void resetResultRow(SResultRow* pResultRow, size_t entrySize); struct SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset); @@ -128,6 +127,7 @@ SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode); SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs); SqlFunctionCtx* createSqlFunctionCtx(SExprInfo* pExprInfo, int32_t numOfOutput, int32_t** rowEntryInfoOffset); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 1dfa9da7d53b24981074148d7dbab9fc869282be..1e985a74763450cef880dd91c799514afa0e78fa 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -38,11 +38,11 @@ extern "C" { #include "tlockfree.h" #include "tmsg.h" #include "tpagedbuf.h" -#include "tstreamUpdate.h" #include "tstream.h" +#include "tstreamUpdate.h" -#include "vnode.h" #include "executorInt.h" +#include "vnode.h" typedef int32_t (*__block_search_fn_t)(char* data, int32_t num, int64_t key, int32_t order); @@ -139,23 +139,23 @@ enum { }; typedef struct { - //TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - - SSchemaWrapper *schema; - char tbName[TSDB_TABLE_NAME_LEN]; - SSDataBlock* pullOverBlk; // for streaming - SWalFilterCond cond; - int64_t lastScanUid; - int8_t recoverStep; + // TODO remove prepareStatus + STqOffsetVal prepareStatus; // for tmq + STqOffsetVal lastStatus; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int8_t returned; + int64_t snapshotVer; + + SSchemaWrapper* schema; + char tbName[TSDB_TABLE_NAME_LEN]; + SSDataBlock* pullOverBlk; // for streaming + SWalFilterCond cond; + int64_t lastScanUid; + int8_t recoverStep; SQueryTableDataCond tableCond; - int64_t recoverStartVer; - int64_t recoverEndVer; - SStreamState* pState; + int64_t recoverStartVer; + int64_t recoverEndVer; + SStreamState* pState; } SStreamTaskInfo; typedef struct { @@ -167,30 +167,30 @@ typedef struct { } SSchemaInfo; typedef struct SExecTaskInfo { - STaskIdInfo id; - uint32_t status; - STimeWindow window; - STaskCostInfo cost; - int64_t owner; // if it is in execution - int32_t code; - - int64_t version; // used for stream to record wal version - SStreamTaskInfo streamInfo; - SSchemaInfo schemaInfo; - STableListInfo tableqinfoList; // this is a table list - const char* sql; // query sql string - jmp_buf env; // jump to this position when error happens. - EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] - SSubplan* pSubplan; + STaskIdInfo id; + uint32_t status; + STimeWindow window; + STaskCostInfo cost; + int64_t owner; // if it is in execution + int32_t code; + + int64_t version; // used for stream to record wal version + SStreamTaskInfo streamInfo; + SSchemaInfo schemaInfo; + STableListInfo tableqinfoList; // this is a table list + const char* sql; // query sql string + jmp_buf env; // jump to this position when error happens. + EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] + SSubplan* pSubplan; struct SOperatorInfo* pRoot; SLocalFetch localFetch; } SExecTaskInfo; enum { - OP_NOT_OPENED = 0x0, - OP_OPENED = 0x1, + OP_NOT_OPENED = 0x0, + OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, - OP_EXEC_DONE = 0x9, + OP_EXEC_DONE = 0x9, }; typedef struct SOperatorFpSet { @@ -229,7 +229,7 @@ typedef struct SOperatorInfo { typedef enum { EX_SOURCE_DATA_NOT_READY = 0x1, - EX_SOURCE_DATA_READY = 0x2, + EX_SOURCE_DATA_READY = 0x2, EX_SOURCE_DATA_EXHAUSTED = 0x3, } EX_SOURCE_STATUS; @@ -252,26 +252,26 @@ typedef struct SLoadRemoteDataInfo { } SLoadRemoteDataInfo; typedef struct SLimitInfo { - SLimit limit; - SLimit slimit; - uint64_t currentGroupId; - int64_t remainGroupOffset; - int64_t numOfOutputGroups; - int64_t remainOffset; - int64_t numOfOutputRows; + SLimit limit; + SLimit slimit; + uint64_t currentGroupId; + int64_t remainGroupOffset; + int64_t numOfOutputGroups; + int64_t remainOffset; + int64_t numOfOutputRows; } SLimitInfo; typedef struct SExchangeInfo { - SArray* pSources; - SArray* pSourceDataInfo; - tsem_t ready; - void* pTransporter; + SArray* pSources; + SArray* pSourceDataInfo; + tsem_t ready; + void* pTransporter; // SArray, result block list, used to keep the multi-block that // passed by downstream operator SArray* pResultBlockList; - int32_t rspBlockIndex; // indicate the return block index in pResultBlockList - SSDataBlock* pDummyBlock; // dummy block, not keep data - bool seqLoadData; // sequential load data or not, false by default + int32_t rspBlockIndex; // indicate the return block index in pResultBlockList + SSDataBlock* pDummyBlock; // dummy block, not keep data + bool seqLoadData; // sequential load data or not, false by default int32_t current; SLoadRemoteDataInfo loadInfo; uint64_t self; @@ -279,22 +279,22 @@ typedef struct SExchangeInfo { } SExchangeInfo; typedef struct SColMatchInfo { - int32_t srcSlotId; // source slot id + int32_t srcSlotId; // source slot id int32_t colId; int32_t targetSlotId; - bool output; // todo remove this? + bool output; // todo remove this? bool reserved; - int32_t matchType; // determinate the source according to col id or slot id + int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; typedef struct SScanInfo { - int32_t numOfAsc; - int32_t numOfDesc; + int32_t numOfAsc; + int32_t numOfDesc; } SScanInfo; typedef struct SSampleExecInfo { - double sampleRatio; // data block sample ratio, 1 by default - uint32_t seed; // random seed value + double sampleRatio; // data block sample ratio, 1 by default + uint32_t seed; // random seed value } SSampleExecInfo; enum { @@ -306,39 +306,40 @@ typedef struct SAggSupporter { SSHashObj* pResultRowHashTable; // quick locate the window object for each result char* keyBuf; // window key buffer SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // current write page id + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t currentPageId; // current write page id } SAggSupporter; typedef struct { - // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if current data block needs to be loaded. - SInterval interval; - SAggSupporter *pAggSup; - SExprSupp *pExprSup; // expr supporter of aggregate operator + // if the upstream is an interval operator, the interval info is also kept here to get the time window to check if + // current data block needs to be loaded. + SInterval interval; + SAggSupporter* pAggSup; + SExprSupp* pExprSup; // expr supporter of aggregate operator } SAggOptrPushDownInfo; typedef struct STableScanInfo { - STsdbReader* dataReader; - SReadHandle readHandle; + STsdbReader* dataReader; + SReadHandle readHandle; SFileBlockLoadRecorder readRecorder; - SScanInfo scanInfo; - int32_t scanTimes; - SNode* pFilterNode; // filter info, which is push down by optimizer + SScanInfo scanInfo; + int32_t scanTimes; + SNode* pFilterNode; // filter info, which is push down by optimizer - SSDataBlock* pResBlock; - SArray* pColMatchInfo; - SExprSupp pseudoSup; - SQueryTableDataCond cond; - int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan - int32_t dataBlockLoadFlag; - SSampleExecInfo sample; // sample execution info - int32_t currentGroupId; - int32_t currentTable; - int8_t scanMode; - int8_t noTable; + SSDataBlock* pResBlock; + SArray* pColMatchInfo; + SExprSupp pseudoSup; + SQueryTableDataCond cond; + int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan + int32_t dataBlockLoadFlag; + SSampleExecInfo sample; // sample execution info + int32_t currentGroupId; + int32_t currentTable; + int8_t scanMode; + int8_t noTable; SAggOptrPushDownInfo pdInfo; - int8_t assignBlockUid; + int8_t assignBlockUid; } STableScanInfo; typedef struct STableMergeScanInfo { @@ -371,7 +372,7 @@ typedef struct STableMergeScanInfo { SArray* pColMatchInfo; int32_t numOfOutput; - SExprSupp pseudoSup; + SExprSupp pseudoSup; SQueryTableDataCond cond; int32_t scanFlag; // table scan flag to denote if it is a repeat/reverse/main scan @@ -385,32 +386,33 @@ typedef struct STableMergeScanInfo { } STableMergeScanInfo; typedef struct STagScanInfo { - SColumnInfo *pCols; - SSDataBlock *pRes; - SArray *pColMatchInfo; - int32_t curPos; - SReadHandle readHandle; - STableListInfo *pTableList; + SColumnInfo* pCols; + SSDataBlock* pRes; + SArray* pColMatchInfo; + int32_t curPos; + SReadHandle readHandle; + STableListInfo* pTableList; } STagScanInfo; typedef struct SLastrowScanInfo { - SSDataBlock *pRes; - SReadHandle readHandle; - void *pLastrowReader; - SArray *pColMatchInfo; - int32_t *pSlotIds; - SExprSupp pseudoExprSup; - int32_t retrieveType; - int32_t currentGroupIndex; - SSDataBlock *pBufferredRes; - SArray *pUidList; - int32_t indexOfBufferedRes; + SSDataBlock* pRes; + SReadHandle readHandle; + void* pLastrowReader; + SArray* pColMatchInfo; + int32_t* pSlotIds; + SExprSupp pseudoExprSup; + int32_t retrieveType; + int32_t currentGroupIndex; + SSDataBlock* pBufferredRes; + SArray* pUidList; + int32_t indexOfBufferedRes; } SLastrowScanInfo; typedef enum EStreamScanMode { STREAM_SCAN_FROM_READERHANDLE = 1, STREAM_SCAN_FROM_RES, STREAM_SCAN_FROM_UPDATERES, + STREAM_SCAN_FROM_DELETE_DATA, STREAM_SCAN_FROM_DATAREADER_RETRIEVE, STREAM_SCAN_FROM_DATAREADER_RANGE, } EStreamScanMode; @@ -432,25 +434,38 @@ typedef struct SStreamAggSupporter { SArray* pCurWins; int32_t valueSize; int32_t keySize; - char* pKeyBuf; // window key buffer - SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file - int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row - int32_t currentPageId; // buffer page that is active + char* pKeyBuf; // window key buffer + SDiskbasedBuf* pResultBuf; // query result buffer based on blocked-wised disk file + int32_t resultRowSize; // the result buffer size for each result row, with the meta data size for each row + int32_t currentPageId; // buffer page that is active SSDataBlock* pScanBlock; } SStreamAggSupporter; -typedef struct SessionWindowSupporter { +typedef struct SWindowSupporter { SStreamAggSupporter* pStreamAggSup; int64_t gap; uint16_t parentType; SAggSupporter* pIntervalAggSup; -} SessionWindowSupporter; +} SWindowSupporter; -typedef struct STimeWindowSupp { - int8_t calTrigger; - int64_t waterMark; - TSKEY maxTs; - SColumnInfoData timeWindowData; // query time window info for scalar function execution. +typedef struct SPartitionBySupporter { + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + char* keyBuf; // group by keys for hash + bool needCalc; // partition by column +} SPartitionBySupporter; + +typedef struct SPartitionDataInfo { + uint64_t groupId; + SArray* rowIds; +} SPartitionDataInfo; + +typedef struct STimeWindowAggSupp { + int8_t calTrigger; + int64_t waterMark; + TSKEY maxTs; + TSKEY minTs; + SColumnInfoData timeWindowData; // query time window info for scalar function execution. } STimeWindowAggSupp; typedef struct SStreamScanInfo { @@ -475,20 +490,22 @@ typedef struct SStreamScanInfo { uint64_t groupId; SUpdateInfo* pUpdateInfo; - EStreamScanMode scanMode; - SOperatorInfo* pStreamScanOp; - SOperatorInfo* pTableScanOp; - SArray* childIds; - SessionWindowSupporter sessionSup; - bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. - int32_t scanWinIndex; // for state operator - int32_t pullDataResIndex; - SSDataBlock* pPullDataRes; // pull data SSDataBlock - SSDataBlock* pDeleteDataRes; // delete data SSDataBlock - int32_t deleteDataIndex; - STimeWindow updateWin; - STimeWindowAggSupp twAggSup; - SSDataBlock* pUpdateDataRes; + EStreamScanMode scanMode; + SOperatorInfo* pStreamScanOp; + SOperatorInfo* pTableScanOp; + SArray* childIds; + SWindowSupporter windowSup; + SPartitionBySupporter partitionSup; + SExprSupp* pPartScalarSup; + bool assignBlockUid; // assign block uid to groupId, temporarily used for generating rollup SMA. + int32_t scanWinIndex; // for state operator + int32_t pullDataResIndex; + SSDataBlock* pPullDataRes; // pull data SSDataBlock + SSDataBlock* pDeleteDataRes; // delete data SSDataBlock + int32_t deleteDataIndex; + STimeWindow updateWin; + STimeWindowAggSupp twAggSup; + SSDataBlock* pUpdateDataRes; // status for tmq SNodeList* pGroupTags; SNode* pTagCond; @@ -557,26 +574,27 @@ typedef struct SIntervalAggOperatorInfo { EOPTR_EXEC_MODEL execModel; // operator execution model [batch model|stream model] STimeWindowAggSupp twAggSup; bool invertible; - SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. + SArray* pPrevValues; // SArray used to keep the previous not null value for interpolation. bool ignoreExpiredData; SArray* pRecycledPages; - SArray* pDelWins; // SWinRes + SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; SNode* pCondition; } SIntervalAggOperatorInfo; typedef struct SMergeAlignedIntervalAggOperatorInfo { - SIntervalAggOperatorInfo *intervalAggOperatorInfo; + SIntervalAggOperatorInfo* intervalAggOperatorInfo; - bool hasGroupId; +// bool hasGroupId; uint64_t groupId; // current groupId int64_t curTs; // current ts SSDataBlock* prefetchedBlock; SNode* pCondition; + SResultRow* pResultRow; } SMergeAlignedIntervalAggOperatorInfo; -typedef struct SStreamFinalIntervalOperatorInfo { +typedef struct SStreamIntervalOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter @@ -584,57 +602,74 @@ typedef struct SStreamFinalIntervalOperatorInfo { SGroupResInfo groupResInfo; // multiple results build supporter SInterval interval; // interval info int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. - int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + bool invertible; + bool ignoreExpiredData; + SArray* pRecycledPages; + SArray* pDelWins; // SWinRes + int32_t delIndex; + SSDataBlock* pDelRes; + bool isFinal; +} SStreamIntervalOperatorInfo; + +typedef struct SStreamFinalIntervalOperatorInfo { + // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode + SOptrBasicInfo binfo; // basic info + SAggSupporter aggSup; // aggregate supporter + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; // multiple results build supporter + SInterval interval; // interval info + int32_t primaryTsIndex; // primary time stamp slot id from result of downstream operator. + int32_t order; // current SSDataBlock scan order STimeWindowAggSupp twAggSup; SArray* pChildren; SSDataBlock* pUpdateRes; bool returnUpdate; - SPhysiNode* pPhyNode; // create new child + SPhysiNode* pPhyNode; // create new child bool isFinal; SHashObj* pPullDataMap; - SArray* pPullWins; // SPullWindowInfo + SArray* pPullWins; // SPullWindowInfo int32_t pullIndex; SSDataBlock* pPullDataRes; bool ignoreExpiredData; SArray* pRecycledPages; - SArray* pDelWins; // SWinRes + SArray* pDelWins; // SWinRes int32_t delIndex; SSDataBlock* pDelRes; } SStreamFinalIntervalOperatorInfo; typedef struct SAggOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; - STableQueryInfo *current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; - SNode *pCondition; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; + SNode* pCondition; } SAggOperatorInfo; typedef struct SProjectOperatorInfo { - // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SNode* pFilterNode; // filter info, which is push down by optimizer - SArray* pPseudoColInfo; - SLimitInfo limitInfo; - bool mergeDataBlocks; - SSDataBlock* pFinalRes; - SNode* pCondition; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SNode* pFilterNode; // filter info, which is push down by optimizer + SArray* pPseudoColInfo; + SLimitInfo limitInfo; + bool mergeDataBlocks; + SSDataBlock* pFinalRes; + SNode* pCondition; } SProjectOperatorInfo; typedef struct SIndefOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - SArray* pPseudoColInfo; - SExprSupp scalarSup; - SNode* pCondition; - uint64_t groupId; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + SArray* pPseudoColInfo; + SExprSupp scalarSup; + SNode* pCondition; + uint64_t groupId; - SSDataBlock* pNextGroupRes; + SSDataBlock* pNextGroupRes; } SIndefOperatorInfo; typedef struct SFillOperatorInfo { @@ -649,7 +684,7 @@ typedef struct SFillOperatorInfo { SArray* pColMatchColInfo; int32_t primaryTsCol; int32_t primarySrcSlotId; - uint64_t curGroupId; // current handled group id + uint64_t curGroupId; // current handled group id SExprInfo* pExprInfo; int32_t numOfExpr; SExprInfo* pNotFillExprInfo; @@ -657,24 +692,23 @@ typedef struct SFillOperatorInfo { } SFillOperatorInfo; typedef struct SGroupbyOperatorInfo { - // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; - - SArray* pGroupCols; // group by columns, SArray - SArray* pGroupColVals; // current group column values, SArray - SNode* pCondition; - bool isInit; // denote if current val is initialized or not - char* keyBuf; // group by keys for hash - int32_t groupKeyLen; // total group by column width - SGroupResInfo groupResInfo; - SExprSupp scalarSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; + + SArray* pGroupCols; // group by columns, SArray + SArray* pGroupColVals; // current group column values, SArray + SNode* pCondition; + bool isInit; // denote if current val is initialized or not + char* keyBuf; // group by keys for hash + int32_t groupKeyLen; // total group by column width + SGroupResInfo groupResInfo; + SExprSupp scalarSup; } SGroupbyOperatorInfo; typedef struct SDataGroupInfo { - uint64_t groupId; - int64_t numOfRows; - SArray* pPageList; + uint64_t groupId; + int64_t numOfRows; + SArray* pPageList; } SDataGroupInfo; // The sort in partition may be needed later. @@ -686,13 +720,12 @@ typedef struct SPartitionOperatorInfo { int32_t groupKeyLen; // total group by column width SHashObj* pGroupSet; // quick locate the window object for each result - SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file - int32_t rowCapacity; // maximum number of rows for each buffer page - int32_t* columnOffset; // start position for each column data - SArray* sortedGroupArray; // SDataGroupInfo sorted by group id - int32_t groupIndex; // group index - int32_t pageIndex; // page index of current group - SSDataBlock* pUpdateRes; + SDiskbasedBuf* pBuf; // query result buffer based on blocked-wised disk file + int32_t rowCapacity; // maximum number of rows for each buffer page + int32_t* columnOffset; // start position for each column data + SArray* sortedGroupArray; // SDataGroupInfo sorted by group id + int32_t groupIndex; // group index + int32_t pageIndex; // page index of current group SExprSupp scalarSup; } SPartitionOperatorInfo; @@ -705,76 +738,86 @@ typedef struct SWindowRowsSup { } SWindowRowsSup; typedef struct SSessionAggOperatorInfo { - // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; - bool reptScan; // next round scan - int64_t gap; // session window gap - int32_t tsSlotId; // primary timestamp slot id + bool reptScan; // next round scan + int64_t gap; // session window gap + int32_t tsSlotId; // primary timestamp slot id STimeWindowAggSupp twAggSup; - const SNode* pCondition; + const SNode* pCondition; } SSessionAggOperatorInfo; typedef struct SResultWindowInfo { SResultRowPosition pos; - STimeWindow win; - uint64_t groupId; - bool isOutput; - bool isClosed; + STimeWindow win; + uint64_t groupId; + bool isOutput; + bool isClosed; } SResultWindowInfo; typedef struct SStateWindowInfo { SResultWindowInfo winInfo; - SStateKeys stateKey; + SStateKeys stateKey; } SStateWindowInfo; typedef struct SStreamSessionAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int64_t gap; // session window gap - int32_t primaryTsIndex; // primary timestamp slot id - int32_t endTsIndex; // window end timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SSDataBlock* pWinBlock; // window result - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; // delete result - bool returnDelete; - SSDataBlock* pUpdateRes; // update window - SHashObj* pStDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; final stream operator - SPhysiNode* pPhyNode; // create new child - bool isFinal; - bool ignoreExpiredData; + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int64_t gap; // session window gap + int32_t primaryTsIndex; // primary timestamp slot id + int32_t endTsIndex; // window end timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SSDataBlock* pWinBlock; // window result + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; // delete result + SSDataBlock* pUpdateRes; // update window + bool returnUpdate; + SHashObj* pStDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; final stream operator + SPhysiNode* pPhyNode; // create new child + bool isFinal; + bool ignoreExpiredData; } SStreamSessionAggOperatorInfo; +typedef struct SStreamPartitionOperatorInfo { + SOptrBasicInfo binfo; + SPartitionBySupporter partitionSup; + SExprSupp scalarSup; + SHashObj* pPartitions; + void* parIte; + SSDataBlock* pInputDataBlock; + int32_t tsColIndex; + SSDataBlock* pDelRes; +} SStreamPartitionOperatorInfo; + typedef struct STimeSliceOperatorInfo { - SSDataBlock* pRes; - STimeWindow win; - SInterval interval; - int64_t current; - SArray* pPrevRow; // SArray - SArray* pNextRow; // SArray - SArray* pLinearInfo; // SArray - bool fillLastPoint; - bool isPrevRowSet; - bool isNextRowSet; - int32_t fillType; // fill type - SColumn tsCol; // primary timestamp column - SExprSupp scalarSup; // scalar calculation - struct SFillColInfo* pFillColInfo; // fill column info + SSDataBlock* pRes; + STimeWindow win; + SInterval interval; + int64_t current; + SArray* pPrevRow; // SArray + SArray* pNextRow; // SArray + SArray* pLinearInfo; // SArray + bool fillLastPoint; + bool isPrevRowSet; + bool isNextRowSet; + int32_t fillType; // fill type + SColumn tsCol; // primary timestamp column + SExprSupp scalarSup; // scalar calculation + struct SFillColInfo* pFillColInfo; // fill column info } STimeSliceOperatorInfo; typedef struct SStateWindowOperatorInfo { // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; + SOptrBasicInfo binfo; + SAggSupporter aggSup; SGroupResInfo groupResInfo; SWindowRowsSup winSup; @@ -783,57 +826,37 @@ typedef struct SStateWindowOperatorInfo { SStateKeys stateKey; int32_t tsSlotId; // primary timestamp column slot id STimeWindowAggSupp twAggSup; - // bool reptScan; const SNode* pCondition; } SStateWindowOperatorInfo; typedef struct SStreamStateAggOperatorInfo { - SOptrBasicInfo binfo; - SStreamAggSupporter streamAggSup; - SExprSupp scalarSupp; // supporter for perform scalar function - SGroupResInfo groupResInfo; - int32_t primaryTsIndex; // primary timestamp slot id - int32_t order; // current SSDataBlock scan order - STimeWindowAggSupp twAggSup; - SColumn stateCol; - SqlFunctionCtx* pDummyCtx; // for combine - SSDataBlock* pDelRes; - SHashObj* pSeDeleted; - void* pDelIterator; - SArray* pChildren; // cache for children's result; - bool ignoreExpiredData; + SOptrBasicInfo binfo; + SStreamAggSupporter streamAggSup; + SExprSupp scalarSupp; // supporter for perform scalar function + SGroupResInfo groupResInfo; + int32_t primaryTsIndex; // primary timestamp slot id + int32_t order; // current SSDataBlock scan order + STimeWindowAggSupp twAggSup; + SColumn stateCol; + SqlFunctionCtx* pDummyCtx; // for combine + SSDataBlock* pDelRes; + SHashObj* pSeDeleted; + void* pDelIterator; + SArray* pChildren; // cache for children's result; + bool ignoreExpiredData; } SStreamStateAggOperatorInfo; -typedef struct SSortedMergeOperatorInfo { - // SOptrBasicInfo should be first, SAggSupporter should be second for stream encode - SOptrBasicInfo binfo; - SAggSupporter aggSup; - - SArray* pSortInfo; - int32_t numOfSources; - SSortHandle *pSortHandle; - int32_t bufPageSize; - uint32_t sortBufSize; // max buffer size for in-memory sort - int32_t resultRowFactor; - bool hasGroupVal; - SDiskbasedBuf *pTupleStore; // keep the final results - int32_t numOfResPerPage; - char** groupVal; - SArray *groupInfo; -} SSortedMergeOperatorInfo; - typedef struct SSortOperatorInfo { SOptrBasicInfo binfo; - uint32_t sortBufSize; // max buffer size for in-memory sort - SArray* pSortInfo; - SSortHandle* pSortHandle; - SArray* pColMatchInfo; // for index map from table scan output - int32_t bufPageSize; - - int64_t startTs; // sort start time - uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. - SLimitInfo limitInfo; - SNode* pCondition; + uint32_t sortBufSize; // max buffer size for in-memory sort + SArray* pSortInfo; + SSortHandle* pSortHandle; + SArray* pColMatchInfo; // for index map from table scan output + int32_t bufPageSize; + int64_t startTs; // sort start time + uint64_t sortElapsed; // sort elapsed time, time to flush to disk not included. + SLimitInfo limitInfo; + SNode* pCondition; } SSortOperatorInfo; typedef struct STagFilterOperatorInfo { @@ -841,18 +864,18 @@ typedef struct STagFilterOperatorInfo { } STagFilterOperatorInfo; typedef struct SJoinOperatorInfo { - SSDataBlock *pRes; - int32_t joinType; - int32_t inputOrder; - - SSDataBlock *pLeft; - int32_t leftPos; - SColumnInfo leftCol; - - SSDataBlock *pRight; - int32_t rightPos; - SColumnInfo rightCol; - SNode *pCondAfterMerge; + SSDataBlock* pRes; + int32_t joinType; + int32_t inputOrder; + + SSDataBlock* pLeft; + int32_t leftPos; + SColumnInfo leftCol; + + SSDataBlock* pRight; + int32_t rightPos; + SColumnInfo rightCol; + SNode* pCondAfterMerge; } SJoinOperatorInfo; #define OPTR_IS_OPENED(_optr) (((_optr)->status & OP_OPENED) == OP_OPENED) @@ -861,11 +884,10 @@ typedef struct SJoinOperatorInfo { void doDestroyExchangeOperatorInfo(void* param); SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, __optr_fn_t streamFn, - __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, - __optr_decode_fn_t decode, __optr_explain_fn_t explain); + __optr_fn_t cleanup, __optr_close_fn_t closeFn, __optr_encode_fn_t encode, + __optr_decode_fn_t decode, __optr_explain_fn_t explain); int32_t operatorDummyOpenFn(SOperatorInfo* pOperator); -void operatorDummyCloseFn(void* param, int32_t numOfCols); int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t num); void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); @@ -873,24 +895,25 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo); int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); void cleanupExprSupp(SExprSupp* pSup); void destroyExprInfo(SExprInfo* pExpr, int32_t numOfExprs); -int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, +int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); -void initResultSizeInfo(SResultInfo * pResultInfo, int32_t numOfRows); -void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, SDiskbasedBuf* pBuf); -int32_t handleLimitOffset(SOperatorInfo *pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); +void initResultSizeInfo(SResultInfo* pResultInfo, int32_t numOfRows); +void doBuildResultDatablock(SOperatorInfo* pOperator, SOptrBasicInfo* pbInfo, SGroupResInfo* pGroupResInfo, + SDiskbasedBuf* pBuf); +int32_t handleLimitOffset(SOperatorInfo* pOperator, SLimitInfo* pLimitInfo, SSDataBlock* pBlock, bool holdDataInBuf); bool hasLimitOffsetInfo(SLimitInfo* pLimitInfo); void initLimitInfo(const SNode* pLimit, const SNode* pSLimit, SLimitInfo* pLimitInfo); -void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset, - int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); +void doApplyFunctions(SExecTaskInfo* taskInfo, SqlFunctionCtx* pCtx, SColumnInfoData* pTimeWindowData, int32_t offset, + int32_t forwardStep, int32_t numOfTotal, int32_t numOfOutput); -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, char** pNextStart); -void updateLoadRemoteInfo(SLoadRemoteDataInfo *pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, - SOperatorInfo* pOperator); +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart); +void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t dataLen, int64_t startTs, + SOperatorInfo* pOperator); STimeWindow getFirstQualifiedTimeWindow(int64_t ts, STimeWindow* pWindow, SInterval* pInterval, int32_t order); -int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t *order, int32_t* scanFlag); +int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag); int32_t getBufferPgSize(int32_t rowSize, uint32_t* defaultPgsz, uint32_t* defaultBufsz); void doSetOperatorCompleted(SOperatorInfo* pOperator); @@ -898,10 +921,9 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock, const SArray* pC int32_t addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_t numOfPseudoExpr, SSDataBlock* pBlock, const char* idStr); -void cleanupAggSup(SAggSupporter* pAggSup); -void destroyBasicOperatorInfo(void* param, int32_t numOfOutput); -void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); -void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); +void cleanupAggSup(SAggSupporter* pAggSup); +void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle); +void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* pColInfoData, int32_t functionId); int32_t doPrepareScan(SOperatorInfo* pOperator, uint64_t uid, int64_t ts); int32_t doGetScanStatus(SOperatorInfo* pOperator, uint64_t* uid, int64_t* ts); @@ -910,29 +932,37 @@ SSDataBlock* loadNextDataBlock(void* param); void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset); -SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, - char* pData, int16_t bytes, bool masterscan, uint64_t groupId, - SExecTaskInfo* pTaskInfo, bool isIntervalQuery, SAggSupporter* pSup); +SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pResultRowInfo, char* pData, + int16_t bytes, bool masterscan, uint64_t groupId, SExecTaskInfo* pTaskInfo, + bool isIntervalQuery, SAggSupporter* pSup); SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode* pExNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysiNode* pPhyNode, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode *pScanPhyNode, const char* pUser, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScanPhysiNode* pScanPhyNode, + const char* pUser, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, +SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, SNode* pCondition, SExprInfo* pScalarExprInfo, int32_t numOfScalarExpr, bool mergeResult, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode *pNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createProjectOperatorInfo(SOperatorInfo* downstream, SProjectPhysiNode* pProjPhyNode, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMultiwayMergeOperatorInfo(SOperatorInfo** dowStreams, size_t numStreams, + SMergePhysiNode* pMergePhysiNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pTableScanNode, SReadHandle* readHandle, + SExecTaskInfo* pTaskInfo); SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, - STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, bool isStream); + STimeWindowAggSupp* pTwAggSupp, SIntervalPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, bool isStream); SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMergeIntervalPhysiNode* pIntervalPhyNode, SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SMergeAlignedIntervalPhysiNode* pNode, @@ -952,30 +982,41 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys SOperatorInfo* createRawScanOperatorInfo(SReadHandle* pHandle, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SSortMergeJoinPhysiNode* pJoinNode, +SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWinodwPhysiNode* pStateNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, + SSortMergeJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); + +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild); +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, + SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo); int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBlock* pSrcBlock, SqlFunctionCtx* pCtx, - int32_t numOfOutput, SArray* pPseudoList); + int32_t numOfOutput, SArray* pPseudoList); -void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, int32_t scanFlag, bool createDummyCol); +void setInputDataBlock(SOperatorInfo* pOperator, SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t order, + int32_t scanFlag, bool createDummyCol); bool isTaskKilled(SExecTaskInfo* pTaskInfo); int32_t checkForQueryBuf(size_t numOfTables); -void setTaskKilled(SExecTaskInfo* pTaskInfo); -void queryCostStatis(SExecTaskInfo* pTaskInfo); +void setTaskKilled(SExecTaskInfo* pTaskInfo); +void queryCostStatis(SExecTaskInfo* pTaskInfo); void doDestroyTask(SExecTaskInfo* pTaskInfo); int32_t getMaximumIdleDurationSec(); @@ -987,7 +1028,7 @@ int32_t getMaximumIdleDurationSec(); * nOptrWithVal: *nOptrWithVal save the number of optr with value * return: result code, 0 means success */ -int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t *length, int32_t *nOptrWithVal); +int32_t encodeOperator(SOperatorInfo* ops, char** data, int32_t* length, int32_t* nOptrWithVal); /* * ops: root operator, created by caller @@ -1000,7 +1041,7 @@ int32_t decodeOperator(SOperatorInfo* ops, const char* data, int32_t length); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode *pNode, void **pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pTaskInfo, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result); @@ -1009,41 +1050,48 @@ int32_t aggEncodeResultRow(SOperatorInfo* pOperator, char** result, int32_t* len STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, int32_t order); int32_t getNumOfRowsInTimeWindow(SDataBlockInfo* pDataBlockInfo, TSKEY* pPrimaryColumn, int32_t startPos, TSKEY ekey, - __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); + __block_search_fn_t searchFn, STableQueryInfo* item, int32_t order); int32_t binarySearchForKey(char* pValue, int num, TSKEY key, int order); int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput, - int32_t size); -SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); -SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex); -bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); -bool functionNeedToExecute(SqlFunctionCtx* pCtx); -bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); -bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); -bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); -void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, int32_t uidCol, uint64_t* pID); -void printDataBlock(SSDataBlock* pBlock, const char* flag); - -int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, - SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, - SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); - -int32_t createScanTableListInfo(SScanPhysiNode *pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, const char* idstr); + int32_t size); +SResultRow* getNewResultRow(SDiskbasedBuf* pResultBuf, int32_t* currentPageId, int32_t interBufSize); +SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex); +SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex); +bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap); +bool functionNeedToExecute(SqlFunctionCtx* pCtx); +bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup); +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup); +bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup); +void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp); +void printDataBlock(SSDataBlock* pBlock, const char* flag); +uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId); + +int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, + SExprSupp* pSup, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); + +int32_t createScanTableListInfo(SScanPhysiNode* pScanNode, SNodeList* pGroupTags, bool groupSort, SReadHandle* pHandle, + STableListInfo* pTableListInfo, SNode* pTagCond, SNode* pTagIndexCond, + const char* idstr); SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); -bool groupbyTbname(SNodeList* pGroupList); +bool groupbyTbname(SNodeList* pGroupList); int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SNodeList* groupKey); -SSDataBlock* createSpecialDataBlock(EStreamType type); -void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); +void* destroySqlFunctionCtx(SqlFunctionCtx* pCtx, int32_t numOfOutput); +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo); +int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo); +int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult); +int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize); #ifdef __cplusplus } diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 06b7c13fa2cb52b8255098c5efb652d56ec57974..55a1a1fdb914ff80acdaf5fb9d23dcb2bd5921f2 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -79,25 +79,33 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp pEntry->dataLen = sizeof(SDeleterRes); ASSERT(1 == pEntry->numOfRows); - ASSERT(1 == pEntry->numOfCols); + ASSERT(3 == pEntry->numOfCols); pBuf->useSize = sizeof(SDataCacheEntry); SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); + SColumnInfoData* pColSKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 1); + SColumnInfoData* pColEKey = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 2); SDeleterRes* pRes = (SDeleterRes*)pEntry->data; pRes->suid = pHandle->pParam->suid; pRes->uidList = pHandle->pParam->pUidList; - pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; - pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; strcpy(pRes->tableName, pHandle->pDeleter->tableFName); strcpy(pRes->tsColName, pHandle->pDeleter->tsColName); pRes->affectedRows = *(int64_t*)pColRes->pData; + if (pRes->affectedRows) { + pRes->skey = *(int64_t*)pColSKey->pData; + pRes->ekey = *(int64_t*)pColEKey->pData; + ASSERT(pRes->skey <= pRes->ekey); + } else { + pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; + pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; + } pBuf->useSize += pEntry->dataLen; - - atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); - atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + + atomic_add_fetch_64(&pHandle->cachedSize, pEntry->dataLen); + atomic_add_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); } static bool allocBuf(SDataDeleterHandle* pDeleter, const SInputData* pInput, SDataDeleterBuf* pBuf) { @@ -168,9 +176,12 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); memcpy(&pDeleter->nextOutput, pBuf, sizeof(SDataDeleterBuf)); taosFreeQitem(pBuf); - *pLen = ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->dataLen; + + SDataCacheEntry* pEntry = (SDataCacheEntry*)pDeleter->nextOutput.pData; + *pLen = pEntry->dataLen; *pQueryEnd = pDeleter->queryEnd; - qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); + qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, + ((SDataCacheEntry*)(pDeleter->nextOutput.pData))->numOfRows); } static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { @@ -184,14 +195,14 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { return TSDB_CODE_SUCCESS; } SDataCacheEntry* pEntry = (SDataCacheEntry*)(pDeleter->nextOutput.pData); - memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); + memcpy(pOutput->pData, pEntry->data, pEntry->dataLen); pDeleter->pParam->pUidList = NULL; pOutput->numOfRows = pEntry->numOfRows; pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; - atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); - atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&pDeleter->cachedSize, pEntry->dataLen); + atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); taosMemoryFreeClear(pDeleter->nextOutput.pData); // todo persistent pOutput->bufStatus = updateStatus(pDeleter); @@ -200,7 +211,7 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->useconds = pDeleter->useconds; pOutput->precision = pDeleter->pSchema->precision; taosThreadMutexUnlock(&pDeleter->mutex); - + return TSDB_CODE_SUCCESS; } @@ -209,7 +220,7 @@ static int32_t destroyDataSinker(SDataSinkHandle* pHandle) { atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pDeleter->cachedSize); taosMemoryFreeClear(pDeleter->nextOutput.pData); taosArrayDestroy(pDeleter->pParam->pUidList); - taosMemoryFree(pDeleter->pParam); + taosMemoryFree(pDeleter->pParam); while (!taosQueueEmpty(pDeleter->pDataBlocks)) { SDataDeleterBuf* pBuf = NULL; taosReadQitem(pDeleter->pDataBlocks, (void**)&pBuf); @@ -228,14 +239,15 @@ static int32_t getCacheSize(struct SDataSinkHandle* pHandle, uint64_t* size) { return TSDB_CODE_SUCCESS; } -int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, void *pParam) { +int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pDataSink, DataSinkHandle* pHandle, + void* pParam) { SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { terrno = TSDB_CODE_QRY_OUT_OF_MEMORY; return TSDB_CODE_QRY_OUT_OF_MEMORY; } - SDataDeleterNode* pDeleterNode = (SDataDeleterNode *)pDataSink; + SDataDeleterNode* pDeleterNode = (SDataDeleterNode*)pDataSink; deleter->sink.fPut = putDataBlock; deleter->sink.fEndPut = endPut; deleter->sink.fGetLen = getDataLength; diff --git a/source/libs/executor/src/dataDispatcher.c b/source/libs/executor/src/dataDispatcher.c index 20396046ba5daa34c3caaf76796c2b8a3b06527c..1697ed63fb196aa2a571aa26f8ffe29ee1d6c5d5 100644 --- a/source/libs/executor/src/dataDispatcher.c +++ b/source/libs/executor/src/dataDispatcher.c @@ -93,6 +93,8 @@ static void toDataCacheEntry(SDataDispatchHandle* pHandle, const SInputData* pIn pBuf->useSize = sizeof(SDataCacheEntry); blockEncode(pInput->pData, pEntry->data, &pEntry->dataLen, numOfCols, pEntry->compressed); + ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8)); + ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4)); pBuf->useSize += pEntry->dataLen; @@ -170,7 +172,13 @@ static void getDataLength(SDataSinkHandle* pHandle, int64_t* pLen, bool* pQueryE taosReadQitem(pDispatcher->pDataBlocks, (void**)&pBuf); memcpy(&pDispatcher->nextOutput, pBuf, sizeof(SDataDispatchBuf)); taosFreeQitem(pBuf); - *pLen = ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->dataLen; + + SDataCacheEntry* pEntry = (SDataCacheEntry*)pDispatcher->nextOutput.pData; + *pLen = pEntry->dataLen; + + ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8)); + ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4)); + *pQueryEnd = pDispatcher->queryEnd; qDebug("got data len %" PRId64 ", row num %d in sink", *pLen, ((SDataCacheEntry*)(pDispatcher->nextOutput.pData))->numOfRows); } @@ -191,6 +199,9 @@ static int32_t getDataBlock(SDataSinkHandle* pHandle, SOutputData* pOutput) { pOutput->numOfCols = pEntry->numOfCols; pOutput->compressed = pEntry->compressed; + ASSERT(pEntry->numOfRows == *(int32_t*)(pEntry->data+8)); + ASSERT(pEntry->numOfCols == *(int32_t*)(pEntry->data+8+4)); + atomic_sub_fetch_64(&pDispatcher->cachedSize, pEntry->dataLen); atomic_sub_fetch_64(&gDataSinkStat.cachedSize, pEntry->dataLen); diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 70180d6dc0a3e4b8b84ce68c0be78eef9ffdf8bf..d75f0580e17d7754a28295c0db790e51bcfc3f8d 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -33,6 +33,17 @@ void initResultRowInfo(SResultRowInfo* pResultRowInfo) { void closeResultRow(SResultRow* pResultRow) { pResultRow->closed = true; } +void resetResultRow(SResultRow* pResultRow, size_t entrySize) { + pResultRow->numOfRows = 0; + pResultRow->closed = false; + pResultRow->endInterp = false; + pResultRow->startInterp = false; + + if (entrySize > 0) { + memset(pResultRow->pEntryInfo, 0, entrySize); + } +} + // TODO refactor: use macro SResultRowEntryInfo* getResultEntryInfo(const SResultRow* pRow, int32_t index, const int32_t* offset) { assert(index >= 0 && offset != NULL); @@ -475,7 +486,6 @@ static SColumnInfoData* getColInfoResult(void* metaHandle, uint64_t suid, SArray if (code != TSDB_CODE_SUCCESS) { terrno = code; qError("failed to create result, reason:%s", tstrerror(code)); - terrno = code; goto end; } @@ -800,9 +810,15 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, taosMemoryFreeClear(pColInfoData); } - for (int i = 0; i < taosArrayGetSize(res); i++) { + size_t numOfTables = taosArrayGetSize(res); + for (int i = 0; i < numOfTables; i++) { STableKeyInfo info = {.uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; - taosArrayPush(pListInfo->pTableList, &info); + void* p = taosArrayPush(pListInfo->pTableList, &info); + if (p == NULL) { + taosArrayDestroy(res); + return TSDB_CODE_OUT_OF_MEMORY; + } + qDebug("tagfilter get uid:%ld", info.uid); } @@ -938,15 +954,17 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod for (int32_t i = 0; i < numOfCols; ++i) { STargetNode* pNode = (STargetNode*)nodesListGetNode(pNodeList, i); - SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; - - SColMatchInfo c = {0}; - c.output = true; - c.colId = pColNode->colId; - c.srcSlotId = pColNode->slotId; - c.matchType = type; - c.targetSlotId = pNode->slotId; - taosArrayPush(pList, &c); + if (nodeType(pNode->pExpr) == QUERY_NODE_COLUMN) { + SColumnNode* pColNode = (SColumnNode*)pNode->pExpr; + + SColMatchInfo c = {0}; + c.output = true; + c.colId = pColNode->colId; + c.srcSlotId = pColNode->slotId; + c.matchType = type; + c.targetSlotId = pNode->slotId; + taosArrayPush(pList, &c); + } } *numOfOutputCols = 0; @@ -1010,6 +1028,100 @@ static SColumn* createColumn(int32_t blockId, int32_t slotId, int32_t colId, SDa return pCol; } +void createExprFromTargetNode(SExprInfo* pExp, STargetNode* pTargetNode) { + pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); + pExp->pExpr->_function.num = 1; + pExp->pExpr->_function.functionId = -1; + + int32_t type = nodeType(pTargetNode->pExpr); + // it is a project query, or group by column + if (type == QUERY_NODE_COLUMN) { + pExp->pExpr->nodeType = QUERY_NODE_COLUMN; + SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pColNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pColNode->colName); + pExp->base.pParam[0].pCol = + createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; + } else if (type == QUERY_NODE_VALUE) { + pExp->pExpr->nodeType = QUERY_NODE_VALUE; + SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pValNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pValNode->node.aliasName); + pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; + nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); + } else if (type == QUERY_NODE_FUNCTION) { + pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; + SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; + + SDataType* pType = &pFuncNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pFuncNode->node.aliasName); + + pExp->pExpr->_function.functionId = pFuncNode->funcId; + pExp->pExpr->_function.pFunctNode = pFuncNode; + + strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, + tListLen(pExp->pExpr->_function.functionName)); +#if 1 + // todo refactor: add the parameter for tbname function + if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { + pFuncNode->pParameterList = nodesMakeList(); + ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); + SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { // todo handle error + } else { + res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; + nodesListAppend(pFuncNode->pParameterList, (SNode*)res); + } + } +#endif + + int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); + + pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); + pExp->base.numOfParams = numOfParam; + + for (int32_t j = 0; j < numOfParam; ++j) { + SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); + if (p1->type == QUERY_NODE_COLUMN) { + SColumnNode* pcn = (SColumnNode*)p1; + + pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; + pExp->base.pParam[j].pCol = + createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); + } else if (p1->type == QUERY_NODE_VALUE) { + SValueNode* pvn = (SValueNode*)p1; + pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; + nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); + } + } + } else if (type == QUERY_NODE_OPERATOR) { + pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; + SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; + + pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); + pExp->base.numOfParams = 1; + + SDataType* pType = &pNode->node.resType; + pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, + pType->precision, pNode->node.aliasName); + pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; + } else { + ASSERT(0); + } +} + SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* numOfExprs) { int32_t numOfFuncs = LIST_LENGTH(pNodeList); int32_t numOfGroupKeys = 0; @@ -1033,98 +1145,7 @@ SExprInfo* createExprInfo(SNodeList* pNodeList, SNodeList* pGroupKeys, int32_t* } SExprInfo* pExp = &pExprs[i]; - - pExp->pExpr = taosMemoryCalloc(1, sizeof(tExprNode)); - pExp->pExpr->_function.num = 1; - pExp->pExpr->_function.functionId = -1; - - int32_t type = nodeType(pTargetNode->pExpr); - // it is a project query, or group by column - if (type == QUERY_NODE_COLUMN) { - pExp->pExpr->nodeType = QUERY_NODE_COLUMN; - SColumnNode* pColNode = (SColumnNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pColNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pColNode->colName); - pExp->base.pParam[0].pCol = - createColumn(pColNode->dataBlockId, pColNode->slotId, pColNode->colId, pType, pColNode->colType); - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_COLUMN; - } else if (type == QUERY_NODE_VALUE) { - pExp->pExpr->nodeType = QUERY_NODE_VALUE; - SValueNode* pValNode = (SValueNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pValNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pValNode->node.aliasName); - pExp->base.pParam[0].type = FUNC_PARAM_TYPE_VALUE; - nodesValueNodeToVariant(pValNode, &pExp->base.pParam[0].param); - } else if (type == QUERY_NODE_FUNCTION) { - pExp->pExpr->nodeType = QUERY_NODE_FUNCTION; - SFunctionNode* pFuncNode = (SFunctionNode*)pTargetNode->pExpr; - - SDataType* pType = &pFuncNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pFuncNode->node.aliasName); - - pExp->pExpr->_function.functionId = pFuncNode->funcId; - pExp->pExpr->_function.pFunctNode = pFuncNode; - - strncpy(pExp->pExpr->_function.functionName, pFuncNode->functionName, - tListLen(pExp->pExpr->_function.functionName)); -#if 1 - // todo refactor: add the parameter for tbname function - if (!pFuncNode->pParameterList && (strcmp(pExp->pExpr->_function.functionName, "tbname") == 0)) { - pFuncNode->pParameterList = nodesMakeList(); - ASSERT(LIST_LENGTH(pFuncNode->pParameterList) == 0); - SValueNode* res = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); - if (NULL == res) { // todo handle error - } else { - res->node.resType = (SDataType){.bytes = sizeof(int64_t), .type = TSDB_DATA_TYPE_BIGINT}; - nodesListAppend(pFuncNode->pParameterList, (SNode*)res); - } - } -#endif - - int32_t numOfParam = LIST_LENGTH(pFuncNode->pParameterList); - - pExp->base.pParam = taosMemoryCalloc(numOfParam, sizeof(SFunctParam)); - pExp->base.numOfParams = numOfParam; - - for (int32_t j = 0; j < numOfParam; ++j) { - SNode* p1 = nodesListGetNode(pFuncNode->pParameterList, j); - if (p1->type == QUERY_NODE_COLUMN) { - SColumnNode* pcn = (SColumnNode*)p1; - - pExp->base.pParam[j].type = FUNC_PARAM_TYPE_COLUMN; - pExp->base.pParam[j].pCol = - createColumn(pcn->dataBlockId, pcn->slotId, pcn->colId, &pcn->node.resType, pcn->colType); - } else if (p1->type == QUERY_NODE_VALUE) { - SValueNode* pvn = (SValueNode*)p1; - pExp->base.pParam[j].type = FUNC_PARAM_TYPE_VALUE; - nodesValueNodeToVariant(pvn, &pExp->base.pParam[j].param); - } - } - } else if (type == QUERY_NODE_OPERATOR) { - pExp->pExpr->nodeType = QUERY_NODE_OPERATOR; - SOperatorNode* pNode = (SOperatorNode*)pTargetNode->pExpr; - - pExp->base.pParam = taosMemoryCalloc(1, sizeof(SFunctParam)); - pExp->base.numOfParams = 1; - - SDataType* pType = &pNode->node.resType; - pExp->base.resSchema = createResSchema(pType->type, pType->bytes, pTargetNode->slotId, pType->scale, - pType->precision, pNode->node.aliasName); - pExp->pExpr->_optrRoot.pRootNode = pTargetNode->pExpr; - } else { - ASSERT(0); - } + createExprFromTargetNode(pExp, pTargetNode); } return pExprs; @@ -1300,7 +1321,6 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - pCond->schemaVersion = -1; // pCond->type = pTableScanNode->scanFlag; int32_t j = 0; diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index d0f0a98e94904377df93298a64be1ecd7b23cf7b..67c2542d9ada3a2ee52e7c438ade1f949799498e 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -52,7 +52,11 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu // TODO: if a block was set but not consumed, // prevent setting a different type of block pInfo->validBlockIndex = 0; - taosArrayClear(pInfo->pBlockLists); + if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { + taosArrayClearP(pInfo->pBlockLists, taosMemoryFree); + } else { + taosArrayClear(pInfo->pBlockLists); + } if (type == STREAM_INPUT__MERGED_SUBMIT) { // ASSERT(numOfBlocks > 1); @@ -93,6 +97,8 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } +static FORCE_INLINE void streamInputBlockDataDestory(void* pBlock) { blockDataDestroy((SSDataBlock*)pBlock); } + void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if (!pTaskInfo || !pTaskInfo->pRoot || pTaskInfo->pRoot->numOfDownstream <= 0) { @@ -103,11 +109,7 @@ void tdCleanupStreamInputDataBlock(qTaskInfo_t tinfo) { if (pOptrInfo->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { SStreamScanInfo* pInfo = pOptrInfo->info; if (pInfo->blockType == STREAM_INPUT__DATA_BLOCK) { - for (int32_t i = 0; i < taosArrayGetSize(pInfo->pBlockLists); ++i) { - SSDataBlock* p = *(SSDataBlock**)taosArrayGet(pInfo->pBlockLists, i); - taosArrayDestroy(p->pDataBlock); - taosMemoryFreeClear(p); - } + taosArrayClearP(pInfo->pBlockLists, streamInputBlockDataDestory); } else { ASSERT(0); } @@ -733,7 +735,6 @@ int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* s pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = sContext->snapVersion; - pCond->schemaVersion = sContext->snapVersion; for (int32_t i = 0; i < pCond->numOfCols; ++i) { pCond->colList[i].type = mtInfo.schema->pSchema[i].type; @@ -815,7 +816,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } } - // TODO after dropping table, table may be not found + // TODO after dropping table, table may not found ASSERT(found); if (pTableScanInfo->dataReader == NULL) { diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index dd00684ed1f3b983be5beb9cc23cf25169e07a01..90560a028bcd14ddf6b80a79d501cd80750e769e 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -132,8 +132,6 @@ SOperatorFpSet createOperatorFpSet(__optr_open_fn_t openFn, __optr_fn_t nextFn, return fpSet; } -void operatorDummyCloseFn(void* param, int32_t numOfCols) {} - static int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, SDiskbasedBuf* pBuf, SGroupResInfo* pGroupResInfo); @@ -263,8 +261,6 @@ SResultRow* doSetResultOutBufByKey(SDiskbasedBuf* pResultBuf, SResultRowInfo* pR ASSERT(pSup->resultRowSize > 0); pResult = getNewResultRow(pResultBuf, &pSup->currentPageId, pSup->resultRowSize); - initResultRow(pResult); - // add a new result set for a new group SResultRowPosition pos = {.pageId = pResult->pageId, .offset = pResult->offset}; tSimpleHashPut(pSup->pResultRowHashTable, pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes), &pos, @@ -817,13 +813,6 @@ void setBlockSMAInfo(SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, SSDataBlock* pB } else { pInput->colDataAggIsSet = false; } - - // set the statistics data for primary time stamp column - // if (pCtx->functionId == FUNCTION_SPREAD && pColumn->colId == PRIMARYKEY_TIMESTAMP_COL_ID) { - // pCtx->isAggSet = true; - // pCtx->agg.min = pBlock->info.window.skey; - // pCtx->agg.max = pBlock->info.window.ekey; - // } } bool isTaskKilled(SExecTaskInfo* pTaskInfo) { @@ -860,146 +849,6 @@ STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int return win; } -#if 0 -static int32_t updateBlockLoadStatus(STaskAttr* pQuery, int32_t status) { - - bool hasFirstLastFunc = false; - bool hasOtherFunc = false; - - if (status == BLK_DATA_DATA_LOAD || status == BLK_DATA_FILTEROUT) { - return status; - } - - for (int32_t i = 0; i < pQuery->numOfOutput; ++i) { - int32_t functionId = getExprFunctionId(&pQuery->pExpr1[i]); - - if (functionId == FUNCTION_TS || functionId == FUNCTION_TS_DUMMY || functionId == FUNCTION_TAG || - functionId == FUNCTION_TAG_DUMMY) { - continue; - } - - if (functionId == FUNCTION_FIRST_DST || functionId == FUNCTION_LAST_DST) { - hasFirstLastFunc = true; - } else { - hasOtherFunc = true; - } - - } - - if (hasFirstLastFunc && status == BLK_DATA_NOT_LOAD) { - if (!hasOtherFunc) { - return BLK_DATA_FILTEROUT; - } else { - return BLK_DATA_DATA_LOAD; - } - } - - return status; -} - -#endif - -// static void updateDataCheckOrder(SQInfo *pQInfo, SQueryTableReq* pQueryMsg, bool stableQuery) { -// STaskAttr* pQueryAttr = pQInfo->runtimeEnv.pQueryAttr; -// -// // in case of point-interpolation query, use asc order scan -// char msg[] = "QInfo:0x%"PRIx64" scan order changed for %s query, old:%d, new:%d, qrange exchanged, old qrange:%" -// PRId64 -// "-%" PRId64 ", new qrange:%" PRId64 "-%" PRId64; -// -// // todo handle the case the the order irrelevant query type mixed up with order critical query type -// // descending order query for last_row query -// if (isFirstLastRowQuery(pQueryAttr)) { -// //qDebug("QInfo:0x%"PRIx64" scan order changed for last_row query, old:%d, new:%d", pQInfo->qId, -// pQueryAttr->order.order, TSDB_ORDER_ASC); -// -// pQueryAttr->order.order = TSDB_ORDER_ASC; -// if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// } -// -// pQueryAttr->needReverseScan = false; -// return; -// } -// -// if (pQueryAttr->groupbyColumn && pQueryAttr->order.order == TSDB_ORDER_DESC) { -// pQueryAttr->order.order = TSDB_ORDER_ASC; -// if (pQueryAttr->window.skey > pQueryAttr->window.ekey) { -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// } -// -// pQueryAttr->needReverseScan = false; -// doUpdateLastKey(pQueryAttr); -// return; -// } -// -// if (pQueryAttr->pointInterpQuery && pQueryAttr->interval.interval == 0) { -// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { -// //qDebug(msg, pQInfo->qId, "interp", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, -// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); TSWAP(pQueryAttr->window.skey, -// pQueryAttr->window.ekey, TSKEY); -// } -// -// pQueryAttr->order.order = TSDB_ORDER_ASC; -// return; -// } -// -// if (pQueryAttr->interval.interval == 0) { -// if (onlyFirstQuery(pQueryAttr)) { -// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { -// //qDebug(msg, pQInfo->qId, "only-first", pQueryAttr->order.order, TSDB_ORDER_ASC, pQueryAttr->window.skey, -//// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); -// -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// doUpdateLastKey(pQueryAttr); -// } -// -// pQueryAttr->order.order = TSDB_ORDER_ASC; -// pQueryAttr->needReverseScan = false; -// } else if (onlyLastQuery(pQueryAttr) && notContainSessionOrStateWindow(pQueryAttr)) { -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// //qDebug(msg, pQInfo->qId, "only-last", pQueryAttr->order.order, TSDB_ORDER_DESC, pQueryAttr->window.skey, -//// pQueryAttr->window.ekey, pQueryAttr->window.ekey, pQueryAttr->window.skey); -// -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// doUpdateLastKey(pQueryAttr); -// } -// -// pQueryAttr->order.order = TSDB_ORDER_DESC; -// pQueryAttr->needReverseScan = false; -// } -// -// } else { // interval query -// if (stableQuery) { -// if (onlyFirstQuery(pQueryAttr)) { -// if (!QUERY_IS_ASC_QUERY(pQueryAttr)) { -// //qDebug(msg, pQInfo->qId, "only-first stable", pQueryAttr->order.order, TSDB_ORDER_ASC, -//// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, -/// pQueryAttr->window.skey); -// -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// doUpdateLastKey(pQueryAttr); -// } -// -// pQueryAttr->order.order = TSDB_ORDER_ASC; -// pQueryAttr->needReverseScan = false; -// } else if (onlyLastQuery(pQueryAttr)) { -// if (QUERY_IS_ASC_QUERY(pQueryAttr)) { -// //qDebug(msg, pQInfo->qId, "only-last stable", pQueryAttr->order.order, TSDB_ORDER_DESC, -//// pQueryAttr->window.skey, pQueryAttr->window.ekey, pQueryAttr->window.ekey, -/// pQueryAttr->window.skey); -// -// TSWAP(pQueryAttr->window.skey, pQueryAttr->window.ekey); -// doUpdateLastKey(pQueryAttr); -// } -// -// pQueryAttr->order.order = TSDB_ORDER_DESC; -// pQueryAttr->needReverseScan = false; -// } -// } -// } -//} - #if 0 static bool overlapWithTimeWindow(STaskAttr* pQueryAttr, SDataBlockInfo* pBlockInfo) { STimeWindow w = {0}; @@ -1225,24 +1074,6 @@ static void updateTableQueryInfoForReverseScan(STableQueryInfo* pTableQueryInfo) if (pTableQueryInfo == NULL) { return; } - - // TSWAP(pTableQueryInfo->win.skey, pTableQueryInfo->win.ekey); - // pTableQueryInfo->lastKey = pTableQueryInfo->win.skey; - - // SWITCH_ORDER(pTableQueryInfo->cur.order); - // pTableQueryInfo->cur.vgroupIndex = -1; - - // set the index to be the end slot of result rows array - // SResultRowInfo* pResultRowInfo = &pTableQueryInfo->resInfo; - // if (pResultRowInfo->size > 0) { - // pResultRowInfo->curPos = pResultRowInfo->size - 1; - // } else { - // pResultRowInfo->curPos = -1; - // } -} - -void initResultRow(SResultRow* pResultRow) { - // pResultRow->pEntryInfo = (struct SResultRowEntryInfo*)((char*)pResultRow + sizeof(SResultRow)); } void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { @@ -1255,15 +1086,6 @@ void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status) { } } -void destroyTableQueryInfoImpl(STableQueryInfo* pTableQueryInfo) { - if (pTableQueryInfo == NULL) { - return; - } - - // taosVariantDestroy(&pTableQueryInfo->tag); - // cleanupResultRowInfo(&pTableQueryInfo->resInfo); -} - void setResultRowInitCtx(SResultRow* pResult, SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t* rowEntryInfoOffset) { bool init = false; for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1445,33 +1267,12 @@ static void doUpdateNumOfRows(SqlFunctionCtx* pCtx, SResultRow* pRow, int32_t nu } } -// todo extract method with copytoSSDataBlock -int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, - SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, - const int32_t* rowCellOffset, SSDataBlock* pBlock, - SExecTaskInfo* pTaskInfo) { - SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); - SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); - - doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowCellOffset); - if (pRow->numOfRows == 0) { - releaseBufPage(pBuf, page); - return 0; - } - - while (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { - int32_t code = blockDataEnsureCapacity(pBlock, pBlock->info.capacity * 1.25); - if (TAOS_FAILED(code)) { - releaseBufPage(pBuf, page); - qError("%s ensure result data capacity failed, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, code); - } - } - +static void doCopyResultToDataBlock(SExprInfo* pExprInfo, int32_t numOfExprs, SResultRow* pRow, SqlFunctionCtx* pCtx, + SSDataBlock* pBlock, const int32_t* rowEntryOffset, SExecTaskInfo* pTaskInfo) { for (int32_t j = 0; j < numOfExprs; ++j) { int32_t slotId = pExprInfo[j].base.resSchema.slotId; - pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowCellOffset); + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); if (pCtx[j].fpSet.finalize) { int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); if (TAOS_FAILED(code)) { @@ -1479,7 +1280,7 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi T_LONG_JMP(pTaskInfo->env, code); } } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { - // do nothing, todo refactor + // do nothing } else { // expand the result into multiple rows. E.g., _wstart, top(k, 20) // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. @@ -1490,10 +1291,40 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi } } } +} + +// todo refactor. SResultRow has direct pointer in miainfo +int32_t finalizeResultRows(SDiskbasedBuf* pBuf, SResultRowPosition* resultRowPosition, SExprSupp* pSup, + SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo) { + SFilePage* page = getBufPage(pBuf, resultRowPosition->pageId); + SResultRow* pRow = (SResultRow*)((char*)page + resultRowPosition->offset); + + SqlFunctionCtx* pCtx = pSup->pCtx; + SExprInfo* pExprInfo = pSup->pExprInfo; + const int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + + doUpdateNumOfRows(pCtx, pRow, pSup->numOfExprs, rowEntryOffset); + if (pRow->numOfRows == 0) { + releaseBufPage(pBuf, page); + return 0; + } + + int32_t size = pBlock->info.capacity; + while (pBlock->info.rows + pRow->numOfRows > size) { + size = size * 1.25; + } + + int32_t code = blockDataEnsureCapacity(pBlock, size); + if (TAOS_FAILED(code)) { + releaseBufPage(pBuf, page); + qError("%s ensure result data capacity failed, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); + } + + doCopyResultToDataBlock(pExprInfo, pSup->numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; - return 0; } @@ -1538,32 +1369,7 @@ int32_t doCopyToSDataBlock(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprS } pGroupResInfo->index += 1; - - for (int32_t j = 0; j < numOfExprs; ++j) { - int32_t slotId = pExprInfo[j].base.resSchema.slotId; - - pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); - if (pCtx[j].fpSet.finalize) { -#ifdef BUF_PAGE_DEBUG - qDebug("\npage_finalize %d", numOfExprs); -#endif - int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); - if (TAOS_FAILED(code)) { - qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); - T_LONG_JMP(pTaskInfo->env, code); - } - } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { - // do nothing, todo refactor - } else { - // expand the result into multiple rows. E.g., _wstart, top(k, 20) - // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); - char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); - for (int32_t k = 0; k < pRow->numOfRows; ++k) { - colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); - } - } - } + doCopyResultToDataBlock(pExprInfo, numOfExprs, pRow, pCtx, pBlock, rowEntryOffset, pTaskInfo); releaseBufPage(pBuf, page); pBlock->info.rows += pRow->numOfRows; @@ -1903,22 +1709,6 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t static void doDestroyTableList(STableListInfo* pTableqinfoList); -static void doTableQueryInfoTimeWindowCheck(SExecTaskInfo* pTaskInfo, STableQueryInfo* pTableQueryInfo, int32_t order) { -#if 0 - if (order == TSDB_ORDER_ASC) { - assert( - (pTableQueryInfo->win.skey <= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey >= pTaskInfo->window.skey) && - (pTableQueryInfo->win.skey >= pTaskInfo->window.skey && pTableQueryInfo->win.ekey <= pTaskInfo->window.ekey)); - } else { - assert( - (pTableQueryInfo->win.skey >= pTableQueryInfo->win.ekey) && - (pTableQueryInfo->lastKey <= pTaskInfo->window.skey) && - (pTableQueryInfo->win.skey <= pTaskInfo->window.skey && pTableQueryInfo->win.ekey >= pTaskInfo->window.ekey)); - } -#endif -} - typedef struct SFetchRspHandleWrapper { uint32_t exchangeId; int32_t sourceIndex; @@ -2049,13 +1839,11 @@ void updateLoadRemoteInfo(SLoadRemoteDataInfo* pInfo, int32_t numOfRows, int32_t pOperator->resultInfo.totalRows += numOfRows; } -int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, int32_t numOfOutput, SArray* pColList, - char** pNextStart) { +int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, char* pData, SArray* pColList, char** pNextStart) { if (pColList == NULL) { // data from other sources blockDataCleanup(pRes); *pNextStart = (char*)blockDecode(pRes, pData); } else { // extract data according to pColList - ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; int32_t numOfCols = htonl(*(int32_t*)pStart); @@ -2153,7 +1941,7 @@ static void concurrentlyLoadRemoteDataImpl(SOperatorInfo* pOperator, SExchangeIn char* pStart = pRetrieveRsp->data; while (index++ < pRetrieveRsp->numOfBlocks) { SSDataBlock* pb = createOneDataBlock(pExchangeInfo->pDummyBlock, false); - code = extractDataBlockFromFetchRsp(pb, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); + code = extractDataBlockFromFetchRsp(pb, pStart, NULL, &pStart); if (code != 0) { taosMemoryFreeClear(pDataInfo->pRsp); goto _error; @@ -2278,7 +2066,7 @@ static int32_t seqLoadRemoteData(SOperatorInfo* pOperator) { SRetrieveTableRsp* pRetrieveRsp = pDataInfo->pRsp; char* pStart = pRetrieveRsp->data; - int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, pRetrieveRsp->numOfCols, NULL, &pStart); + int32_t code = extractDataBlockFromFetchRsp(NULL, pStart, NULL, &pStart); if (pRsp->completed == 1) { qDebug("%s fetch msg rsp from vgId:%d, taskId:0x%" PRIx64 " execId:%d numOfRows:%d, rowsOfSource:%" PRIu64 @@ -2492,21 +2280,6 @@ _error: static int32_t doInitAggInfoSup(SAggSupporter* pAggSup, SqlFunctionCtx* pCtx, int32_t numOfOutput, size_t keyBufSize, const char* pKey); -static void destroySortedMergeOperatorInfo(void* param, int32_t numOfOutput) { - SSortedMergeOperatorInfo* pInfo = (SSortedMergeOperatorInfo*)param; - taosArrayDestroy(pInfo->pSortInfo); - taosArrayDestroy(pInfo->groupInfo); - - if (pInfo->pSortHandle != NULL) { - tsortDestroySortHandle(pInfo->pSortHandle); - } - - blockDataDestroy(pInfo->binfo.pRes); - cleanupAggSup(&pInfo->aggSup); - - taosMemoryFreeClear(param); -} - static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int32_t rowIndex) { size_t size = taosArrayGetSize(groupInfo); if (size == 0) { @@ -2542,41 +2315,6 @@ static bool needToMerge(SSDataBlock* pBlock, SArray* groupInfo, char** buf, int3 return 0; } -static void doMergeResultImpl(SSortedMergeOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfExpr, - int32_t rowIndex) { - for (int32_t j = 0; j < numOfExpr; ++j) { // TODO set row index - // pCtx[j].startRow = rowIndex; - } - - for (int32_t j = 0; j < numOfExpr; ++j) { - int32_t functionId = pCtx[j].functionId; - // pCtx[j].fpSet->addInput(&pCtx[j]); - - // if (functionId < 0) { - // SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - // doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_MERGE); - // } else { - // assert(!TSDB_FUNC_IS_SCALAR(functionId)); - // aAggs[functionId].mergeFunc(&pCtx[j]); - // } - } -} - -static void doFinalizeResultImpl(SqlFunctionCtx* pCtx, int32_t numOfExpr) { - for (int32_t j = 0; j < numOfExpr; ++j) { - int32_t functionId = pCtx[j].functionId; - // if (functionId == FUNC_TAG_DUMMY || functionId == FUNC_TS_DUMMY) { - // continue; - // } - - // if (functionId < 0) { - // SUdfInfo* pUdfInfo = taosArrayGet(pInfo->udfInfo, -1 * functionId - 1); - // doInvokeUdf(pUdfInfo, &pCtx[j], 0, TSDB_UDF_FUNC_FINALIZE); - // } else { - // pCtx[j].fpSet.finalize(&pCtx[j]); - } -} - static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock* pBlock, int32_t rowIndex) { int32_t size = (int32_t)taosArrayGetSize(pColumnList); @@ -2591,210 +2329,6 @@ static bool saveCurrentTuple(char** rowColData, SArray* pColumnList, SSDataBlock return true; } -static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock* pBlock) { - SSortedMergeOperatorInfo* pInfo = pOperator->info; - - SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - - for (int32_t i = 0; i < pBlock->info.rows; ++i) { - if (!pInfo->hasGroupVal) { - ASSERT(i == 0); - doMergeResultImpl(pInfo, pCtx, numOfExpr, i); - pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i); - } else { - if (needToMerge(pBlock, pInfo->groupInfo, pInfo->groupVal, i)) { - doMergeResultImpl(pInfo, pCtx, numOfExpr, i); - } else { - doFinalizeResultImpl(pCtx, numOfExpr); - int32_t numOfRows = getNumOfResult(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs, NULL); - // setTagValueForMultipleRows(pCtx, pOperator->exprSupp.numOfExprs, numOfRows); - - // TODO check for available buffer; - - // next group info data - pInfo->binfo.pRes->info.rows += numOfRows; - for (int32_t j = 0; j < numOfExpr; ++j) { - if (pCtx[j].functionId < 0) { - continue; - } - - pCtx[j].fpSet.process(&pCtx[j]); - } - - doMergeResultImpl(pInfo, pCtx, numOfExpr, i); - pInfo->hasGroupVal = saveCurrentTuple(pInfo->groupVal, pInfo->groupInfo, pBlock, i); - } - } - } -} - -static SSDataBlock* doMerge(SOperatorInfo* pOperator) { - SSortedMergeOperatorInfo* pInfo = pOperator->info; - SSortHandle* pHandle = pInfo->pSortHandle; - - SSDataBlock* pDataBlock = createOneDataBlock(pInfo->binfo.pRes, false); - blockDataEnsureCapacity(pDataBlock, pOperator->resultInfo.capacity); - - while (1) { - blockDataCleanup(pDataBlock); - while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { - break; - } - - // build datablock for merge for one group - appendOneRowToDataBlock(pDataBlock, pTupleHandle); - if (pDataBlock->info.rows >= pOperator->resultInfo.capacity) { - break; - } - } - - if (pDataBlock->info.rows == 0) { - break; - } - - setInputDataBlock(pOperator, pOperator->exprSupp.pCtx, pDataBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - // updateOutputBuf(&pInfo->binfo, &pAggInfo->bufCapacity, pBlock->info.rows * pAggInfo->resultRowFactor, - // pOperator->pRuntimeEnv, true); - doMergeImpl(pOperator, pOperator->exprSupp.numOfExprs, pDataBlock); - // flush to tuple store, and after all data have been handled, return to upstream node or sink node - } - - doFinalizeResultImpl(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs); - int32_t numOfRows = getNumOfResult(pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs, NULL); - // setTagValueForMultipleRows(pCtx, pOperator->exprSupp.numOfExprs, numOfRows); - - // TODO check for available buffer; - - // next group info data - pInfo->binfo.pRes->info.rows += numOfRows; - return (pInfo->binfo.pRes->info.rows > 0) ? pInfo->binfo.pRes : NULL; -} - -SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, - SArray* pColMatchInfo, SSortedMergeOperatorInfo* pInfo) { - blockDataCleanup(pDataBlock); - - SSDataBlock* p = tsortGetSortedDataBlock(pHandle); - if (p == NULL) { - return NULL; - } - - blockDataEnsureCapacity(p, capacity); - - while (1) { - STupleHandle* pTupleHandle = tsortNextTuple(pHandle); - if (pTupleHandle == NULL) { - break; - } - - appendOneRowToDataBlock(p, pTupleHandle); - if (p->info.rows >= capacity) { - break; - } - } - - if (p->info.rows > 0) { - int32_t numOfCols = taosArrayGetSize(pColMatchInfo); - for (int32_t i = 0; i < numOfCols; ++i) { - SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); - ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID); - - SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); - SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); - } - - pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; - } - - blockDataDestroy(p); - return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; -} - -static SSDataBlock* doSortedMerge(SOperatorInfo* pOperator) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SSortedMergeOperatorInfo* pInfo = pOperator->info; - if (pOperator->status == OP_RES_TO_RETURN) { - return getSortedMergeBlockData(pInfo->pSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, NULL, pInfo); - } - - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, - pInfo->binfo.pRes, "GET_TASKID(pTaskInfo)"); - - tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - - for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { - SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); - ps->param = pOperator->pDownstream[i]; - tsortAddSource(pInfo->pSortHandle, ps); - } - - int32_t code = tsortOpen(pInfo->pSortHandle); - if (code != TSDB_CODE_SUCCESS) { - T_LONG_JMP(pTaskInfo->env, terrno); - } - - pOperator->status = OP_RES_TO_RETURN; - return doMerge(pOperator); -} - -static int32_t initGroupCol(SExprInfo* pExprInfo, int32_t numOfCols, SArray* pGroupInfo, - SSortedMergeOperatorInfo* pInfo) { - if (pGroupInfo == NULL || taosArrayGetSize(pGroupInfo) == 0) { - return 0; - } - - int32_t len = 0; - SArray* plist = taosArrayInit(3, sizeof(SColumn)); - pInfo->groupInfo = taosArrayInit(3, sizeof(int32_t)); - - if (plist == NULL || pInfo->groupInfo == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - size_t numOfGroupCol = taosArrayGetSize(pInfo->groupInfo); - for (int32_t i = 0; i < numOfGroupCol; ++i) { - SColumn* pCol = taosArrayGet(pGroupInfo, i); - for (int32_t j = 0; j < numOfCols; ++j) { - SExprInfo* pe = &pExprInfo[j]; - if (pe->base.resSchema.slotId == pCol->colId) { - taosArrayPush(plist, pCol); - taosArrayPush(pInfo->groupInfo, &j); - len += pCol->bytes; - break; - } - } - } - - ASSERT(taosArrayGetSize(pGroupInfo) == taosArrayGetSize(plist)); - - pInfo->groupVal = taosMemoryCalloc(1, (POINTER_BYTES * numOfGroupCol + len)); - if (pInfo->groupVal == NULL) { - taosArrayDestroy(plist); - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t offset = 0; - char* start = (char*)(pInfo->groupVal + (POINTER_BYTES * numOfGroupCol)); - for (int32_t i = 0; i < numOfGroupCol; ++i) { - pInfo->groupVal[i] = start + offset; - SColumn* pCol = taosArrayGet(plist, i); - offset += pCol->bytes; - } - - taosArrayDestroy(plist); - - return TSDB_CODE_SUCCESS; -} - int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scanFlag) { // todo add more information about exchange operation int32_t type = pOperator->operatorType; @@ -3029,7 +2563,6 @@ int32_t aggDecodeResultRow(SOperatorInfo* pOperator, char* result) { resultRow->offset = pOffset; offset += valueLen; - initResultRow(resultRow); pInfo->resultRowInfo.cur = (SResultRowPosition){.pageId = resultRow->pageId, .offset = resultRow->offset}; // releaseBufPage(pSup->pResultBuf, getBufPage(pSup->pResultBuf, pageId)); } @@ -3528,13 +3061,6 @@ void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -void destroyBasicOperatorInfo(void* param, int32_t numOfOutput) { - SOptrBasicInfo* pInfo = (SOptrBasicInfo*)param; - cleanupBasicInfo(pInfo); - - taosMemoryFreeClear(param); -} - static void freeItem(void* pItem) { void** p = pItem; if (*p != NULL) { @@ -3625,6 +3151,44 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } } +static bool isWstartColumnExist(SFillOperatorInfo* pInfo) { + if (pInfo->numOfNotFillExpr == 0) { + return false; + } + for (int32_t i = 0; i < pInfo->numOfNotFillExpr; ++i) { + SExprInfo* exprInfo = pInfo->pNotFillExprInfo + i; + if (exprInfo->pExpr->nodeType == QUERY_NODE_COLUMN && exprInfo->base.numOfParams == 1 && + exprInfo->base.pParam[0].pCol->colType == COLUMN_TYPE_WINDOW_START) { + return true; + } + } + return false; +} + +static int32_t createWStartTsAsNotFillExpr(SFillOperatorInfo* pInfo, SFillPhysiNode* pPhyFillNode) { + bool wstartExist = isWstartColumnExist(pInfo); + if (wstartExist == false) { + if (pPhyFillNode->pWStartTs->type != QUERY_NODE_TARGET) { + qError("pWStartTs of fill physical node is not a target node"); + return TSDB_CODE_QRY_SYS_ERROR; + } + + SExprInfo* notFillExprs = + taosMemoryRealloc(pInfo->pNotFillExprInfo, (pInfo->numOfNotFillExpr + 1) * sizeof(SExprInfo)); + if (notFillExprs == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + createExprFromTargetNode(notFillExprs + pInfo->numOfNotFillExpr, (STargetNode*)pPhyFillNode->pWStartTs); + + ++pInfo->numOfNotFillExpr; + pInfo->pNotFillExprInfo = notFillExprs; + return TSDB_CODE_SUCCESS; + } + + return TSDB_CODE_SUCCESS; +} + SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); @@ -3636,7 +3200,10 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SSDataBlock* pResBlock = createResDataBlock(pPhyFillNode->node.pOutputDataBlockDesc); SExprInfo* pExprInfo = createExprInfo(pPhyFillNode->pFillExprs, NULL, &pInfo->numOfExpr); pInfo->pNotFillExprInfo = createExprInfo(pPhyFillNode->pNotFillExprs, NULL, &pInfo->numOfNotFillExpr); - + int32_t code = createWStartTsAsNotFillExpr(pInfo, pPhyFillNode); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } SInterval* pInterval = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == downstream->operatorType ? &((SMergeAlignedIntervalAggOperatorInfo*)downstream->info)->intervalAggOperatorInfo->interval @@ -3657,9 +3224,9 @@ SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* SArray* pColMatchColInfo = extractColMatchInfo(pPhyFillNode->pFillExprs, pPhyFillNode->node.pOutputDataBlockDesc, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); - int32_t code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, - (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, - pTaskInfo->id.str, pInterval, type, order); + code = initFillInfo(pInfo, pExprInfo, pInfo->numOfExpr, pInfo->pNotFillExprInfo, pInfo->numOfNotFillExpr, + (SNodeListNode*)pPhyFillNode->pValues, pPhyFillNode->timeRange, pResultInfo->capacity, + pTaskInfo->id.str, pInterval, type, order); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -3915,7 +3482,6 @@ static int32_t initTableblockDistQueryCond(uint64_t uid, SQueryTableDataCond* pC pCond->type = TIMEWINDOW_RANGE_CONTAINED; pCond->startVersion = -1; pCond->endVersion = -1; - pCond->schemaVersion = -1; return TSDB_CODE_SUCCESS; } @@ -4001,7 +3567,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; int32_t code = getTableList(pHandle->meta, pHandle->vnode, pScanPhyNode, pTagCond, pTagIndexCond, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { - pTaskInfo->code = terrno; + pTaskInfo->code = code; qError("failed to getTableList, code: %s", tstrerror(code)); return NULL; } @@ -4100,7 +3666,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createAggregateOperatorInfo(ops[0], pExprInfo, num, pResBlock, pAggNode->node.pConditions, pScalarExprInfo, numOfScalarExpr, pAggNode->mergeDataBlock, pTaskInfo); } - } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type || QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + } else if (QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL == type) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); @@ -4125,6 +3691,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pIntervalPhyNode, pTaskInfo, isStream); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL == type) { + pOptr = createStreamIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pIntervalPhyNode, pTaskInfo); @@ -4157,6 +3725,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createStreamFinalSessionAggOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_PARTITION == type) { pOptr = createPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION == type) { + pOptr = createStreamPartitionOperatorInfo(ops[0], (SPartitionPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE == type) { SStateWinodwPhysiNode* pStateNode = (SStateWinodwPhysiNode*)pPhyNode; pOptr = createStatewindowOperatorInfo(ops[0], pStateNode, pTaskInfo); @@ -4595,3 +4165,112 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlF return code; } + +int32_t setOutputBuf(STimeWindow* win, SResultRow** pResult, int64_t tableGroupId, SqlFunctionCtx* pCtx, + int32_t numOfOutput, int32_t* rowEntryInfoOffset, SAggSupporter* pAggSup, + SExecTaskInfo* pTaskInfo) { + SWinKey key = { + .ts = win->skey, + .groupId = tableGroupId, + }; + char* value = NULL; + int32_t size = pAggSup->resultRowSize; + /*if (streamStateGet(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) {*/ + /*value = taosMemoryCalloc(1, size);*/ + /*}*/ + if (streamStateAddIfNotExist(pTaskInfo->streamInfo.pState, &key, (void**)&value, &size) < 0) { + return TSDB_CODE_QRY_OUT_OF_MEMORY; + } + *pResult = (SResultRow*)value; + ASSERT(*pResult); + // set time window for current result + (*pResult)->win = (*win); + setResultRowInitCtx(*pResult, pCtx, numOfOutput, rowEntryInfoOffset); + return TSDB_CODE_SUCCESS; +} + +int32_t releaseOutputBuf(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult) { + streamStateReleaseBuf(pTaskInfo->streamInfo.pState, pKey, pResult); + /*taosMemoryFree((*(void**)pResult));*/ + return TSDB_CODE_SUCCESS; +} + +int32_t saveOutput(SExecTaskInfo* pTaskInfo, SWinKey* pKey, SResultRow* pResult, int32_t resSize) { + streamStatePut(pTaskInfo->streamInfo.pState, pKey, pResult, resSize); + return TSDB_CODE_SUCCESS; +} + +int32_t buildDataBlockFromGroupRes(SExecTaskInfo* pTaskInfo, SSDataBlock* pBlock, SExprSupp* pSup, + SGroupResInfo* pGroupResInfo) { + SExprInfo* pExprInfo = pSup->pExprInfo; + int32_t numOfExprs = pSup->numOfExprs; + int32_t* rowEntryOffset = pSup->rowEntryInfoOffset; + SqlFunctionCtx* pCtx = pSup->pCtx; + + int32_t numOfRows = getNumOfTotalRes(pGroupResInfo); + + for (int32_t i = pGroupResInfo->index; i < numOfRows; i += 1) { + SResKeyPos* pPos = taosArrayGetP(pGroupResInfo->pRows, i); + int32_t size = 0; + void* pVal = NULL; + SWinKey key = { + .ts = *(TSKEY*)pPos->key, + .groupId = pPos->groupId, + }; + int32_t code = streamStateGet(pTaskInfo->streamInfo.pState, &key, &pVal, &size); + ASSERT(code == 0); + SResultRow* pRow = (SResultRow*)pVal; + doUpdateNumOfRows(pCtx, pRow, numOfExprs, rowEntryOffset); + // no results, continue to check the next one + if (pRow->numOfRows == 0) { + pGroupResInfo->index += 1; + releaseOutputBuf(pTaskInfo, &key, pRow); + continue; + } + + if (pBlock->info.groupId == 0) { + pBlock->info.groupId = pPos->groupId; + } else { + // current value belongs to different group, it can't be packed into one datablock + if (pBlock->info.groupId != pPos->groupId) { + releaseOutputBuf(pTaskInfo, &key, pRow); + break; + } + } + + if (pBlock->info.rows + pRow->numOfRows > pBlock->info.capacity) { + ASSERT(pBlock->info.rows > 0); + releaseOutputBuf(pTaskInfo, &key, pRow); + break; + } + + pGroupResInfo->index += 1; + + for (int32_t j = 0; j < numOfExprs; ++j) { + int32_t slotId = pExprInfo[j].base.resSchema.slotId; + + pCtx[j].resultInfo = getResultEntryInfo(pRow, j, rowEntryOffset); + if (pCtx[j].fpSet.finalize) { + int32_t code = pCtx[j].fpSet.finalize(&pCtx[j], pBlock); + if (TAOS_FAILED(code)) { + qError("%s build result data block error, code %s", GET_TASKID(pTaskInfo), tstrerror(code)); + T_LONG_JMP(pTaskInfo->env, code); + } + } else if (strcmp(pCtx[j].pExpr->pExpr->_function.functionName, "_select_value") == 0) { + // do nothing, todo refactor + } else { + // expand the result into multiple rows. E.g., _wstart, top(k, 20) + // the _wstart needs to copy to 20 following rows, since the results of top-k expands to 20 different rows. + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); + char* in = GET_ROWCELL_INTERBUF(pCtx[j].resultInfo); + for (int32_t k = 0; k < pRow->numOfRows; ++k) { + colDataAppend(pColInfoData, pBlock->info.rows + k, in, pCtx[j].resultInfo->isNullRes); + } + } + } + releaseOutputBuf(pTaskInfo, &key, pRow); + pBlock->info.rows += pRow->numOfRows; + } + blockDataUpdateTsWindow(pBlock, 0); + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 5d123f723e01d98faa791e612f95235ffef5a04d..5eb6557dbd4093f8c4599eb5d35107c95673e702 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -13,26 +13,26 @@ * along with this program. If not, see . */ -#include "os.h" #include "function.h" +#include "os.h" #include "tname.h" #include "tdatablock.h" #include "tmsg.h" +#include "executorInt.h" #include "executorimpl.h" #include "tcompare.h" #include "thash.h" #include "ttypes.h" -#include "executorInt.h" static void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInfo** pGroupInfo, int32_t len); static int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity); -static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); +static int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, + int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup); static void freeGroupKey(void* param) { - SGroupKeys* pKey = (SGroupKeys*) param; + SGroupKeys* pKey = (SGroupKeys*)param; taosMemoryFree(pKey->pData); } @@ -62,13 +62,13 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** int32_t numOfGroupCols = taosArrayGetSize(pGroupColList); for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupColList, i); - (*keyLen) += pCol->bytes; // actual data + null_flag + (*keyLen) += pCol->bytes; // actual data + null_flag SGroupKeys key = {0}; - key.bytes = pCol->bytes; - key.type = pCol->type; + key.bytes = pCol->bytes; + key.type = pCol->type; key.isNull = false; - key.pData = taosMemoryCalloc(1, pCol->bytes); + key.pData = taosMemoryCalloc(1, pCol->bytes); if (key.pData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -87,7 +87,8 @@ static int32_t initGroupOptrInfo(SArray** pGroupColVals, int32_t* keyLen, char** return TSDB_CODE_SUCCESS; } -static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, int32_t numOfGroupCols) { +static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlock* pBlock, int32_t rowIndex, + int32_t numOfGroupCols) { SColumnDataAgg* pColAgg = NULL; for (int32_t i = 0; i < numOfGroupCols; ++i) { SColumn* pCol = taosArrayGet(pGroupCols, i); @@ -112,7 +113,7 @@ static bool groupKeyCompare(SArray* pGroupCols, SArray* pGroupColVals, SSDataBlo if (pkey->type == TSDB_DATA_TYPE_JSON) { int32_t dataLen = getJsonValueLen(val); - if (memcmp(pkey->pData, val, dataLen) == 0){ + if (memcmp(pkey->pData, val, dataLen) == 0) { continue; } else { return false; @@ -154,7 +155,7 @@ static void recordNewGroupKeys(SArray* pGroupCols, SArray* pGroupColVals, SSData pkey->isNull = false; char* val = colDataGetData(pColInfoData, rowIndex); if (pkey->type == TSDB_DATA_TYPE_JSON) { - if(tTagIsJson(val)){ + if (tTagIsJson(val)) { terrno = TSDB_CODE_QRY_JSON_IN_GROUP_ERROR; return; } @@ -198,13 +199,13 @@ static int32_t buildGroupKeys(void* pKey, const SArray* pGroupColVals) { } } - return (int32_t) (pStart - (char*)pKey); + return (int32_t)(pStart - (char*)pKey); } // assign the group keys or user input constant values if required static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t totalRows, int32_t rowIndex) { for (int32_t i = 0; i < numOfOutput; ++i) { - if (pCtx[i].functionId == -1) { // select count(*),key from t group by key. + if (pCtx[i].functionId == -1) { // select count(*),key from t group by key. SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pCtx[i]); SColumnInfoData* pColInfoData = pCtx[i].input.pData[0]; @@ -221,7 +222,7 @@ static void doAssignGroupKeys(SqlFunctionCtx* pCtx, int32_t numOfOutput, int32_t } else { memcpy(dest, data, pColInfoData->info.bytes); } - } else { // it is a NULL value + } else { // it is a NULL value pEntryInfo->isNullRes = 1; } @@ -275,7 +276,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, + len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -291,9 +293,8 @@ static void doHashGroupbyAgg(SOperatorInfo* pOperator, SSDataBlock* pBlock) { if (num > 0) { len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); - int32_t ret = - setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, len, - pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); + int32_t ret = setGroupResultOutputBuf(pOperator, &(pInfo->binfo), pOperator->exprSupp.numOfExprs, pInfo->keyBuf, + len, pBlock->info.groupId, pInfo->aggSup.pResultBuf, &pInfo->aggSup); if (ret != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -308,7 +309,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { SGroupbyOperatorInfo* pInfo = pOperator->info; SSDataBlock* pRes = pInfo->binfo.pRes; - while(1) { + while (1) { doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); doFilter(pInfo->pCondition, pRes, NULL); @@ -323,7 +324,7 @@ static SSDataBlock* buildGroupResultDataBlock(SOperatorInfo* pOperator) { } pOperator->resultInfo.totalRows += pRes->info.rows; - return (pRes->info.rows == 0)? NULL:pRes; + return (pRes->info.rows == 0) ? NULL : pRes; } static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { @@ -334,7 +335,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SGroupbyOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { return buildGroupResultDataBlock(pOperator); @@ -343,7 +344,7 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { int32_t order = TSDB_ORDER_ASC; int32_t scanFlag = MAIN_SCAN; - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -362,7 +363,8 @@ static SSDataBlock* hashGroupbyAggregate(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -403,8 +405,8 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx goto _error; } - pInfo->pGroupCols = pGroupColList; - pInfo->pCondition = pCondition; + pInfo->pGroupCols = pGroupColList; + pInfo->pCondition = pCondition; int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); if (code != TSDB_CODE_SUCCESS) { @@ -425,14 +427,15 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx initBasicInfo(&pInfo->binfo, pResultBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pOperator->name = "GroupbyAggOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "GroupbyAggOperator"; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; // pOperator->operatorType = OP_Groupby; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashGroupbyAggregate, NULL, NULL, + destroyGroupOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -440,7 +443,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; destroyGroupOperatorInfo(pInfo); taosMemoryFreeClear(pOperator); @@ -448,7 +451,7 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx } static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { -// SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SPartitionOperatorInfo* pInfo = pOperator->info; @@ -457,7 +460,7 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { int32_t len = buildGroupKeys(pInfo->keyBuf, pInfo->pGroupColVals); SDataGroupInfo* pGroupInfo = NULL; - void *pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); + void* pPage = getCurrentDataGroupInfo(pInfo, &pGroupInfo, len); pGroupInfo->numOfRows += 1; @@ -467,32 +470,32 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } // number of rows - int32_t* rows = (int32_t*) pPage; + int32_t* rows = (int32_t*)pPage; size_t numOfCols = pOperator->exprSupp.numOfExprs; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SExprInfo* pExpr = &pOperator->exprSupp.pExprInfo[i]; - int32_t slotId = pExpr->base.pParam[0].pCol->slotId; + int32_t slotId = pExpr->base.pParam[0].pCol->slotId; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, slotId); int32_t bytes = pColInfoData->info.bytes; int32_t startOffset = pInfo->columnOffset[i]; - int32_t* columnLen = NULL; + int32_t* columnLen = NULL; int32_t contentLen = 0; if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { int32_t* offset = (int32_t*)((char*)pPage + startOffset); - columnLen = (int32_t*) ((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); - char* data = (char*)((char*) columnLen + sizeof(int32_t)); + columnLen = (int32_t*)((char*)pPage + startOffset + sizeof(int32_t) * pInfo->rowCapacity); + char* data = (char*)((char*)columnLen + sizeof(int32_t)); if (colDataIsNull_s(pColInfoData, j)) { offset[(*rows)] = -1; contentLen = 0; - } else if(pColInfoData->info.type == TSDB_DATA_TYPE_JSON){ + } else if (pColInfoData->info.type == TSDB_DATA_TYPE_JSON) { offset[*rows] = (*columnLen); - char* src = colDataGetData(pColInfoData, j); + char* src = colDataGetData(pColInfoData, j); int32_t dataLen = getJsonValueLen(src); memcpy(data + (*columnLen), src, dataLen); @@ -511,8 +514,8 @@ static void doHashPartition(SOperatorInfo* pOperator, SSDataBlock* pBlock) { } } else { char* bitmap = (char*)pPage + startOffset; - columnLen = (int32_t*) ((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); - char* data = (char*) columnLen + sizeof(int32_t); + columnLen = (int32_t*)((char*)pPage + startOffset + BitmapLen(pInfo->rowCapacity)); + char* data = (char*)columnLen + sizeof(int32_t); bool isNull = colDataIsNull_f(pColInfoData->nullbitmap, j); if (isNull) { @@ -539,7 +542,7 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf SDataGroupInfo* p = taosHashGet(pInfo->pGroupSet, pInfo->keyBuf, len); void* pPage = NULL; - if (p == NULL) { // it is a new group + if (p == NULL) { // it is a new group SDataGroupInfo gi = {0}; gi.pPageList = taosArrayInit(100, sizeof(int32_t)); taosHashPut(pInfo->pGroupSet, pInfo->keyBuf, len, &gi, sizeof(SDataGroupInfo)); @@ -550,12 +553,12 @@ void* getCurrentDataGroupInfo(const SPartitionOperatorInfo* pInfo, SDataGroupInf pPage = getNewBufPage(pInfo->pBuf, &pageId); taosArrayPush(p->pPageList, &pageId); - *(int32_t *) pPage = 0; + *(int32_t*)pPage = 0; } else { int32_t* curId = taosArrayGetLast(p->pPageList); pPage = getBufPage(pInfo->pBuf, *curId); - int32_t *rows = (int32_t*) pPage; + int32_t* rows = (int32_t*)pPage; if (*rows >= pInfo->rowCapacity) { // release buffer releaseBufPage(pInfo->pBuf, pPage); @@ -585,17 +588,18 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); - offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format + offset[0] = sizeof(int32_t) + + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format - for(int32_t i = 0; i < numOfCols - 1; ++i) { + for (int32_t i = 0; i < numOfCols - 1; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); int32_t bytes = pColInfoData->info.bytes; int32_t payloadLen = bytes * rowCapacity; - + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { // offset segment + content length + payload offset[i + 1] = rowCapacity * sizeof(int32_t) + sizeof(int32_t) + payloadLen + offset[i]; @@ -609,9 +613,9 @@ int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { } static void clearPartitionOperator(SPartitionOperatorInfo* pInfo) { - void *ite = NULL; - while( (ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL ) { - taosArrayDestroy( ((SDataGroupInfo *)ite)->pPageList); + void* ite = NULL; + while ((ite = taosHashIterate(pInfo->pGroupSet, ite)) != NULL) { + taosArrayDestroy(((SDataGroupInfo*)ite)->pPageList); } taosArrayClear(pInfo->sortedGroupArray); clearDiskbasedBuf(pInfo->pBuf); @@ -626,13 +630,14 @@ static int compareDataGroupInfo(const void* group1, const void* group2) { return 0; } - return (pGroupInfo1->groupId < pGroupInfo2->groupId)? -1:1; + return (pGroupInfo1->groupId < pGroupInfo2->groupId) ? -1 : 1; } static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { SPartitionOperatorInfo* pInfo = pOperator->info; - SDataGroupInfo* pGroupInfo = (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; + SDataGroupInfo* pGroupInfo = + (pInfo->groupIndex != -1) ? taosArrayGet(pInfo->sortedGroupArray, pInfo->groupIndex) : NULL; if (pInfo->groupIndex == -1 || pInfo->pageIndex >= taosArrayGetSize(pGroupInfo->pPageList)) { // try next group data ++pInfo->groupIndex; @@ -647,7 +652,7 @@ static SSDataBlock* buildPartitionResult(SOperatorInfo* pOperator) { } int32_t* pageId = taosArrayGet(pGroupInfo->pPageList, pInfo->pageIndex); - void* page = getBufPage(pInfo->pBuf, *pageId); + void* page = getBufPage(pInfo->pBuf, *pageId); blockDataEnsureCapacity(pInfo->binfo.pRes, pInfo->rowCapacity); blockDataFromBuf1(pInfo->binfo.pRes, page, pInfo->rowCapacity); @@ -670,14 +675,14 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SPartitionOperatorInfo* pInfo = pOperator->info; - SSDataBlock* pRes = pInfo->binfo.pRes; + SSDataBlock* pRes = pInfo->binfo.pRes; if (pOperator->status == OP_RES_TO_RETURN) { blockDataCleanup(pRes); return buildPartitionResult(pOperator); } - int64_t st = taosGetTimestampUs(); + int64_t st = taosGetTimestampUs(); SOperatorInfo* downstream = pOperator->pDownstream[0]; while (1) { @@ -688,7 +693,8 @@ static SSDataBlock* hashPartition(SOperatorInfo* pOperator) { // there is an scalar expression that needs to be calculated right before apply the group aggregation. if (pInfo->scalarSup.pExprInfo != NULL) { - pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, pInfo->scalarSup.numOfExprs, NULL); + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); if (pTaskInfo->code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, pTaskInfo->code); } @@ -727,7 +733,7 @@ static void destroyPartitionOperatorInfo(void* param) { cleanupBasicInfo(&pInfo->binfo); taosArrayDestroy(pInfo->pGroupCols); - for(int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++){ + for (int i = 0; i < taosArrayGetSize(pInfo->pGroupColVals); i++) { SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->pGroupColVals, i); taosMemoryFree(key.pData); } @@ -743,24 +749,25 @@ static void destroyPartitionOperatorInfo(void* param) { taosMemoryFreeClear(param); } -SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo) { SPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SPartitionOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { - int32_t num = 0; + int32_t num = 0; SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); - int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -772,7 +779,7 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - uint32_t defaultPgsz = 0; + uint32_t defaultPgsz = 0; uint32_t defaultBufsz = 0; getBufferPgSize(pResBlock->info.rowSize, &defaultPgsz, &defaultBufsz); @@ -794,15 +801,15 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition goto _error; } - pOperator->name = "PartitionOperator"; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; + pOperator->name = "PartitionOperator"; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_PARTITION; - pInfo->binfo.pRes = pResBlock; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pInfo->binfo.pRes = pResBlock; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, hashPartition, NULL, NULL, destroyPartitionOperatorInfo, NULL, NULL, NULL); @@ -810,16 +817,16 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition code = appendDownstream(pOperator, &downstream, 1); return pOperator; - _error: +_error: pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFreeClear(pInfo); taosMemoryFreeClear(pOperator); return NULL; } -int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, int16_t bytes, - uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; +int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, int32_t numOfCols, char* pData, + int16_t bytes, uint64_t groupId, SDiskbasedBuf* pBuf, SAggSupporter* pAggSup) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultRowInfo* pResultRowInfo = &binfo->resultRowInfo; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; @@ -830,3 +837,211 @@ int32_t setGroupResultOutputBuf(SOperatorInfo* pOperator, SOptrBasicInfo* binfo, setResultRowInitCtx(pResultRow, pCtx, numOfCols, pOperator->exprSupp.rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } + +uint64_t calGroupIdByData(SPartitionBySupporter* pParSup, SExprSupp* pExprSup, SSDataBlock* pBlock, int32_t rowId) { + if (pExprSup->pExprInfo != NULL) { + int32_t code = + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + if (code != TSDB_CODE_SUCCESS) { + qError("calaculate group id error, code:%d", code); + } + } + recordNewGroupKeys(pParSup->pGroupCols, pParSup->pGroupColVals, pBlock, rowId); + int32_t len = buildGroupKeys(pParSup->keyBuf, pParSup->pGroupColVals); + uint64_t groupId = calcGroupId(pParSup->keyBuf, len); + return groupId; +} + +static bool hasRemainPartion(SStreamPartitionOperatorInfo* pInfo) { return pInfo->parIte != NULL; } + +static SSDataBlock* buildStreamPartitionResult(SOperatorInfo* pOperator) { + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + SSDataBlock* pDest = pInfo->binfo.pRes; + ASSERT(hasRemainPartion(pInfo)); + SPartitionDataInfo* pParInfo = (SPartitionDataInfo*)pInfo->parIte; + blockDataCleanup(pDest); + int32_t rows = taosArrayGetSize(pParInfo->rowIds); + SSDataBlock* pSrc = pInfo->pInputDataBlock; + for (int32_t i = 0; i < rows; i++) { + int32_t rowIndex = *(int32_t*)taosArrayGet(pParInfo->rowIds, i); + for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; j++) { + int32_t slotId = pOperator->exprSupp.pExprInfo[j].base.pParam[0].pCol->slotId; + SColumnInfoData* pSrcCol = taosArrayGet(pSrc->pDataBlock, slotId); + SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, j); + bool isNull = colDataIsNull(pSrcCol, pSrc->info.rows, rowIndex, NULL); + char* pSrcData = colDataGetData(pSrcCol, rowIndex); + colDataAppend(pDestCol, pDest->info.rows, pSrcData, isNull); + } + pDest->info.rows++; + } + blockDataUpdateTsWindow(pDest, pInfo->tsColIndex); + pDest->info.groupId = pParInfo->groupId; + pOperator->resultInfo.totalRows += pDest->info.rows; + pInfo->parIte = taosHashIterate(pInfo->pPartitions, pInfo->parIte); + ASSERT(pDest->info.rows > 0); + printDataBlock(pDest, "stream partitionby"); + return pDest; +} + +static void doStreamHashPartitionImpl(SStreamPartitionOperatorInfo* pInfo, SSDataBlock* pBlock) { + pInfo->pInputDataBlock = pBlock; + for (int32_t i = 0; i < pBlock->info.rows; ++i) { + recordNewGroupKeys(pInfo->partitionSup.pGroupCols, pInfo->partitionSup.pGroupColVals, pBlock, i); + int32_t keyLen = buildGroupKeys(pInfo->partitionSup.keyBuf, pInfo->partitionSup.pGroupColVals); + SPartitionDataInfo* pParData = + (SPartitionDataInfo*)taosHashGet(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen); + if (pParData) { + taosArrayPush(pParData->rowIds, &i); + } else { + SPartitionDataInfo newParData = {0}; + newParData.groupId = calcGroupId(pInfo->partitionSup.keyBuf, keyLen); + newParData.rowIds = taosArrayInit(64, sizeof(int32_t)); + taosArrayPush(newParData.rowIds, &i); + taosHashPut(pInfo->pPartitions, pInfo->partitionSup.keyBuf, keyLen, &newParData, sizeof(SPartitionDataInfo)); + } + } +} + +static SSDataBlock* doStreamHashPartition(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SStreamPartitionOperatorInfo* pInfo = pOperator->info; + if (hasRemainPartion(pInfo)) { + return buildStreamPartitionResult(pOperator); + } + + int64_t st = taosGetTimestampUs(); + SOperatorInfo* downstream = pOperator->pDownstream[0]; + { + pInfo->pInputDataBlock = NULL; + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + doSetOperatorCompleted(pOperator); + return NULL; + } + printDataBlock(pBlock, "stream partitionby recv"); + switch (pBlock->info.type) { + case STREAM_NORMAL: + case STREAM_PULL_DATA: + case STREAM_INVALID: + pInfo->binfo.pRes->info.type = pBlock->info.type; + break; + case STREAM_DELETE_DATA: { + copyDataBlock(pInfo->pDelRes, pBlock); + pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; + return pInfo->pDelRes; + } break; + default: + return pBlock; + } + + // there is an scalar expression that needs to be calculated right before apply the group aggregation. + if (pInfo->scalarSup.pExprInfo != NULL) { + pTaskInfo->code = projectApplyFunctions(pInfo->scalarSup.pExprInfo, pBlock, pBlock, pInfo->scalarSup.pCtx, + pInfo->scalarSup.numOfExprs, NULL); + if (pTaskInfo->code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, pTaskInfo->code); + } + } + taosHashClear(pInfo->pPartitions); + doStreamHashPartitionImpl(pInfo, pBlock); + } + pOperator->cost.openCost = (taosGetTimestampUs() - st) / 1000.0; + + pInfo->parIte = taosHashIterate(pInfo->pPartitions, NULL); + return buildStreamPartitionResult(pOperator); +} + +static void destroyStreamPartitionOperatorInfo(void* param) { + SStreamPartitionOperatorInfo* pInfo = (SStreamPartitionOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + taosArrayDestroy(pInfo->partitionSup.pGroupCols); + + for (int i = 0; i < taosArrayGetSize(pInfo->partitionSup.pGroupColVals); i++) { + SGroupKeys key = *(SGroupKeys*)taosArrayGet(pInfo->partitionSup.pGroupColVals, i); + taosMemoryFree(key.pData); + } + taosArrayDestroy(pInfo->partitionSup.pGroupColVals); + + taosMemoryFree(pInfo->partitionSup.keyBuf); + cleanupExprSupp(&pInfo->scalarSup); + blockDataDestroy(pInfo->pDelRes); + taosMemoryFreeClear(param); +} + +void initParDownStream(SOperatorInfo* downstream, SPartitionBySupporter* pParSup, SExprSupp* pExpr) { + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + return; + } + SStreamScanInfo* pScanInfo = downstream->info; + pScanInfo->partitionSup = *pParSup; + pScanInfo->pPartScalarSup = pExpr; +} + +SOperatorInfo* createStreamPartitionOperatorInfo(SOperatorInfo* downstream, SStreamPartitionPhysiNode* pPartNode, + SExecTaskInfo* pTaskInfo) { + SStreamPartitionOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamPartitionOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + int32_t code = TSDB_CODE_SUCCESS; + pInfo->partitionSup.pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); + + if (pPartNode->pExprs != NULL) { + int32_t num = 0; + SExprInfo* pCalExprInfo = createExprInfo(pPartNode->pExprs, NULL, &num); + code = initExprSupp(&pInfo->scalarSup, pCalExprInfo, num); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + int32_t keyLen = 0; + code = initGroupOptrInfo(&pInfo->partitionSup.pGroupColVals, &keyLen, &pInfo->partitionSup.keyBuf, + pInfo->partitionSup.pGroupCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + pInfo->partitionSup.needCalc = true; + + SSDataBlock* pResBlock = createResDataBlock(pPartNode->node.pOutputDataBlockDesc); + if (!pResBlock) { + goto _error; + } + blockDataEnsureCapacity(pResBlock, 4096); + pInfo->binfo.pRes = pResBlock; + pInfo->parIte = NULL; + pInfo->pInputDataBlock = NULL; + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + pInfo->pPartitions = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + pInfo->tsColIndex = 0; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pPartNode->pTargets, NULL, &numOfCols); + + pOperator->name = "StreamPartitionOperator"; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamHashPartition, NULL, NULL, + destroyStreamPartitionOperatorInfo, NULL, NULL, NULL); + + initParDownStream(downstream, &pInfo->partitionSup, &pInfo->scalarSup); + code = appendDownstream(pOperator, &downstream, 1); + return pOperator; + +_error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + destroyStreamPartitionOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + return NULL; +} diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index 9a5368e90e6b5c7f014314c7bc8fababff1c3370..ad9cd1ffe7909c9a67e5af2e98193995757a05c2 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -695,6 +695,7 @@ static void destroyTableScanOperatorInfo(void* param) { cleanupQueryTableDataCond(&pTableScanInfo->cond); tsdbReaderClose(pTableScanInfo->dataReader); + pTableScanInfo->dataReader = NULL; if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); @@ -920,49 +921,28 @@ static void doClearBufferedBlocks(SStreamScanInfo* pInfo) { } static bool isSessionWindow(SStreamScanInfo* pInfo) { - return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || - pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION || + pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; } static bool isStateWindow(SStreamScanInfo* pInfo) { - return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE; } static bool isIntervalWindow(SStreamScanInfo* pInfo) { - return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || - pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || - pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL || + pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL || + pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; } static bool isSignleIntervalWindow(SStreamScanInfo* pInfo) { - return pInfo->sessionSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; + return pInfo->windowSup.parentType == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; } static bool isSlidingWindow(SStreamScanInfo* pInfo) { return isIntervalWindow(pInfo) && pInfo->interval.interval != pInfo->interval.sliding; } -static uint64_t getGroupId(SOperatorInfo* pOperator, uint64_t uid) { - uint64_t* groupId = taosHashGet(pOperator->pTaskInfo->tableqinfoList.map, &uid, sizeof(int64_t)); - if (groupId) { - return *groupId; - } - return 0; - /* Todo(liuyao) for partition by column - recordNewGroupKeys(pTableScanInfo->pGroupCols, pTableScanInfo->pGroupColVals, pBlock, rowId); - int32_t len = buildGroupKeys(pTableScanInfo->keyBuf, pTableScanInfo->pGroupColVals); - uint64_t resId = 0; - uint64_t* groupId = taosHashGet(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len); - if (groupId) { - return *groupId; - } else if (len != 0) { - resId = calcGroupId(pTableScanInfo->keyBuf, len); - taosHashPut(pTableScanInfo->pGroupSet, pTableScanInfo->keyBuf, len, &resId, sizeof(uint64_t)); - } - return resId; - */ -} - static void setGroupId(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t groupColIndex, int32_t rowIndex) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, groupColIndex); uint64_t* groupCol = (uint64_t*)pColInfo->pData; @@ -976,6 +956,61 @@ void resetTableScanInfo(STableScanInfo* pTableScanInfo, STimeWindow* pWin) { pTableScanInfo->currentGroupId = -1; } +static void freeArray(void* array) { taosArrayDestroy(array); } + +static void resetTableScanOperator(SOperatorInfo* pTableScanOp) { + STableScanInfo* pTableScanInfo = pTableScanOp->info; + pTableScanInfo->cond.startVersion = -1; + pTableScanInfo->cond.endVersion = -1; + SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList; + SArray* allTbls = pTableScanOp->pTaskInfo->tableqinfoList.pTableList; + taosArrayClearP(gpTbls, freeArray); + taosArrayPush(gpTbls, &allTbls); + STimeWindow win = {.skey = INT64_MIN, .ekey = INT64_MAX}; + resetTableScanInfo(pTableScanOp->info, &win); +} + +static SSDataBlock* readPreVersionData(SOperatorInfo* pTableScanOp, uint64_t tbUid, TSKEY startTs, TSKEY endTs, + int64_t maxVersion) { + SArray* gpTbls = pTableScanOp->pTaskInfo->tableqinfoList.pGroupList; + taosArrayClear(gpTbls); + STableKeyInfo tblInfo = {.uid = tbUid, .groupId = 0}; + SArray* tbls = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(tbls, &tblInfo); + taosArrayPush(gpTbls, &tbls); + + STimeWindow win = {.skey = startTs, .ekey = endTs}; + STableScanInfo* pTableScanInfo = pTableScanOp->info; + pTableScanInfo->cond.startVersion = -1; + pTableScanInfo->cond.endVersion = maxVersion; + resetTableScanInfo(pTableScanOp->info, &win); + SSDataBlock* pRes = doTableScan(pTableScanOp); + resetTableScanOperator(pTableScanOp); + return pRes; +} + +static uint64_t getGroupIdByCol(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { + SSDataBlock* pPreRes = readPreVersionData(pInfo->pTableScanOp, uid, ts, ts, maxVersion); + if (!pPreRes || pPreRes->info.rows == 0) { + return 0; + } + ASSERT(pPreRes->info.rows == 1); + return calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pPreRes, 0); +} + +static uint64_t getGroupIdByData(SStreamScanInfo* pInfo, uint64_t uid, TSKEY ts, int64_t maxVersion) { + if (pInfo->partitionSup.needCalc) { + return getGroupIdByCol(pInfo, uid, ts, maxVersion); + } + + SHashObj* map = pInfo->pTableScanOp->pTaskInfo->tableqinfoList.map; + uint64_t* groupId = taosHashGet(map, &uid, sizeof(int64_t)); + if (groupId) { + return *groupId; + } + return 0; +} + static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_t* pRowIndex) { if ((*pRowIndex) == pBlock->info.rows) { return false; @@ -987,6 +1022,9 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); TSKEY* endData = (TSKEY*)pEndTsCol->pData; STimeWindow win = {.skey = startData[*pRowIndex], .ekey = endData[*pRowIndex]}; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* gpData = (uint64_t*)pGpCol->pData; + uint64_t groupId = gpData[*pRowIndex]; SColumnInfoData* pCalStartTsCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); TSKEY* calStartData = (TSKEY*)pCalStartTsCol->pData; @@ -1001,11 +1039,11 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ (*pRowIndex)++; for (; *pRowIndex < pBlock->info.rows; (*pRowIndex)++) { - if (win.skey == startData[*pRowIndex]) { + if (win.skey == startData[*pRowIndex] && groupId == gpData[*pRowIndex]) { win.ekey = TMAX(win.ekey, endData[*pRowIndex]); continue; } - if (win.skey == endData[*pRowIndex]) { + if (win.skey == endData[*pRowIndex] && groupId == gpData[*pRowIndex]) { win.skey = TMIN(win.skey, startData[*pRowIndex]); continue; } @@ -1019,20 +1057,24 @@ static bool prepareRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pBlock, int32_ return true; } -static STimeWindow getSlidingWindow(TSKEY* tsCol, SInterval* pInterval, SDataBlockInfo* pDataBlockInfo, - int32_t* pRowIndex) { +static STimeWindow getSlidingWindow(TSKEY* startTsCol, TSKEY* endTsCol, SInterval* pInterval, + SDataBlockInfo* pDataBlockInfo, int32_t* pRowIndex, bool hasGroup) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCol[*pRowIndex], pInterval, TSDB_ORDER_ASC); STimeWindow endWin = win; STimeWindow preWin = win; while (1) { - (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, tsCol, *pRowIndex, endWin.ekey, binarySearchForKey, NULL, - TSDB_ORDER_ASC); + if (hasGroup) { + (*pRowIndex) += 1; + } else { + (*pRowIndex) += getNumOfRowsInTimeWindow(pDataBlockInfo, startTsCol, *pRowIndex, endWin.ekey, binarySearchForKey, + NULL, TSDB_ORDER_ASC); + } do { preWin = endWin; getNextTimeWindow(pInterval, &endWin, TSDB_ORDER_ASC); - } while (tsCol[(*pRowIndex) - 1] >= endWin.skey); + } while (endTsCol[(*pRowIndex) - 1] >= endWin.skey); endWin = preWin; if (win.ekey == endWin.ekey || (*pRowIndex) == pDataBlockInfo->rows) { win.ekey = endWin.ekey; @@ -1060,7 +1102,31 @@ static SSDataBlock* doRangeScan(SStreamScanInfo* pInfo, SSDataBlock* pSDB, int32 return NULL; } - if (pResult->info.groupId == pInfo->groupId) { + doFilter(pInfo->pCondition, pResult, NULL); + if (pResult->info.rows == 0) { + continue; + } + + if (pInfo->partitionSup.needCalc) { + SSDataBlock* tmpBlock = createOneDataBlock(pResult, true); + blockDataCleanup(pResult); + for (int32_t i = 0; i < tmpBlock->info.rows; i++) { + if (calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, tmpBlock, i) == pInfo->groupId) { + for (int32_t j = 0; j < pInfo->pTableScanOp->exprSupp.numOfExprs; j++) { + SColumnInfoData* pSrcCol = taosArrayGet(tmpBlock->pDataBlock, j); + SColumnInfoData* pDestCol = taosArrayGet(pResult->pDataBlock, j); + bool isNull = colDataIsNull(pSrcCol, tmpBlock->info.rows, i, NULL); + char* pSrcData = colDataGetData(pSrcCol, i); + colDataAppend(pDestCol, pResult->info.rows, pSrcData, isNull); + } + pResult->info.rows++; + } + } + if (pResult->info.rows > 0) { + pResult->info.calWin = pInfo->updateWin; + return pResult; + } + } else if (pResult->info.groupId == pInfo->groupId) { pResult->info.calWin = pInfo->updateWin; return pResult; } @@ -1091,17 +1157,18 @@ static int32_t generateSessionScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSr SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); int32_t dummy = 0; + int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { - uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[i]); + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); // gap must be 0. SResultWindowInfo* pStartWin = - getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy); + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, startData[i], endData[i], groupId, 0, &dummy); if (!pStartWin) { // window has been closed. continue; } SResultWindowInfo* pEndWin = - getCurSessionWindow(pInfo->sessionSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy); + getCurSessionWindow(pInfo->windowSup.pStreamAggSup, endData[i], endData[i], groupId, 0, &dummy); ASSERT(pEndWin); TSKEY ts = INT64_MIN; colDataAppend(pDestStartCol, i, (const char*)&pStartWin->win.skey, false); @@ -1121,34 +1188,90 @@ static int32_t generateIntervalScanRange(SStreamScanInfo* pInfo, SSDataBlock* pS if (rows == 0) { return TSDB_CODE_SUCCESS; } - int32_t code = blockDataEnsureCapacity(pDestBlock, rows); + int32_t code = blockDataEnsureCapacity(pDestBlock, rows * 2); if (code != TSDB_CODE_SUCCESS) { return code; } - SColumnInfoData* pTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); - uint64_t* uidCol = (uint64_t*)pUidCol->pData; - ASSERT(pTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); - TSKEY* tsCol = (TSKEY*)pTsCol->pData; + SColumnInfoData* pSrcStartTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pSrcEndTsCol = (SColumnInfoData*)taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pSrcUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* srcUidData = (uint64_t*)pSrcUidCol->pData; + SColumnInfoData* pSrcGpCol = taosArrayGet(pSrcBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* srcGp = (uint64_t*)pSrcGpCol->pData; + ASSERT(pSrcStartTsCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); + TSKEY* srcStartTsCol = (TSKEY*)pSrcStartTsCol->pData; + TSKEY* srcEndTsCol = (TSKEY*)pSrcEndTsCol->pData; SColumnInfoData* pStartTsCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDeUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); SColumnInfoData* pGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); SColumnInfoData* pCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); SColumnInfoData* pCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); - uint64_t groupId = getGroupId(pInfo->pTableScanOp, uidCol[0]); + int64_t version = pSrcBlock->info.version - 1; for (int32_t i = 0; i < rows;) { - colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(tsCol + i), false); - STimeWindow win = getSlidingWindow(tsCol, &pInfo->interval, &pSrcBlock->info, &i); - colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(tsCol + i - 1), false); - + uint64_t srcUid = srcUidData[i]; + uint64_t groupId = getGroupIdByData(pInfo, srcUid, srcStartTsCol[i], version); + uint64_t srcGpId = srcGp[i]; + TSKEY calStartTs = srcStartTsCol[i]; + colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); + STimeWindow win = getSlidingWindow(srcStartTsCol, srcEndTsCol, &pInfo->interval, &pSrcBlock->info, &i, + pInfo->partitionSup.needCalc); + TSKEY calEndTs = srcStartTsCol[i - 1]; + colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); + colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&groupId), false); pDestBlock->info.rows++; + if (pInfo->partitionSup.needCalc && srcGpId != 0 && groupId != srcGpId) { + colDataAppend(pCalStartTsCol, pDestBlock->info.rows, (const char*)(&calStartTs), false); + colDataAppend(pCalEndTsCol, pDestBlock->info.rows, (const char*)(&calEndTs), false); + colDataAppend(pDeUidCol, pDestBlock->info.rows, (const char*)(&srcUid), false); + colDataAppend(pStartTsCol, pDestBlock->info.rows, (const char*)(&win.skey), false); + colDataAppend(pEndTsCol, pDestBlock->info.rows, (const char*)(&win.ekey), false); + colDataAppend(pGpCol, pDestBlock->info.rows, (const char*)(&srcGpId), false); + pDestBlock->info.rows++; + } + } + return TSDB_CODE_SUCCESS; +} + +static int32_t generateDeleteResultBlock(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, SSDataBlock* pDestBlock) { + if (pSrcBlock->info.rows == 0) { + return TSDB_CODE_SUCCESS; + } + blockDataCleanup(pDestBlock); + int32_t code = blockDataEnsureCapacity(pDestBlock, pSrcBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + ASSERT(taosArrayGetSize(pSrcBlock->pDataBlock) >= 3); + SColumnInfoData* pStartTsCol = taosArrayGet(pSrcBlock->pDataBlock, START_TS_COLUMN_INDEX); + TSKEY* startData = (TSKEY*)pStartTsCol->pData; + SColumnInfoData* pEndTsCol = taosArrayGet(pSrcBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* endData = (TSKEY*)pEndTsCol->pData; + SColumnInfoData* pUidCol = taosArrayGet(pSrcBlock->pDataBlock, UID_COLUMN_INDEX); + uint64_t* uidCol = (uint64_t*)pUidCol->pData; + + SColumnInfoData* pDestStartCol = taosArrayGet(pDestBlock->pDataBlock, START_TS_COLUMN_INDEX); + SColumnInfoData* pDestEndCol = taosArrayGet(pDestBlock->pDataBlock, END_TS_COLUMN_INDEX); + SColumnInfoData* pDestUidCol = taosArrayGet(pDestBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pDestGpCol = taosArrayGet(pDestBlock->pDataBlock, GROUPID_COLUMN_INDEX); + SColumnInfoData* pDestCalStartTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + SColumnInfoData* pDestCalEndTsCol = taosArrayGet(pDestBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + int32_t dummy = 0; + int64_t version = pSrcBlock->info.version - 1; + for (int32_t i = 0; i < pSrcBlock->info.rows; i++) { + uint64_t groupId = getGroupIdByData(pInfo, uidCol[i], startData[i], version); + colDataAppend(pDestStartCol, i, (const char*)(startData + i), false); + colDataAppend(pDestEndCol, i, (const char*)(endData + i), false); + colDataAppendNULL(pDestUidCol, i); + colDataAppend(pDestGpCol, i, (const char*)&groupId, false); + colDataAppendNULL(pDestCalStartTsCol, i); + colDataAppendNULL(pDestCalEndTsCol, i); + pDestBlock->info.rows++; } - // all rows have same group id - pDestBlock->info.groupId = groupId; return TSDB_CODE_SUCCESS; } @@ -1156,21 +1279,24 @@ static int32_t generateScanRange(SStreamScanInfo* pInfo, SSDataBlock* pSrcBlock, int32_t code = TSDB_CODE_SUCCESS; if (isIntervalWindow(pInfo)) { code = generateIntervalScanRange(pInfo, pSrcBlock, pDestBlock); - } else { + } else if (isSessionWindow(pInfo) || isStateWindow(pInfo)) { code = generateSessionScanRange(pInfo, pSrcBlock, pDestBlock); } pDestBlock->info.type = STREAM_CLEAR; + pDestBlock->info.version = pSrcBlock->info.version; blockDataUpdateTsWindow(pDestBlock, 0); return code; } -void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, int32_t uidCol, uint64_t* pID) { +void appendOneRow(SSDataBlock* pBlock, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t* pUid, uint64_t* pGp) { SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); - SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, uidCol); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)pStartTs, false); colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)pEndTs, false); - colDataAppend(pUidCol, pBlock->info.rows, (const char*)pID, false); + colDataAppend(pUidCol, pBlock->info.rows, (const char*)pUid, false); + colDataAppend(pGpCol, pBlock->info.rows, (const char*)pGp, false); pBlock->info.rows++; } @@ -1195,24 +1321,18 @@ static void checkUpdateData(SStreamScanInfo* pInfo, bool invertible, SSDataBlock // must check update info first. bool update = updateInfoIsUpdated(pInfo->pUpdateInfo, pBlock->info.uid, tsCol[rowId]); bool closedWin = isClosed && isSignleIntervalWindow(pInfo) && - isDeletedWindow(&win, pBlock->info.groupId, pInfo->sessionSup.pIntervalAggSup); + isDeletedWindow(&win, pBlock->info.groupId, pInfo->windowSup.pIntervalAggSup); if ((update || closedWin) && out) { - appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, UID_COLUMN_INDEX, &pBlock->info.uid); + uint64_t gpId = closedWin && pInfo->partitionSup.needCalc + ? calGroupIdByData(&pInfo->partitionSup, pInfo->pPartScalarSup, pBlock, rowId) + : 0; + appendOneRow(pInfo->pUpdateDataRes, tsCol + rowId, tsCol + rowId, &pBlock->info.uid, &gpId); } } - if (out) { + if (out && pInfo->pUpdateDataRes->info.rows > 0) { + pInfo->pUpdateDataRes->info.version = pBlock->info.version; blockDataUpdateTsWindow(pInfo->pUpdateDataRes, 0); - pInfo->pUpdateDataRes->info.type = STREAM_CLEAR; - } -} - -static void setBlockGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_t uidColIndex) { - ASSERT(taosArrayGetSize(pBlock->pDataBlock) >= 3); - SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, uidColIndex); - uint64_t* uidCol = (uint64_t*)pColDataInfo->pData; - ASSERT(pBlock->info.rows > 0); - for (int32_t i = 0; i < pBlock->info.rows; i++) { - uidCol[i] = getGroupId(pOperator, uidCol[i]); + pInfo->pUpdateDataRes->info.type = pInfo->partitionSup.needCalc ? STREAM_DELETE_DATA : STREAM_CLEAR; } } @@ -1436,17 +1556,28 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { updateInfoAddCloseWindowSBF(pInfo->pUpdateInfo); } break; case STREAM_DELETE_DATA: { - pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; - pInfo->updateResIndex = 0; - generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); - prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); - copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); - pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; - return pInfo->pDeleteDataRes; + printDataBlock(pBlock, "stream scan delete recv"); + if (!isIntervalWindow(pInfo) && !isSessionWindow(pInfo) && !isStateWindow(pInfo)) { + generateDeleteResultBlock(pInfo, pBlock, pInfo->pDeleteDataRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_RESULT; + printDataBlock(pBlock, "stream scan delete result"); + return pInfo->pDeleteDataRes; + } else { + pInfo->blockType = STREAM_INPUT__DATA_SUBMIT; + pInfo->updateResIndex = 0; + generateScanRange(pInfo, pBlock, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + printDataBlock(pBlock, "stream scan delete data"); + return pInfo->pDeleteDataRes; + } } break; default: break; } + // printDataBlock(pBlock, "stream scan recv"); return pBlock; } else if (pInfo->blockType == STREAM_INPUT__DATA_SUBMIT) { qDebug("scan mode %d", pInfo->scanMode); @@ -1456,6 +1587,14 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; return pInfo->pRes; } break; + case STREAM_SCAN_FROM_DELETE_DATA: { + generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); + prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); + pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; + copyDataBlock(pInfo->pDeleteDataRes, pInfo->pUpdateRes); + pInfo->pDeleteDataRes->info.type = STREAM_DELETE_DATA; + return pInfo->pDeleteDataRes; + } break; case STREAM_SCAN_FROM_UPDATERES: { generateScanRange(pInfo, pInfo->pUpdateDataRes, pInfo->pUpdateRes); prepareRangeScan(pInfo, pInfo->pUpdateRes, &pInfo->updateResIndex); @@ -1471,6 +1610,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { updateInfoSetScanRange(pInfo->pUpdateInfo, &pTableScanInfo->cond.twindows, pInfo->groupId, version); pSDB->info.type = pInfo->scanMode == STREAM_SCAN_FROM_DATAREADER_RANGE ? STREAM_NORMAL : STREAM_PULL_DATA; checkUpdateData(pInfo, true, pSDB, false); + // printDataBlock(pSDB, "stream scan update"); return pSDB; } pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -1479,7 +1619,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { break; } - SStreamAggSupporter* pSup = pInfo->sessionSup.pStreamAggSup; + SStreamAggSupporter* pSup = pInfo->windowSup.pStreamAggSup; if (isStateWindow(pInfo) && pSup->pScanBlock->info.rows > 0) { pInfo->scanMode = STREAM_SCAN_FROM_DATAREADER_RANGE; pInfo->updateResIndex = 0; @@ -1545,7 +1685,7 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { // record the scan action. pInfo->numOfExec++; pOperator->resultInfo.totalRows += pBlockInfo->rows; - printDataBlock(pInfo->pRes, "stream scan"); + // printDataBlock(pInfo->pRes, "stream scan"); if (pBlockInfo->rows == 0) { updateInfoDestoryColseWinSBF(pInfo->pUpdateInfo); @@ -1554,19 +1694,20 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { checkUpdateData(pInfo, true, pInfo->pRes, true); pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlockInfo->window.ekey); if (pInfo->pUpdateDataRes->info.rows > 0) { + pInfo->updateResIndex = 0; if (pInfo->pUpdateDataRes->info.type == STREAM_CLEAR) { - pInfo->updateResIndex = 0; pInfo->scanMode = STREAM_SCAN_FROM_UPDATERES; } else if (pInfo->pUpdateDataRes->info.type == STREAM_INVERT) { pInfo->scanMode = STREAM_SCAN_FROM_RES; return pInfo->pUpdateDataRes; + } else if (pInfo->pUpdateDataRes->info.type == STREAM_DELETE_DATA) { + pInfo->scanMode = STREAM_SCAN_FROM_DELETE_DATA; } } } qDebug("scan rows: %d", pBlockInfo->rows); return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; - } else { ASSERT(0); return NULL; @@ -1831,12 +1972,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->tqReader = pHandle->tqReader; } - if (pTSInfo->pdInfo.interval.interval > 0) { - pInfo->pUpdateInfo = updateInfoInitP(&pTSInfo->pdInfo.interval, pInfo->twAggSup.waterMark); - } else { - pInfo->pUpdateInfo = NULL; - } - + pInfo->pUpdateInfo = NULL; pInfo->pTableScanOp = pTableScanOp; pInfo->interval = pTSInfo->pdInfo.interval; @@ -1867,8 +2003,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->pUpdateRes = createSpecialDataBlock(STREAM_CLEAR); pInfo->pCondition = pScanPhyNode->node.pConditions; pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; - pInfo->sessionSup = - (SessionWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; + pInfo->windowSup = (SWindowSupporter){.pStreamAggSup = NULL, .gap = -1, .parentType = QUERY_NODE_PHYSICAL_PLAN}; pInfo->groupId = 0; pInfo->pPullDataRes = createSpecialDataBlock(STREAM_RETRIEVE); pInfo->pStreamScanOp = pOperator; @@ -1877,6 +2012,7 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys pInfo->updateWin = (STimeWindow){.skey = INT64_MAX, .ekey = INT64_MAX}; pInfo->pUpdateDataRes = createSpecialDataBlock(STREAM_CLEAR); pInfo->assignBlockUid = pTableScanNode->assignBlockUid; + pInfo->partitionSup.needCalc = false; pOperator->name = "StreamScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN; @@ -2568,7 +2704,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } char* pStart = pRsp->data; - extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pOperator->exprSupp.numOfExprs, pInfo->scanCols, &pStart); + extractDataBlockFromFetchRsp(pInfo->pRes, pRsp->data, pInfo->scanCols, &pStart); updateLoadRemoteInfo(&pInfo->loadInfo, pRsp->numOfRows, pRsp->compLen, startTs, pOperator); // todo log the filter info diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 152bd5939dbe8b5bed819f19f26b9c14d9cc4475..c221aaf4fdde6662fefc39d78c6d55d3570fa15a 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -35,29 +35,17 @@ typedef struct SPullWindowInfo { typedef struct SOpenWindowInfo { SResultRowPosition pos; - uint64_t groupId; + uint64_t groupId; } SOpenWindowInfo; static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator); static int64_t* extractTsCol(SSDataBlock* pBlock, const SIntervalAggOperatorInfo* pInfo); -static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, uint64_t groupId); +static SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SResultRow* pResult, + uint64_t groupId); static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOperatorInfo* pInfo, SResultRow* pResult); -///* -// * There are two cases to handle: -// * -// * 1. Query range is not set yet (queryRangeSet = 0). we need to set the query range info, including -// * pQueryAttr->lastKey, pQueryAttr->window.skey, and pQueryAttr->eKey. -// * 2. Query range is set and query is in progress. There may be another result with the same query ranges to be -// * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there -// * is a previous result generated or not. -// */ -// static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { -// // do nothing -//} - static TSKEY getStartTsKey(STimeWindow* win, const TSKEY* tsCols) { return tsCols == NULL ? win->skey : tsCols[0]; } static int32_t setTimeWindowOutputBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, bool masterscan, @@ -601,15 +589,15 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExprSupp* pSup = &pOperatorInfo->exprSupp; - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; + int32_t startPos = 0; + int32_t numOfOutput = pSup->numOfExprs; SResultRow* pResult = NULL; while (1) { - SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); - SOpenWindowInfo* pOpenWin = (SOpenWindowInfo *)pn->data; - uint64_t groupId = pOpenWin->groupId; + SListNode* pn = tdListGetHead(pResultRowInfo->openWindow); + SOpenWindowInfo* pOpenWin = (SOpenWindowInfo*)pn->data; + uint64_t groupId = pOpenWin->groupId; SResultRowPosition* p1 = &pOpenWin->pos; if (p->pageId == p1->pageId && p->offset == p1->offset) { break; @@ -621,7 +609,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (pr->closed) { ASSERT(isResultRowInterpolated(pr, RESULT_ROW_START_INTERP) && isResultRowInterpolated(pr, RESULT_ROW_END_INTERP)); - tdListPopHead(pResultRowInfo->openWindow); + SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); continue; } @@ -650,7 +639,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pr); - tdListPopHead(pResultRowInfo->openWindow); + SListNode* pNode = tdListPopHead(pResultRowInfo->openWindow); + taosMemoryFree(pNode); } else { // the remains are can not be closed yet. break; } @@ -897,7 +887,7 @@ int64_t getWinReskey(void* data, int32_t index) { int32_t compareWinRes(void* pKey, void* data, int32_t index) { SArray* res = (SArray*)data; - SWinKey* pos = taosArrayGetP(res, index); + SWinKey* pos = taosArrayGet(res, index); SResKeyPos* pData = (SResKeyPos*)pKey; if (*(int64_t*)pData->key == pos->ts) { if (pData->groupId > pos->groupId) { @@ -919,10 +909,11 @@ static void removeDeleteResults(SHashObj* pUpdatedMap, SArray* pDelWins) { } void* pIte = NULL; while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - SResKeyPos* pResKey = (SResKeyPos*)pIte; + SResKeyPos* pResKey = *(SResKeyPos**)pIte; int32_t index = binarySearchCom(pDelWins, delSize, pResKey, TSDB_ORDER_DESC, compareWinRes); if (index >= 0 && 0 == compareWinRes(pResKey, pDelWins, index)) { taosArrayRemove(pDelWins, index); + delSize = taosArrayGetSize(pDelWins); } } } @@ -935,7 +926,7 @@ bool isOverdue(TSKEY ts, STimeWindowAggSupp* pSup) { bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { return isOverdue(pWin->ekey, pSup); } static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SHashObj* pUpdatedMap) { + int32_t scanFlag) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -951,21 +942,11 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->inputOrder); - int32_t ret = TSDB_CODE_SUCCESS; - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); - } - - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - TSKEY ekey = ascScan ? win.ekey : win.skey; int32_t forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); @@ -987,12 +968,9 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doWindowBorderInterpolation(pInfo, pBlock, pResult, &win, startPos, forwardRows, pSup); } - if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && - inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { - updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, - numOfOutput); - } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, + numOfOutput); doCloseWindow(pResultRowInfo, pInfo, pResult); @@ -1003,13 +981,6 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul if (startPos < 0) { break; } - if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { - ekey = ascScan ? nextWin.ekey : nextWin.skey; - forwardRows = - getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); - continue; - } - // null data, failed to allocate more memory buffer int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); @@ -1017,17 +988,12 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - if (pInfo->execModel == OPTR_EXEC_MODEL_STREAM && pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { - saveWinResultRow(pResult, tableGroupId, pUpdatedMap); - setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); - } - ekey = ascScan ? nextWin.ekey : nextWin.skey; forwardRows = getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, pInfo->inputOrder); // window start(end) key interpolation doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup); - //TODO: add to open window? how to close the open windows after input blocks exhausted? + // TODO: add to open window? how to close the open windows after input blocks exhausted? #if 0 if ((ascScan && ekey <= pBlock->info.window.ekey) || (!ascScan && ekey >= pBlock->info.window.skey)) { @@ -1061,13 +1027,13 @@ SResultRowPosition addToOpenWindowList(SResultRowInfo* pResultRowInfo, const SRe openWin.pos.pageId = pResult->pageId; openWin.pos.offset = pResult->offset; openWin.groupId = groupId; - SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); + SListNode* pn = tdListGetTail(pResultRowInfo->openWindow); if (pn == NULL) { tdListAppend(pResultRowInfo->openWindow, &openWin); return openWin.pos; } - SOpenWindowInfo * px = (SOpenWindowInfo *)pn->data; + SOpenWindowInfo* px = (SOpenWindowInfo*)pn->data; if (px->pos.pageId != openWin.pos.pageId || px->pos.offset != openWin.pos.offset || px->groupId != openWin.groupId) { tdListAppend(pResultRowInfo->openWindow, &openWin); } @@ -1126,7 +1092,7 @@ static int32_t doOpenIntervalAgg(SOperatorInfo* pOperator) { setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, scanFlag, true); blockDataUpdateTsWindow(pBlock, pInfo->primaryTsIndex); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag, NULL); + hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, scanFlag); } initGroupedResultInfo(&pInfo->groupResInfo, pInfo->aggSup.pResultRowHashTable, pInfo->resultTsOrder); @@ -1376,7 +1342,7 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } -void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { +static void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1, false); SqlFunctionCtx* pCtx = pSup->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1395,8 +1361,8 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS releaseBufPage(pResultBuf, bufPage); } -bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, - int32_t numOfOutput) { +static bool doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, + int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pAggSup->pResultRowHashTable, pAggSup->keyBuf, GET_RES_WINDOW_KEY_LEN(bytes)); @@ -1423,20 +1389,32 @@ bool doDeleteIntervalWindow(SAggSupporter* pAggSup, TSKEY ts, uint64_t groupId) return true; } -void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, SSDataBlock* pBlock, SArray* pUpWins, SInterval* pInterval) { +static void doDeleteSpecifyIntervalWindow(SAggSupporter* pAggSup, STimeWindowAggSupp* pTwSup, SSDataBlock* pBlock, + SArray* pDelWins, SInterval* pInterval, SHashObj* pUpdatedMap) { SColumnInfoData* pStartCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); TSKEY* tsStarts = (TSKEY*)pStartCol->pData; + SColumnInfoData* pEndCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + TSKEY* tsEnds = (TSKEY*)pEndCol->pData; SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* groupIds = (uint64_t*)pGroupCol->pData; + int64_t numOfWin = tSimpleHashGetSize(pAggSup->pResultRowHashTable); for (int32_t i = 0; i < pBlock->info.rows; i++) { + TSKEY startTs = TMAX(tsStarts[i], pTwSup->minTs); + TSKEY endTs = TMIN(tsEnds[i], pTwSup->maxTs); SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; - STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, tsStarts[i], pInterval, TSDB_ORDER_ASC); - doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); - if (pUpWins) { + STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTs, pInterval, TSDB_ORDER_ASC); + do { + doDeleteIntervalWindow(pAggSup, win.skey, groupIds[i]); SWinKey winRes = {.ts = win.skey, .groupId = groupIds[i]}; - taosArrayPush(pUpWins, &winRes); - } + if (pDelWins) { + taosArrayPush(pDelWins, &winRes); + } + if (pUpdatedMap) { + taosHashRemove(pUpdatedMap, &winRes, sizeof(SWinKey)); + } + getNextTimeWindow(pInterval, pInterval->precision, TSDB_ORDER_ASC, &win); + } while (win.skey <= endTs); } } @@ -1446,19 +1424,14 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* TSKEY* startTsCols = (TSKEY*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); TSKEY* endTsCols = (TSKEY*)pEndTsCol->pData; - uint64_t* pGpDatas = NULL; - if (pBlock->info.type == STREAM_RETRIEVE) { - SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); - pGpDatas = (uint64_t*)pGpCol->pData; - } - int32_t step = 0; - int32_t startPos = 0; + SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t* pGpDatas = (uint64_t*)pGpCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { SResultRowInfo dumyInfo; dumyInfo.cur.pageId = -1; STimeWindow win = getActiveTimeWindow(NULL, &dumyInfo, startTsCols[i], pInterval, TSDB_ORDER_ASC); while (win.ekey <= endTsCols[i]) { - uint64_t winGpId = pGpDatas ? pGpDatas[startPos] : pBlock->info.groupId; + uint64_t winGpId = pGpDatas[i]; bool res = doClearWindow(pAggSup, pSup1, (char*)&win.skey, sizeof(TSKEY), winGpId, numOfOutput); if (pUpWins && res) { SWinKey winRes = {.ts = win.skey, .groupId = winGpId}; @@ -1470,7 +1443,6 @@ static void doClearWindows(SAggSupporter* pAggSup, SExprSupp* pSup1, SInterval* } static int32_t getAllIntervalWindow(SSHashObj* pHashMap, SHashObj* resWins) { - void* pIte = NULL; size_t keyLen = 0; int32_t iter = 0; @@ -1571,150 +1543,14 @@ static void doBuildDeleteResult(SArray* pWins, int32_t* index, SSDataBlock* pBlo return; } blockDataEnsureCapacity(pBlock, size - *index); - SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); + uint64_t uid = 0; for (int32_t i = *index; i < size; i++) { SWinKey* pWin = taosArrayGet(pWins, i); - colDataAppend(pTsCol, pBlock->info.rows, (const char*)&pWin->ts, false); - colDataAppend(pGroupCol, pBlock->info.rows, (const char*)&pWin->groupId, false); - pBlock->info.rows++; + appendOneRow(pBlock, &pWin->ts, &pWin->ts, &uid, &pWin->groupId); (*index)++; } } -static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { - SIntervalAggOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - pInfo->inputOrder = TSDB_ORDER_ASC; - SExprSupp* pSup = &pOperator->exprSupp; - - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - if (pOperator->status == OP_RES_TO_RETURN) { - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); - if (pInfo->pDelRes->info.rows > 0) { - return pInfo->pDelRes; - } - - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { - pOperator->status = OP_EXEC_DONE; - qDebug("===stream===single interval is done"); - freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - } - printDataBlock(pInfo->binfo.pRes, "single interval"); - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; - } - - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos - _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); - SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); - - SStreamState* pState = pTaskInfo->streamInfo.pState; - - while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - break; - } - // qInfo("===stream===%ld", pBlock->info.version); - printDataBlock(pBlock, "single interval recv"); - - if (pBlock->info.type == STREAM_CLEAR) { - doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, - NULL); - qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); - continue; - } - if (pBlock->info.type == STREAM_DELETE_DATA) { - doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval); - continue; - } else if (pBlock->info.type == STREAM_GET_ALL) { - getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); - continue; - } - - if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { - // set input version - pTaskInfo->version = pBlock->info.version; - } - - if (pInfo->scalarSupp.pExprInfo != NULL) { - SExprSupp* pExprSup = &pInfo->scalarSupp; - projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); - } - - // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the - // caller. Note that all the time window are not close till now. - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->inputOrder, MAIN_SCAN, true); - if (pInfo->invertible) { - setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); - } - - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); - hashIntervalAgg(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); - } - -#if 0 - if (pState) { - printf(">>>>>>>> stream read backend\n"); - SWinKey key = { - .ts = 1, - .groupId = 2, - }; - char* val = NULL; - int32_t sz; - if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) { - ASSERT(0); - } - printf("stream read %s %d\n", val, sz); - streamFreeVal(val); - - SStreamStateCur* pCur = streamStateGetCur(pState, &key); - ASSERT(pCur); - while (streamStateCurNext(pState, pCur) == 0) { - SWinKey key1; - const void* val1; - if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) { - break; - } - printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz); - } - streamStateFreeCur(pCur); - } -#endif - - pOperator->status = OP_RES_TO_RETURN; - closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, - pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); - - void* pIte = NULL; - while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { - taosArrayPush(pUpdated, pIte); - } - taosArraySort(pUpdated, resultrowComparAsc); - - finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); - initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); - blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); - removeDeleteResults(pUpdatedMap, pInfo->pDelWins); - taosHashCleanup(pUpdatedMap); - doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); - if (pInfo->pDelRes->info.rows > 0) { - return pInfo->pDelRes; - } - - doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); - printDataBlock(pInfo->binfo.pRes, "single interval"); - return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; -} - static void destroyStateWindowOperatorInfo(void* param) { SStateWindowOperatorInfo* pInfo = (SStateWindowOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); @@ -1732,6 +1568,10 @@ void destroyIntervalOperatorInfo(void* param) { SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)param; cleanupBasicInfo(&pInfo->binfo); cleanupAggSup(&pInfo->aggSup); + cleanupExprSupp(&pInfo->scalarSupp); + + tdListFree(pInfo->binfo.resultRowInfo.openWindow); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); pInfo->pInterpCols = taosArrayDestroy(pInfo->pInterpCols); taosArrayDestroyEx(pInfo->pPrevValues, freeItem); @@ -1828,14 +1668,17 @@ static bool timeWindowinterpNeeded(SqlFunctionCtx* pCtx, int32_t numOfCols, SInt return needed; } -void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup) { +void initIntervalDownStream(SOperatorInfo* downstream, uint16_t type, SAggSupporter* pSup, SInterval* pInterval, + int64_t waterMark) { if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - // Todo(liuyao) support partition by column + initIntervalDownStream(downstream->pDownstream[0], type, pSup, pInterval, waterMark); return; } SStreamScanInfo* pScanInfo = downstream->info; - pScanInfo->sessionSup.parentType = type; - pScanInfo->sessionSup.pIntervalAggSup = pSup; + pScanInfo->windowSup.parentType = type; + pScanInfo->windowSup.pIntervalAggSup = pSup; + pScanInfo->pUpdateInfo = updateInfoInitP(pInterval, waterMark); + pScanInfo->interval = *pInterval; } void initStreamFunciton(SqlFunctionCtx* pCtx, int32_t numOfExpr) { @@ -1917,11 +1760,12 @@ SOperatorInfo* createIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, doStreamIntervalAgg, NULL, + pOperator->fpSet = createOperatorFpSet(doOpenIntervalAgg, doBuildIntervalResult, NULL, NULL, destroyIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (nodeType(pPhyNode) == QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL) { - initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup); + initIntervalDownStream(downstream, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, &pInfo->aggSup, &pInfo->interval, + pInfo->twAggSup.waterMark); } code = appendDownstream(pOperator, &downstream, 1); @@ -2248,7 +2092,6 @@ static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp if (hasInterp) { pResBlock->info.rows += 1; } - } static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { @@ -2721,7 +2564,7 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi goto _error; } - int32_t num = 0; + int32_t num = 0; SExprInfo* pExprInfo = createExprInfo(pStateNode->window.pFuncs, NULL, &num); SSDataBlock* pResBlock = createResDataBlock(pStateNode->window.node.pOutputDataBlockDesc); int32_t tsSlotId = ((SColumnNode*)pStateNode->window.pTspk)->slotId; @@ -2748,7 +2591,9 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SStateWi initBasicInfo(&pInfo->binfo, pResBlock); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pInfo->twAggSup = (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType};; + pInfo->twAggSup = + (STimeWindowAggSupp){.waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType}; + ; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); pInfo->tsSlotId = tsSlotId; @@ -2849,14 +2694,26 @@ _error: } void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int32_t numOfOutput, - SExecTaskInfo* pTaskInfo) { + SExecTaskInfo* pTaskInfo, SColumnInfoData* pTimeWindowData) { for (int32_t k = 0; k < numOfOutput; ++k) { if (fmIsWindowPseudoColumnFunc(pDestCtx[k].functionId)) { - continue; - } - int32_t code = TSDB_CODE_SUCCESS; - if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { - code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); + if (!pTimeWindowData) { + continue; + } + + SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(&pDestCtx[k]); + char* p = GET_ROWCELL_INTERBUF(pEntryInfo); + SColumnInfoData idata = {0}; + idata.info.type = TSDB_DATA_TYPE_BIGINT; + idata.info.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + idata.pData = p; + + SScalarParam out = {.columnData = &idata}; + SScalarParam tw = {.numOfRows = 5, .columnData = pTimeWindowData}; + pDestCtx[k].sfp.process(&tw, 1, &out); + pEntryInfo->numOfRes = 1; + } else if (functionNeedToExecute(&pDestCtx[k]) && pDestCtx[k].fpSet.combine != NULL) { + int32_t code = pDestCtx[k].fpSet.combine(&pDestCtx[k], &pSourceCtx[k]); if (code != TSDB_CODE_SUCCESS) { qError("%s apply functions error, code: %s", GET_TASKID(pTaskInfo), tstrerror(code)); pTaskInfo->code = code; @@ -2874,8 +2731,15 @@ bool hasIntervalWindow(SAggSupporter* pSup, TSKEY ts, uint64_t groupId) { return p1 != NULL; } +STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { + STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; + w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; + return w; +} + static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, - int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, SArray* pUpdated) { + int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo, + SHashObj* pUpdatedMap) { int32_t size = taosArrayGetSize(pWinArray); if (!pInfo->pChildren) { return; @@ -2883,11 +2747,14 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr for (int32_t i = 0; i < size; i++) { SWinKey* pWinRes = taosArrayGet(pWinArray, i); SResultRow* pCurResult = NULL; - STimeWindow ParentWin = {.skey = pWinRes->ts, .ekey = pWinRes->ts + 1}; - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &ParentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, + STimeWindow parentWin = getFinalTimeWindow(pWinRes->ts, &pInfo->interval); + if (isDeletedWindow(&parentWin, pWinRes->groupId, &pInfo->aggSup) && isCloseWindow(&parentWin, &pInfo->twAggSup)) { + continue; + } + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &parentWin, true, &pCurResult, pWinRes->groupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); - bool find = true; + int32_t num = 0; for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); SIntervalAggOperatorInfo* pChInfo = pChildOp->info; @@ -2895,15 +2762,16 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr if (!hasIntervalWindow(&pChInfo->aggSup, pWinRes->ts, pWinRes->groupId)) { continue; } - find = true; + num++; SResultRow* pChResult = NULL; - setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &ParentWin, true, &pChResult, pWinRes->groupId, + setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, &parentWin, true, &pChResult, pWinRes->groupId, pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, pTaskInfo); - compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &parentWin, true); + compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); } - if (find && pUpdated) { - saveResultRow(pCurResult, pWinRes->groupId, pUpdated); + if (num > 0 && pUpdatedMap) { + saveWinResultRow(pCurResult, pWinRes->groupId, pUpdatedMap); setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pInfo->binfo.resultRowInfo.cur); } } @@ -2912,7 +2780,7 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(pSup->pResultRowHashTable, pSup->keyBuf, - GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); + GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); return p1 == NULL; } @@ -2934,21 +2802,14 @@ void addPullWindow(SHashObj* pMap, SWinKey* pWinRes, int32_t size) { static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } -STimeWindow getFinalTimeWindow(int64_t ts, SInterval* pInterval) { - STimeWindow w = {.skey = ts, .ekey = INT64_MAX}; - w.ekey = taosTimeAdd(w.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - return w; -} - -static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, - SHashObj* pUpdatedMap) { +static void doHashIntervalAgg(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SHashObj* pUpdatedMap) { SStreamFinalIntervalOperatorInfo* pInfo = (SStreamFinalIntervalOperatorInfo*)pOperatorInfo->info; SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; int32_t numOfOutput = pSup->numOfExprs; int32_t step = 1; - bool ascScan = true; TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t forwardRows = 0; @@ -2957,7 +2818,7 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; - int32_t startPos = ascScan ? 0 : (pSDataBlock->info.rows - 1); + int32_t startPos = 0; TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); STimeWindow nextWin = {0}; if (IS_FINAL_OP(pInfo)) { @@ -3043,7 +2904,7 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) tSimpleHashClear(pInfo->aggSup.pResultRowHashTable); clearDiskbasedBuf(pInfo->aggSup.pResultBuf); initResultRowInfo(&pInfo->binfo.resultRowInfo); - pInfo->aggSup.currentPageId = -1; + pInfo->aggSup.currentPageId = -1; } static void clearSpecialDataBlock(SSDataBlock* pBlock) { @@ -3126,6 +2987,31 @@ void processPullOver(SSDataBlock* pBlock, SHashObj* pMap) { } } +static void addRetriveWindow(SArray* wins, SStreamFinalIntervalOperatorInfo* pInfo) { + int32_t size = taosArrayGetSize(wins); + for (int32_t i = 0; i < size; i++) { + SWinKey* winKey = taosArrayGet(wins, i); + STimeWindow nextWin = getFinalTimeWindow(winKey->ts, &pInfo->interval); + if (isCloseWindow(&nextWin, &pInfo->twAggSup) && !pInfo->ignoreExpiredData) { + void* chIds = taosHashGet(pInfo->pPullDataMap, winKey, sizeof(SWinKey)); + if (!chIds) { + SPullWindowInfo pull = {.window = nextWin, .groupId = winKey->groupId}; + // add pull data request + savePullWindow(&pull, pInfo->pPullWins); + int32_t size1 = taosArrayGetSize(pInfo->pChildren); + addPullWindow(pInfo->pPullDataMap, winKey, size1); + qDebug("===stream===prepare retrive for delete %" PRId64 ", size:%d", winKey->ts, size1); + } + } + } +} + +static void clearFunctionContext(SExprSupp* pSup) { + for (int32_t i = 0; i < pSup->numOfExprs; i++) { + pSup->pCtx[i].saveHandle.currentPage = -1; + } +} + static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; @@ -3134,6 +3020,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); TSKEY maxTs = INT64_MIN; + TSKEY minTs = INT64_MAX; SExprSupp* pSup = &pOperator->exprSupp; @@ -3150,12 +3037,21 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pPullDataRes; } + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); + if (pInfo->pDelRes->info.rows != 0) { + // process the rest of the data + printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); + return pInfo->pDelRes; + } + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows == 0) { pOperator->status = OP_EXEC_DONE; if (!IS_FINAL_OP(pInfo)) { + clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamIntervalOperator(pInfo); + qDebug("===stream===clear semi operator"); } else { freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); } @@ -3196,8 +3092,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { break; } printDataBlock(pBlock, IS_FINAL_OP(pInfo) ? "interval final recv" : "interval semi recv"); - maxTs = TMAX(maxTs, pBlock->info.window.ekey); - maxTs = TMAX(maxTs, pBlock->info.watermark); ASSERT(pBlock->info.type != STREAM_INVERT); if (pBlock->info.type == STREAM_NORMAL || pBlock->info.type == STREAM_PULL_DATA) { @@ -3219,23 +3113,28 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { } removeResults(pUpWins, pUpdatedMap); copyDataBlock(pInfo->pUpdateRes, pBlock); - // copyUpdateDataBlock(pInfo->pUpdateRes, pBlock, pInfo->primaryTsIndex); pInfo->returnUpdate = true; taosArrayDestroy(pUpWins); break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { - doDeleteSpecifyIntervalWindow(&pInfo->aggSup, pBlock, pInfo->pDelWins, &pInfo->interval); + SArray* delWins = taosArrayInit(8, sizeof(SWinKey)); + doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, delWins, &pInfo->interval, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SStreamFinalIntervalOperatorInfo* pChildInfo = pChildOp->info; SExprSupp* pChildSup = &pChildOp->exprSupp; - doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, pBlock, NULL, &pChildInfo->interval); - rebuildIntervalWindow(pInfo, pSup, pInfo->pDelWins, pInfo->binfo.pRes->info.groupId, - pOperator->exprSupp.numOfExprs, pOperator->pTaskInfo, pUpdated); + doDeleteSpecifyIntervalWindow(&pChildInfo->aggSup, &pInfo->twAggSup, pBlock, NULL, &pChildInfo->interval, NULL); + rebuildIntervalWindow(pInfo, pSup, delWins, pInfo->binfo.pRes->info.groupId, pOperator->exprSupp.numOfExprs, + pOperator->pTaskInfo, pUpdatedMap); + addRetriveWindow(delWins, pInfo); + taosArrayAddAll(pInfo->pDelWins, delWins); + taosArrayDestroy(delWins); continue; } - removeResults(pInfo->pDelWins, pUpdatedMap); + removeResults(delWins, pUpdatedMap); + taosArrayAddAll(pInfo->pDelWins, delWins); + taosArrayDestroy(delWins); break; } else if (pBlock->info.type == STREAM_GET_ALL && IS_FINAL_OP(pInfo)) { getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); @@ -3259,7 +3158,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); } setInputDataBlock(pOperator, pSup->pCtx, pBlock, pInfo->order, MAIN_SCAN, true); - doHashInterval(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); + doHashIntervalAgg(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap); if (IS_FINAL_OP(pInfo)) { int32_t chIndex = getChildIndex(pBlock); int32_t size = taosArrayGetSize(pInfo->pChildren); @@ -3277,11 +3176,15 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); - doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); + doHashIntervalAgg(pChildOp, pBlock, pBlock->info.groupId, NULL); } + maxTs = TMAX(maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.watermark); + minTs = TMIN(minTs, pBlock->info.window.skey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); if (IS_FINAL_OP(pInfo)) { closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, pInfo->pPullDataMap, pUpdatedMap, pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); @@ -3309,6 +3212,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return pInfo->pPullDataRes; } + // we should send result first. doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pInfo->binfo.pRes->info.rows != 0) { printDataBlock(pInfo->binfo.pRes, IS_FINAL_OP(pInfo) ? "interval final" : "interval semi"); @@ -3332,40 +3236,6 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { return NULL; } -SSDataBlock* createSpecialDataBlock(EStreamType type) { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.hasVarCol = false; - pBlock->info.groupId = 0; - pBlock->info.rows = 0; - pBlock->info.type = type; - pBlock->info.rowSize = - sizeof(TSKEY) + sizeof(TSKEY) + sizeof(uint64_t) + sizeof(uint64_t) + sizeof(TSKEY) + sizeof(TSKEY); - pBlock->info.watermark = INT64_MIN; - - pBlock->pDataBlock = taosArrayInit(6, sizeof(SColumnInfoData)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_TIMESTAMP; - infoData.info.bytes = sizeof(TSKEY); - // window start ts - taosArrayPush(pBlock->pDataBlock, &infoData); - // window end ts - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_UBIGINT; - infoData.info.bytes = sizeof(uint64_t); - // uid - taosArrayPush(pBlock->pDataBlock, &infoData); - // group id - taosArrayPush(pBlock->pDataBlock, &infoData); - - // calculate start ts - taosArrayPush(pBlock->pDataBlock, &infoData); - // calculate end ts - taosArrayPush(pBlock->pDataBlock, &infoData); - - return pBlock; -} - SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { SIntervalPhysiNode* pIntervalPhyNode = (SIntervalPhysiNode*)pPhyNode; @@ -3387,6 +3257,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, .waterMark = pIntervalPhyNode->window.watermark, .calTrigger = pIntervalPhyNode->window.triggerType, .maxTs = INT64_MIN, + .minTs = INT64_MAX, }; ASSERT(pInfo->twAggSup.calTrigger != STREAM_TRIGGER_MAX_DELAY); pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -3470,7 +3341,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, createOperatorFpSet(NULL, doStreamFinalIntervalAgg, NULL, NULL, destroyStreamFinalIntervalOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (pPhyNode->type == QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL) { - initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup); + initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); } code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -3567,10 +3438,18 @@ void initDummyFunction(SqlFunctionCtx* pDummy, SqlFunctionCtx* pCtx, int32_t num } void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int64_t gap, int64_t waterMark, - uint16_t type) { - ASSERT(downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN); + uint16_t type, int32_t tsColIndex) { + if (downstream->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION) { + SStreamPartitionOperatorInfo* pScanInfo = downstream->info; + pScanInfo->tsColIndex = tsColIndex; + } + + if (downstream->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + initDownStream(downstream->pDownstream[0], pAggSup, gap, waterMark, type, tsColIndex); + return; + } SStreamScanInfo* pScanInfo = downstream->info; - pScanInfo->sessionSup = (SessionWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; + pScanInfo->windowSup = (SWindowSupporter){.pStreamAggSup = pAggSup, .gap = gap, .parentType = type}; pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } @@ -3622,7 +3501,11 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); pInfo->twAggSup = (STimeWindowAggSupp){ - .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, .maxTs = INT64_MIN}; + .waterMark = pSessionNode->window.watermark, + .calTrigger = pSessionNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + }; initResultRowInfo(&pInfo->binfo.resultRowInfo); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); @@ -3642,7 +3525,6 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->isFinal = false; pInfo->pPhyNode = pPhyNode; pInfo->ignoreExpiredData = pSessionNode->window.igExpired; - pInfo->returnDelete = false; pOperator->name = "StreamSessionWindowAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION; @@ -3653,7 +3535,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh createOperatorFpSet(operatorDummyOpenFn, doStreamSessionAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); if (downstream) { - initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType); + initDownStream(downstream, &pInfo->streamAggSup, pInfo->gap, pInfo->twAggSup.waterMark, pOperator->operatorType, + pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); } return pOperator; @@ -3683,13 +3566,15 @@ bool isInTimeWindow(STimeWindow* pWin, TSKEY ts, int64_t gap) { bool isInWindow(SResultWindowInfo* pWinInfo, TSKEY ts, int64_t gap) { return isInTimeWindow(&pWinInfo->win, ts, gap); } -static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY ts, int32_t index) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; +static SResultWindowInfo* insertNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs, int32_t index) { + SResultWindowInfo win = { + .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; return taosArrayInsert(pWinInfos, index, &win); } -static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { - SResultWindowInfo win = {.pos.offset = -1, .pos.pageId = -1, .win.skey = ts, .win.ekey = ts, .isOutput = false}; +static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY startTs, TSKEY endTs) { + SResultWindowInfo win = { + .pos.offset = -1, .pos.pageId = -1, .win.skey = startTs, .win.ekey = endTs, .isOutput = false}; return taosArrayPush(pWinInfos, &win); } @@ -3708,7 +3593,8 @@ SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { // don't add new window SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex) { - SArray* pWinInfos = getWinInfos(pAggSup, groupId); + STimeWindow searchWin = {.skey = startTs, .ekey = endTs}; + SArray* pWinInfos = getWinInfos(pAggSup, groupId); pAggSup->pCurWins = pWinInfos; int32_t size = taosArrayGetSize(pWinInfos); @@ -3720,7 +3606,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start SResultWindowInfo* pWin = NULL; if (index >= 0) { pWin = taosArrayGet(pWinInfos, index); - if (isInWindow(pWin, startTs, gap)) { + if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { *pIndex = index; return pWin; } @@ -3728,7 +3614,7 @@ SResultWindowInfo* getCurSessionWindow(SStreamAggSupporter* pAggSup, TSKEY start if (index + 1 < size) { pWin = taosArrayGet(pWinInfos, index + 1); - if (isInWindow(pWin, startTs, gap)) { + if (isInWindow(pWin, startTs, gap) || isInTimeWindow(&searchWin, pWin->win.skey, gap)) { *pIndex = index + 1; return pWin; } else if (endTs != INT64_MIN && isInWindow(pWin, endTs, gap)) { @@ -3748,7 +3634,7 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star int32_t size = taosArrayGetSize(pWinInfos); if (size == 0) { *pIndex = 0; - return addNewSessionWindow(pWinInfos, startTs); + return addNewSessionWindow(pWinInfos, startTs, endTs); } // find the first position which is smaller than the key int32_t index = binarySearch(pWinInfos, size, startTs, TSDB_ORDER_DESC, getSessionWindowEndkey); @@ -3774,10 +3660,10 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star if (index == size - 1) { *pIndex = taosArrayGetSize(pWinInfos); - return addNewSessionWindow(pWinInfos, startTs); + return addNewSessionWindow(pWinInfos, startTs, endTs); } *pIndex = index + 1; - return insertNewSessionWindow(pWinInfos, startTs, index + 1); + return insertNewSessionWindow(pWinInfos, startTs, endTs, index + 1); } int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, uint64_t groupId, @@ -3789,7 +3675,7 @@ int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TS if (pWinInfo->win.skey > pStartTs[i]) { if (pStDeleted && pWinInfo->isOutput) { SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; - taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey)); + taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); pWinInfo->isOutput = false; } pWinInfo->win.skey = pStartTs[i]; @@ -3817,7 +3703,6 @@ static int32_t setWindowOutputBuf(SResultWindowInfo* pWinInfo, SResultRow** pRes if (*pResult == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - initResultRow(*pResult); // add a new result set for a new group pWinInfo->pos.pageId = (*pResult)->pageId; @@ -3904,11 +3789,12 @@ void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, setWindowOutputBuf(pWinInfo, &pWinResult, pInfo->pDummyCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->streamAggSup, pTaskInfo); pCurWin->win.ekey = TMAX(pCurWin->win.ekey, pWinInfo->win.ekey); - compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pCurWin->win, true); + compactFunctions(pSup->pCtx, pInfo->pDummyCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); taosHashRemove(pStUpdated, &pWinInfo->pos, sizeof(SResultRowPosition)); - if (pWinInfo->isOutput) { + if (pWinInfo->isOutput && pStDeleted) { SWinKey res = {.ts = pWinInfo->win.skey, .groupId = groupId}; - taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey)); + taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); pWinInfo->isOutput = false; } taosArrayRemove(pInfo->streamAggSup.pCurWins, i); @@ -3999,18 +3885,24 @@ static void doDeleteTimeWindows(SStreamAggSupporter* pAggSup, SSDataBlock* pBloc SColumnInfoData* pGroupCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); uint64_t* gpDatas = (uint64_t*)pGroupCol->pData; for (int32_t i = 0; i < pBlock->info.rows; i++) { - int32_t winIndex = 0; - while (1) { - SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); - if (!pCurWin) { - break; - } + int32_t winIndex = 0; + SResultWindowInfo* pCurWin = getCurSessionWindow(pAggSup, startDatas[i], endDatas[i], gpDatas[i], gap, &winIndex); + if (!pCurWin) { + continue; + } + + do { + SResultWindowInfo delWin = *pCurWin; deleteWindow(pAggSup->pCurWins, winIndex, fp); if (result) { - pCurWin->groupId = gpDatas[i]; - taosArrayPush(result, pCurWin); + delWin.groupId = gpDatas[i]; + taosArrayPush(result, &delWin); } - } + if (winIndex >= taosArrayGetSize(pAggSup->pCurWins)) { + break; + } + pCurWin = taosArrayGet(pAggSup->pCurWins, winIndex); + } while (pCurWin->win.skey <= endDatas[i]); } } @@ -4033,6 +3925,7 @@ static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); if (result) { + pCurWin->groupId = gpCols[i]; taosArrayPush(result, pCurWin); } } @@ -4067,10 +3960,18 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It size_t keyLen = 0; while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SWinKey* res = *Ite; - SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); - colDataAppend(pTsCol, pBlock->info.rows, (const char*)&res->ts, false); + SColumnInfoData* pStartTsCol = taosArrayGet(pBlock->pDataBlock, START_TS_COLUMN_INDEX); + colDataAppend(pStartTsCol, pBlock->info.rows, (const char*)&res->ts, false); + SColumnInfoData* pEndTsCol = taosArrayGet(pBlock->pDataBlock, END_TS_COLUMN_INDEX); + colDataAppend(pEndTsCol, pBlock->info.rows, (const char*)&res->ts, false); + SColumnInfoData* pUidCol = taosArrayGet(pBlock->pDataBlock, UID_COLUMN_INDEX); + colDataAppendNULL(pUidCol, pBlock->info.rows); SColumnInfoData* pGpCol = taosArrayGet(pBlock->pDataBlock, GROUPID_COLUMN_INDEX); colDataAppend(pGpCol, pBlock->info.rows, (const char*)&res->groupId, false); + SColumnInfoData* pCalStCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_START_TS_COLUMN_INDEX); + colDataAppendNULL(pCalStCol, pBlock->info.rows); + SColumnInfoData* pCalEdCol = taosArrayGet(pBlock->pDataBlock, CALCULATE_END_TS_COLUMN_INDEX); + colDataAppendNULL(pCalEdCol, pBlock->info.rows); pBlock->info.rows += 1; if (pBlock->info.rows + 1 >= pBlock->info.capacity) { break; @@ -4081,20 +3982,17 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It } } -static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId, - int32_t numOfOutput, SOperatorInfo* pOperator) { +static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t numOfOutput, + SOperatorInfo* pOperator, SHashObj* pStUpdated) { SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - - int32_t size = taosArrayGetSize(pWinArray); + int32_t size = taosArrayGetSize(pWinArray); ASSERT(pInfo->pChildren); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pParentWin = taosArrayGet(pWinArray, i); - SResultRow* pCurResult = NULL; - setWindowOutputBuf(pParentWin, &pCurResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->streamAggSup, pTaskInfo); - int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); + uint64_t groupId = pParentWin->groupId; + int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChild = taosArrayGetP(pInfo->pChildren, j); SStreamSessionAggOperatorInfo* pChInfo = pChild->info; @@ -4107,22 +4005,36 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin for (int32_t k = index; k < chWinSize; k++) { SResultWindowInfo* pChWin = taosArrayGet(pChWins, k); if (pParentWin->win.skey <= pChWin->win.skey && pChWin->win.ekey <= pParentWin->win.ekey) { + int32_t winIndex = 0; + SResultWindowInfo* pNewParWin = + getSessionTimeWindow(&pInfo->streamAggSup, pChWin->win.skey, pChWin->win.ekey, groupId, 0, &winIndex); + SResultRow* pPareResult = NULL; + setWindowOutputBuf(pNewParWin, &pPareResult, pSup->pCtx, groupId, numOfOutput, pSup->rowEntryInfoOffset, + &pInfo->streamAggSup, pTaskInfo); SResultRow* pChResult = NULL; setWindowOutputBuf(pChWin, &pChResult, pChild->exprSupp.pCtx, groupId, numOfOutput, pChild->exprSupp.rowEntryInfoOffset, &pChInfo->streamAggSup, pTaskInfo); - compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pNewParWin->win, true); + compactFunctions(pSup->pCtx, pChild->exprSupp.pCtx, numOfOutput, pTaskInfo, &pInfo->twAggSup.timeWindowData); + + int32_t winNum = getNumCompactWindow(pInfo->streamAggSup.pCurWins, winIndex, pInfo->gap); + if (winNum > 0) { + compactTimeWindow(pInfo, winIndex, winNum, groupId, numOfOutput, pStUpdated, NULL, pOperator); + } + SFilePage* bufPage = getBufPage(pChInfo->streamAggSup.pResultBuf, pChWin->pos.pageId); releaseBufPage(pChInfo->streamAggSup.pResultBuf, bufPage); - continue; + + bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pNewParWin->pos.pageId); + setBufPageDirty(bufPage, true); + releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); + SWinKey value = {.ts = pNewParWin->win.skey, .groupId = groupId}; + taosHashPut(pStUpdated, &pNewParWin->pos, sizeof(SResultRowPosition), &value, sizeof(SWinKey)); } else if (!pChWin->isClosed) { break; } } } - SFilePage* bufPage = getBufPage(pInfo->streamAggSup.pResultBuf, pParentWin->pos.pageId); - ASSERT(size > 0); - setBufPageDirty(bufPage, true); - releaseBufPage(pInfo->streamAggSup.pResultBuf, bufPage); } } @@ -4198,18 +4110,57 @@ static void copyDeleteWindowInfo(SArray* pResWins, SHashObj* pStDeleted) { for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pWinInfo = taosArrayGet(pResWins, i); SWinKey res = {.ts = pWinInfo->win.skey, .groupId = pWinInfo->groupId}; - taosHashPut(pStDeleted, &pWinInfo->pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey)); + taosHashPut(pStDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); } } -static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; - SStreamSessionAggOperatorInfo* pInfo = pOperator->info; - SOptrBasicInfo* pBInfo = &pInfo->binfo; - TSKEY maxTs = INT64_MIN; - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } else if (pOperator->status == OP_RES_TO_RETURN) { +static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) { + int32_t size = taosArrayGetSize(pWins); + for (int32_t i = 0; i < size; i++) { + SResultWindowInfo* pWin = taosArrayGet(pWins, i); + taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition)); + } +} + +int32_t compareWinKey(void* pKey, void* data, int32_t index) { + SArray* res = (SArray*)data; + SResKeyPos* pos = taosArrayGetP(res, index); + SWinKey* pData = (SWinKey*)pKey; + if (pData->ts == *(int64_t*)pos->key) { + if (pData->groupId > pos->groupId) { + return 1; + } else if (pData->groupId < pos->groupId) { + return -1; + } + return 0; + } else if (pData->ts > *(int64_t*)pos->key) { + return 1; + } + return -1; +} + +static void removeSessionDeleteResults(SArray* update, SHashObj* pStDeleted) { + int32_t size = taosHashGetSize(pStDeleted); + if (size == 0) { + return; + } + + int32_t num = taosArrayGetSize(update); + for (int32_t i = 0; i < num; i++) { + SResKeyPos* pos = taosArrayGetP(update, i); + SWinKey winKey = {.ts = *(int64_t*)pos->key, .groupId = pos->groupId}; + taosHashRemove(pStDeleted, &winKey, sizeof(SWinKey)); + } +} + +static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { + SExprSupp* pSup = &pOperator->exprSupp; + SStreamSessionAggOperatorInfo* pInfo = pOperator->info; + SOptrBasicInfo* pBInfo = &pInfo->binfo; + TSKEY maxTs = INT64_MIN; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, IS_FINAL_OP(pInfo) ? "final session" : "single session"); @@ -4226,7 +4177,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); SHashObj* pStUpdated = taosHashInit(64, hashFn, true, HASH_NO_LOCK); SOperatorInfo* downstream = pOperator->pDownstream[0]; - SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); + SArray* pUpdated = taosArrayInit(16, POINTER_BYTES); // SResKeyPos while (1) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { @@ -4244,7 +4195,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; doClearSessionWindows(&pChildInfo->streamAggSup, &pChildOp->exprSupp, pBlock, START_TS_COLUMN_INDEX, pChildOp->exprSupp.numOfExprs, 0, NULL); - rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); + rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); } taosArrayDestroy(pWins); continue; @@ -4258,9 +4209,10 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pChildInfo = pChildOp->info; // gap must be 0 doDeleteTimeWindows(&pChildInfo->streamAggSup, pBlock, 0, NULL, NULL); - rebuildTimeWindow(pInfo, pWins, pBlock->info.groupId, pOperator->exprSupp.numOfExprs, pOperator); + rebuildTimeWindow(pInfo, pWins, pOperator->exprSupp.numOfExprs, pOperator, pStUpdated); } copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + removeSessionResults(pStUpdated, pWins); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -4303,6 +4255,7 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { pInfo->ignoreExpiredData, NULL); closeChildSessionWindow(pInfo->pChildren, pInfo->twAggSup.maxTs, pInfo->ignoreExpiredData, NULL); copyUpdateResult(pStUpdated, pUpdated); + removeSessionDeleteResults(pUpdated, pInfo->pStDeleted); taosHashCleanup(pStUpdated); finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); @@ -4333,14 +4286,6 @@ static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { pInfo->streamAggSup.currentPageId = -1; } -static void removeSessionResults(SHashObj* pHashMap, SArray* pWins) { - int32_t size = taosArrayGetSize(pWins); - for (int32_t i = 0; i < size; i++) { - SResultWindowInfo* pWin = taosArrayGet(pWins, i); - taosHashRemove(pHashMap, &pWin->pos, sizeof(SResultRowPosition)); - } -} - static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SStreamSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -4349,30 +4294,35 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; - } else if (pOperator->status == OP_RES_TO_RETURN) { + } + + { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows > 0) { printDataBlock(pBInfo->pRes, "semi session"); return pBInfo->pRes; } - // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); - if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) { - pInfo->returnDelete = true; + doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "semi session"); return pInfo->pDelRes; } - if (pInfo->pUpdateRes->info.rows > 0) { + if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; // process the rest of the data - pOperator->status = OP_OPENED; printDataBlock(pInfo->pUpdateRes, "semi session"); return pInfo->pUpdateRes; } - // semi interval operator clear disk buffer - clearStreamSessionOperator(pInfo); - pOperator->status = OP_EXEC_DONE; - return NULL; + + if (pOperator->status == OP_RES_TO_RETURN) { + clearFunctionContext(&pOperator->exprSupp); + // semi interval operator clear disk buffer + clearStreamSessionOperator(pInfo); + pOperator->status = OP_EXEC_DONE; + return NULL; + } } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); @@ -4383,6 +4333,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { clearSpecialDataBlock(pInfo->pUpdateRes); + pOperator->status = OP_RES_TO_RETURN; break; } printDataBlock(pBlock, "semi session recv"); @@ -4393,12 +4344,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { removeSessionResults(pStUpdated, pWins); taosArrayDestroy(pWins); copyDataBlock(pInfo->pUpdateRes, pBlock); + pInfo->returnUpdate = true; break; } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { // gap must be 0 - doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, NULL, NULL); - copyDataBlock(pInfo->pDelRes, pBlock); - pInfo->pDelRes->info.type = STREAM_DELETE_RESULT; + SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); + doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, NULL); + copyDeleteWindowInfo(pWins, pInfo->pStDeleted); + removeSessionResults(pStUpdated, pWins); + taosArrayDestroy(pWins); break; } else if (pBlock->info.type == STREAM_GET_ALL) { getAllSessionWindow(pInfo->streamAggSup.pResultRows, pUpdated, getResWinForSession); @@ -4411,18 +4365,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { } // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); - doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, pInfo->pStDeleted, false); + doStreamSessionAggImpl(pOperator, pBlock, pStUpdated, NULL, false); maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); } pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); pBInfo->pRes->info.watermark = pInfo->twAggSup.maxTs; - // restore the value - pOperator->status = OP_RES_TO_RETURN; - // semi operator - // closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - // getResWinForSession); + copyUpdateResult(pStUpdated, pUpdated); + removeSessionDeleteResults(pUpdated, pInfo->pStDeleted); taosHashCleanup(pStUpdated); finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, @@ -4436,16 +4387,15 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { return pBInfo->pRes; } - // doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); - if (pInfo->pDelRes->info.rows > 0 && !pInfo->returnDelete) { - pInfo->returnDelete = true; + doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); + if (pInfo->pDelRes->info.rows > 0) { printDataBlock(pInfo->pDelRes, "semi session"); return pInfo->pDelRes; } - if (pInfo->pUpdateRes->info.rows > 0) { + if (pInfo->pUpdateRes->info.rows > 0 && pInfo->returnUpdate) { + pInfo->returnUpdate = false; // process the rest of the data - pOperator->status = OP_OPENED; printDataBlock(pInfo->pUpdateRes, "semi session"); return pInfo->pUpdateRes; } @@ -4669,7 +4619,7 @@ int32_t updateStateWindowInfo(SArray* pWinInfos, int32_t winIndex, TSKEY* pTs, u if (pWinInfo->winInfo.win.skey > pTs[i]) { if (pSeDeleted && pWinInfo->winInfo.isOutput) { SWinKey res = {.ts = pWinInfo->winInfo.win.skey, .groupId = groupId}; - taosHashPut(pSeDeleted, &pWinInfo->winInfo.pos, sizeof(SResultRowPosition), &res, sizeof(SWinKey)); + taosHashPut(pSeDeleted, &res, sizeof(SWinKey), &res, sizeof(SWinKey)); pWinInfo->winInfo.isOutput = false; } pWinInfo->winInfo.win.skey = pTs[i]; @@ -4737,8 +4687,8 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, groupId, pKeyColInfo, pSDataBlock->info.rows, i, &allEqual, pStDeleted); if (!allEqual) { - appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, GROUPID_COLUMN_INDEX, - &groupId); + uint64_t uid = 0; + appendOneRow(pAggSup->pScanBlock, &pCurWin->winInfo.win.skey, &pCurWin->winInfo.win.ekey, &uid, &groupId); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); deleteWindow(pAggSup->pCurWins, winIndex, destroyStateWinInfo); continue; @@ -4767,6 +4717,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; + int64_t maxTs = INT64_MIN; if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pSeDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { @@ -4795,10 +4746,11 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_CLEAR) { doClearStateWindows(&pInfo->streamAggSup, pBlock, pSeUpdated, pInfo->pSeDeleted); continue; - } else if (pBlock->info.type == STREAM_DELETE_DATA) { + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); doDeleteTimeWindows(&pInfo->streamAggSup, pBlock, 0, pWins, destroyStateWinInfo); copyDeleteWindowInfo(pWins, pInfo->pSeDeleted); + removeSessionResults(pSeUpdated, pWins); taosArrayDestroy(pWins); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -4813,8 +4765,9 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // the pDataBlock are always the same one, no need to call this again setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamStateAggImpl(pOperator, pBlock, pSeUpdated, pInfo->pSeDeleted); - pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, pBlock->info.window.ekey); + maxTs = TMAX(maxTs, pBlock->info.window.ekey); } + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); // restore the value pOperator->status = OP_RES_TO_RETURN; @@ -4877,6 +4830,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys .waterMark = pStateNode->window.watermark, .calTrigger = pStateNode->window.triggerType, .maxTs = INT64_MIN, + .minTs = INT64_MAX, }; initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); @@ -4913,7 +4867,8 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pOperator->info = pInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doStreamStateAgg, NULL, NULL, destroyStreamStateOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); - initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType); + initDownStream(downstream, &pInfo->streamAggSup, 0, pInfo->twAggSup.waterMark, pOperator->operatorType, + pInfo->primaryTsIndex); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -4927,72 +4882,65 @@ _error: return NULL; } -void destroyMergeAlignedIntervalOperatorInfo(void* param) { +void destroyMAIOperatorInfo(void* param) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param; destroyIntervalOperatorInfo(miaInfo->intervalAggOperatorInfo); taosMemoryFreeClear(param); } -static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, - SSDataBlock* pResultBlock, TSKEY wstartTs) { - SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; - - SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo; - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; - - SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, - GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - ASSERT(p1 != NULL); +static SResultRow* doSetSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, SAggSupporter* pSup) { + SResultRow* pResult = getNewResultRow(pSup->pResultBuf, &pSup->currentPageId, pSup->resultRowSize); + pResultRowInfo->cur = (SResultRowPosition){.pageId = pResult->pageId, .offset = pResult->offset}; + return pResult; +} - finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs, - pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); - tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); +static int32_t setSingleOutputTupleBuf(SResultRowInfo* pResultRowInfo, STimeWindow* win, SResultRow** pResult, + SExprSupp* pExprSup, SAggSupporter* pAggSup) { + if (*pResult == NULL) { + *pResult = doSetSingleOutputTupleBuf(pResultRowInfo, pAggSup); + if (*pResult == NULL) { + return terrno; + } + } + // set time window for current result + (*pResult)->win = (*win); + setResultRowInitCtx((*pResult), pExprSup->pCtx, pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset); return TSDB_CODE_SUCCESS; } static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, - SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) { + SSDataBlock* pBlock, SSDataBlock* pResultBlock) { SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; + SInterval* pInterval = &iaInfo->interval; - int32_t startPos = 0; - int32_t numOfOutput = pSup->numOfExprs; - int64_t* tsCols = extractTsCol(pBlock, iaInfo); - uint64_t tableGroupId = pBlock->info.groupId; - SResultRow* pResult = NULL; + int32_t startPos = 0; + int64_t* tsCols = extractTsCol(pBlock, iaInfo); TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); // there is an result exists if (miaInfo->curTs != INT64_MIN) { - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - if (ts != miaInfo->curTs) { - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); + finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo); + resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow)); miaInfo->curTs = ts; } } else { miaInfo->curTs = ts; - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 0); } STimeWindow win = {0}; win.skey = miaInfo->curTs; - win.ekey = - taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; + win.ekey = taosTimeAdd(win.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; - // TODO: remove the hash table (groupid + winkey => result row position) - int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, - pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + int32_t ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) { + T_LONG_JMP(pTaskInfo->env, ret); } int32_t currPos = startPos; @@ -5005,21 +4953,19 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, - pBlock->info.rows, numOfOutput); + pBlock->info.rows, pSup->numOfExprs); - outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, miaInfo->curTs); + finalizeResultRows(iaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pResultBlock, pTaskInfo); + resetResultRow(miaInfo->pResultRow, iaInfo->aggSup.resultRowSize - sizeof(SResultRow)); miaInfo->curTs = tsCols[currPos]; currWin.skey = miaInfo->curTs; - currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, - iaInfo->interval.precision) - - 1; + currWin.ekey = taosTimeAdd(currWin.skey, pInterval->interval, pInterval->intervalUnit, pInterval->precision) - 1; startPos = currPos; - ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); - if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { - T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + ret = setSingleOutputTupleBuf(pResultRowInfo, &win, &miaInfo->pResultRow, pSup, &iaInfo->aggSup); + if (ret != TSDB_CODE_SUCCESS || miaInfo->pResultRow == NULL) { + T_LONG_JMP(pTaskInfo->env, ret); } miaInfo->curTs = currWin.skey; @@ -5027,68 +4973,79 @@ static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultR updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); doApplyFunctions(pTaskInfo, pSup->pCtx, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, - pBlock->info.rows, numOfOutput); + pBlock->info.rows, pSup->numOfExprs); +} + +static void cleanupAfterGroupResultGen(SMergeAlignedIntervalAggOperatorInfo* pMiaInfo, SSDataBlock* pRes) { + pRes->info.groupId = pMiaInfo->groupId; + pMiaInfo->curTs = INT64_MIN; + pMiaInfo->groupId = 0; } static void doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info; - SIntervalAggOperatorInfo* iaInfo = miaInfo->intervalAggOperatorInfo; - - SExprSupp* pSup = &pOperator->exprSupp; - SSDataBlock* pRes = iaInfo->binfo.pRes; + SMergeAlignedIntervalAggOperatorInfo* pMiaInfo = pOperator->info; + SIntervalAggOperatorInfo* pIaInfo = pMiaInfo->intervalAggOperatorInfo; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - int32_t scanFlag = MAIN_SCAN; + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pRes = pIaInfo->binfo.pRes; + SResultRowInfo* pResultRowInfo = &pIaInfo->binfo.resultRowInfo; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; while (1) { SSDataBlock* pBlock = NULL; - if (miaInfo->prefetchedBlock == NULL) { + if (pMiaInfo->prefetchedBlock == NULL) { pBlock = downstream->fpSet.getNextFn(downstream); } else { - pBlock = miaInfo->prefetchedBlock; - miaInfo->prefetchedBlock = NULL; + pBlock = pMiaInfo->prefetchedBlock; + pMiaInfo->prefetchedBlock = NULL; - miaInfo->groupId = pBlock->info.groupId; + pMiaInfo->groupId = pBlock->info.groupId; } + // no data exists, all query processing is done if (pBlock == NULL) { - // close last unfinalized time window - if (miaInfo->curTs != INT64_MIN) { - ASSERT(tSimpleHashGetSize(iaInfo->aggSup.pResultRowHashTable) == 1); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->curTs = INT64_MIN; + // close last unclosed time window + if (pMiaInfo->curTs != INT64_MIN) { + finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo); + resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow)); + cleanupAfterGroupResultGen(pMiaInfo, pRes); } doSetOperatorCompleted(pOperator); break; } - if (!miaInfo->hasGroupId) { - miaInfo->hasGroupId = true; - miaInfo->groupId = pBlock->info.groupId; - } else if (miaInfo->groupId != pBlock->info.groupId) { - // if there are unclosed time window, close it firstly. - ASSERT(miaInfo->curTs != INT64_MIN); - outputMergeAlignedIntervalResult(pOperator, miaInfo->groupId, pRes, miaInfo->curTs); - miaInfo->prefetchedBlock = pBlock; - miaInfo->curTs = INT64_MIN; - break; + if (pMiaInfo->groupId == 0) { + if (pMiaInfo->groupId != pBlock->info.groupId) { + pMiaInfo->groupId = pBlock->info.groupId; + } + } else { + if (pMiaInfo->groupId != pBlock->info.groupId) { + // if there are unclosed time window, close it firstly. + ASSERT(pMiaInfo->curTs != INT64_MIN); + finalizeResultRows(pIaInfo->aggSup.pResultBuf, &pResultRowInfo->cur, pSup, pRes, pTaskInfo); + resetResultRow(pMiaInfo->pResultRow, pIaInfo->aggSup.resultRowSize - sizeof(SResultRow)); + + pMiaInfo->prefetchedBlock = pBlock; + cleanupAfterGroupResultGen(pMiaInfo, pRes); + break; + } else { + // continue + } } - getTableScanInfo(pOperator, &iaInfo->inputOrder, &scanFlag); - setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->inputOrder, scanFlag, true); - doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + getTableScanInfo(pOperator, &pIaInfo->inputOrder, &scanFlag); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, pIaInfo->inputOrder, scanFlag, true); + doMergeAlignedIntervalAggImpl(pOperator, &pIaInfo->binfo.resultRowInfo, pBlock, pRes); - doFilter(miaInfo->pCondition, pRes, NULL); + doFilter(pMiaInfo->pCondition, pRes, NULL); if (pRes->info.rows >= pOperator->resultInfo.capacity) { break; } } - - pRes->info.groupId = miaInfo->groupId; - miaInfo->hasGroupId = false; } static SSDataBlock* mergeAlignedIntervalAgg(SOperatorInfo* pOperator) { @@ -5152,11 +5109,11 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprSupp* pSup = &pOperator->exprSupp; miaInfo->pCondition = pNode->window.node.pConditions; - miaInfo->curTs = INT64_MIN; - iaInfo->win = pTaskInfo->window; - iaInfo->inputOrder = TSDB_ORDER_ASC; - iaInfo->interval = interval; - iaInfo->execModel = pTaskInfo->execModel; + miaInfo->curTs = INT64_MIN; + iaInfo->win = pTaskInfo->window; + iaInfo->inputOrder = TSDB_ORDER_ASC; + iaInfo->interval = interval; + iaInfo->execModel = pTaskInfo->execModel; iaInfo->primaryTsIndex = ((SColumnNode*)pNode->window.pTspk)->slotId; iaInfo->binfo.mergeResultBlock = pNode->window.mergeDataBlock; @@ -5179,15 +5136,15 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&iaInfo->binfo.resultRowInfo); blockDataEnsureCapacity(iaInfo->binfo.pRes, pOperator->resultInfo.capacity); - pOperator->name = "TimeMergeAlignedIntervalAggOperator"; + pOperator->name = "TimeMergeAlignedIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = miaInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = miaInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, mergeAlignedIntervalAgg, NULL, NULL, - destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL); + destroyMAIOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); if (code != TSDB_CODE_SUCCESS) { @@ -5197,7 +5154,7 @@ SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, return pOperator; _error: - destroyMergeAlignedIntervalOperatorInfo(miaInfo); + destroyMAIOperatorInfo(miaInfo); taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; @@ -5237,11 +5194,10 @@ static int32_t finalizeWindowResult(SOperatorInfo* pOperatorInfo, uint64_t table SExprSupp* pExprSup = &pOperatorInfo->exprSupp; SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &win->skey, TSDB_KEYSIZE, tableGroupId); - SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, - GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + SResultRowPosition* p1 = (SResultRowPosition*)tSimpleHashGet( + iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); - finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, - pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); +// finalizeResultRows(iaInfo->aggSup.pResultBuf, p1, pResultBlock, pTaskInfo); tSimpleHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); return TSDB_CODE_SUCCESS; } @@ -5250,9 +5206,7 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t STimeWindow* newWin) { SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; bool ascScan = (iaInfo->inputOrder == TSDB_ORDER_ASC); - SExprSupp* pExprSup = &pOperatorInfo->exprSupp; SGroupTimeWindow groupTimeWindow = {.groupId = tableGroupId, .window = *newWin}; tdListAppend(miaInfo->groupIntervals, &groupTimeWindow); @@ -5265,9 +5219,10 @@ static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t t if (prevGrpWin->groupId != tableGroupId) { continue; } + STimeWindow* prevWin = &prevGrpWin->window; if ((ascScan && newWin->skey > prevWin->ekey) || ((!ascScan) && newWin->skey < prevWin->ekey)) { - finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); +// finalizeWindowResult(pOperatorInfo, tableGroupId, prevWin, pResultBlock); tdListPopNode(miaInfo->groupIntervals, listNode); } } @@ -5427,7 +5382,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { if (listNode != NULL) { SGroupTimeWindow* grpWin = (SGroupTimeWindow*)(listNode->data); - finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); +// finalizeWindowResult(pOperator, grpWin->groupId, &grpWin->window, pRes); pRes->info.groupId = grpWin->groupId; } } @@ -5463,10 +5418,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge pMergeIntervalInfo->groupIntervals = tdListNew(sizeof(SGroupTimeWindow)); SIntervalAggOperatorInfo* pIntervalInfo = &pMergeIntervalInfo->intervalAggOperatorInfo; - pIntervalInfo->win = pTaskInfo->window; + pIntervalInfo->win = pTaskInfo->window; pIntervalInfo->inputOrder = TSDB_ORDER_ASC; - pIntervalInfo->interval = interval; - pIntervalInfo->execModel = pTaskInfo->execModel; + pIntervalInfo->interval = interval; + pIntervalInfo->execModel = pTaskInfo->execModel; pIntervalInfo->binfo.mergeResultBlock = pIntervalPhyNode->window.mergeDataBlock; pIntervalInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; @@ -5483,7 +5438,6 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initBasicInfo(&pIntervalInfo->binfo, pResBlock); initExecTimeWindowInfo(&pIntervalInfo->twAggSup.timeWindowData, &pIntervalInfo->win); - pIntervalInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, num, pIntervalInfo); if (pIntervalInfo->timeWindowInterpo) { pIntervalInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SOpenWindowInfo)); @@ -5494,12 +5448,12 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SMerge initResultRowInfo(&pIntervalInfo->binfo.resultRowInfo); - pOperator->name = "TimeMergeIntervalAggOperator"; + pOperator->name = "TimeMergeIntervalAggOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->pTaskInfo = pTaskInfo; - pOperator->info = pMergeIntervalInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->pTaskInfo = pTaskInfo; + pOperator->info = pMergeIntervalInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeIntervalAgg, NULL, NULL, destroyMergeIntervalOperatorInfo, NULL, NULL, NULL); @@ -5517,3 +5471,402 @@ _error: pTaskInfo->code = code; return NULL; } + +static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag, SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + + int32_t startPos = 0; + int32_t numOfOutput = pSup->numOfExprs; + SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, pInfo->primaryTsIndex); + TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = true; + TSKEY ts = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; + + STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + int32_t ret = TSDB_CODE_SUCCESS; + if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && + inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); + } + } + + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + ASSERT(forwardRows > 0); + + if ((!pInfo->ignoreExpiredData || !isCloseWindow(&win, &pInfo->twAggSup)) && + inSlidingWindow(&pInfo->interval, &win, &pBlock->info)) { + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, + numOfOutput); + } + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = forwardRows - 1 + startPos; + startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + if (pInfo->ignoreExpiredData && isCloseWindow(&nextWin, &pInfo->twAggSup)) { + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + continue; + } + + // null data, failed to allocate more memory buffer + int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + setResultBufPageDirty(pInfo->aggSup.pResultBuf, &pResultRowInfo->cur); + } + + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, TSDB_ORDER_ASC); + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pBlock->info.rows, + numOfOutput); + } +} + +static void doStreamIntervalAggImpl2(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t tableGroupId, + SHashObj* pUpdatedMap) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + + SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pSup = &pOperatorInfo->exprSupp; + int32_t numOfOutput = pSup->numOfExprs; + int32_t step = 1; + TSKEY* tsCols = NULL; + SResultRow* pResult = NULL; + int32_t forwardRows = 0; + int32_t aa = 4; + + ASSERT(pSDataBlock->pDataBlock != NULL); + SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); + tsCols = (int64_t*)pColDataInfo->pData; + + int32_t startPos = 0; + TSKEY ts = getStartTsKey(&pSDataBlock->info.window, tsCols); + STimeWindow nextWin = + getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, TSDB_ORDER_ASC); + while (1) { + bool isClosed = isCloseWindow(&nextWin, &pInfo->twAggSup); + if ((pInfo->ignoreExpiredData && isClosed) || !inSlidingWindow(&pInfo->interval, &nextWin, &pSDataBlock->info)) { + startPos = getNexWindowPos(&pInfo->interval, &pSDataBlock->info, tsCols, startPos, nextWin.ekey, &nextWin); + if (startPos < 0) { + break; + } + continue; + } + + int32_t code = setOutputBuf(&nextWin, &pResult, tableGroupId, pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, + &pInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + T_LONG_JMP(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + forwardRows = getNumOfRowsInTimeWindow(&pSDataBlock->info, tsCols, startPos, nextWin.ekey, binarySearchForKey, NULL, + TSDB_ORDER_ASC); + if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE && pUpdatedMap) { + saveWinResultRow(pResult, tableGroupId, pUpdatedMap); + } + updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, + pSDataBlock->info.rows, numOfOutput); + SWinKey key = { + .ts = nextWin.skey, + .groupId = tableGroupId, + }; + saveOutput(pTaskInfo, &key, pResult, pInfo->aggSup.resultRowSize); + releaseOutputBuf(pTaskInfo, &key, pResult); + int32_t prevEndPos = (forwardRows - 1) * step + startPos; + ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); + startPos = + getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, TSDB_ORDER_ASC); + if (startPos < 0) { + break; + } + } +} + +void doBuildResult(SOperatorInfo* pOperator, SSDataBlock* pBlock, SGroupResInfo* pGroupResInfo) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + // set output datablock version + pBlock->info.version = pTaskInfo->version; + + blockDataCleanup(pBlock); + if (!hasRemainResults(pGroupResInfo)) { + return; + } + + // clear the existed group id + pBlock->info.groupId = 0; + buildDataBlockFromGroupRes(pTaskInfo, pBlock, &pOperator->exprSupp, pGroupResInfo); +} + +static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { + SStreamIntervalOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + int64_t maxTs = INT64_MIN; + int64_t minTs = INT64_MAX; + SExprSupp* pSup = &pOperator->exprSupp; + + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + if (pOperator->status == OP_RES_TO_RETURN) { + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "single interval"); + return pInfo->pDelRes; + } + + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + if (pInfo->binfo.pRes->info.rows == 0 || !hasRemainResults(&pInfo->groupResInfo)) { + pOperator->status = OP_EXEC_DONE; + qDebug("===stream===single interval is done"); + freeAllPages(pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + } + printDataBlock(pInfo->binfo.pRes, "single interval"); + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; + } + + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + SArray* pUpdated = taosArrayInit(4, POINTER_BYTES); // SResKeyPos + _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); + SHashObj* pUpdatedMap = taosHashInit(1024, hashFn, false, HASH_NO_LOCK); + + SStreamState* pState = pTaskInfo->streamInfo.pState; + + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + printDataBlock(pBlock, "single interval recv"); + + if (pBlock->info.type == STREAM_CLEAR) { + doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, pOperator->exprSupp.numOfExprs, pBlock, + NULL); + qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); + continue; + } else if (pBlock->info.type == STREAM_DELETE_DATA || pBlock->info.type == STREAM_DELETE_RESULT) { + doDeleteSpecifyIntervalWindow(&pInfo->aggSup, &pInfo->twAggSup, pBlock, pInfo->pDelWins, &pInfo->interval, + pUpdatedMap); + continue; + } else if (pBlock->info.type == STREAM_GET_ALL) { + getAllIntervalWindow(pInfo->aggSup.pResultRowHashTable, pUpdatedMap); + continue; + } + + if (pBlock->info.type == STREAM_NORMAL && pBlock->info.version != 0) { + // set input version + pTaskInfo->version = pBlock->info.version; + } + + if (pInfo->scalarSupp.pExprInfo != NULL) { + SExprSupp* pExprSup = &pInfo->scalarSupp; + projectApplyFunctions(pExprSup->pExprInfo, pBlock, pBlock, pExprSup->pCtx, pExprSup->numOfExprs, NULL); + } + + // The timewindow that overlaps the timestamps of the input pBlock need to be recalculated and return to the + // caller. Note that all the time window are not close till now. + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pSup->pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); + if (pInfo->invertible) { + setInverFunction(pSup->pCtx, pOperator->exprSupp.numOfExprs, pBlock->info.type); + } + + maxTs = TMAX(maxTs, pBlock->info.window.ekey); + minTs = TMIN(minTs, pBlock->info.window.skey); + doStreamIntervalAggImpl(pOperator, &pInfo->binfo.resultRowInfo, pBlock, MAIN_SCAN, pUpdatedMap); + // new disc buf + /*doStreamIntervalAggImpl2(pOperator, pBlock, pBlock->info.groupId, pUpdatedMap);*/ + } + pInfo->twAggSup.maxTs = TMAX(pInfo->twAggSup.maxTs, maxTs); + pInfo->twAggSup.minTs = TMIN(pInfo->twAggSup.minTs, minTs); + +#if 0 + if (pState) { + printf(">>>>>>>> stream read backend\n"); + SWinKey key = { + .ts = 1, + .groupId = 2, + }; + char* val = NULL; + int32_t sz; + if (streamStateGet(pState, &key, (void**)&val, &sz) < 0) { + ASSERT(0); + } + printf("stream read %s %d\n", val, sz); + streamFreeVal(val); + + SStreamStateCur* pCur = streamStateGetCur(pState, &key); + ASSERT(pCur); + while (streamStateCurNext(pState, pCur) == 0) { + SWinKey key1; + const void* val1; + if (streamStateGetKVByCur(pCur, &key1, &val1, &sz) < 0) { + break; + } + printf("stream iter key groupId:%d ts:%d, value %s %d\n", key1.groupId, key1.ts, val1, sz); + } + streamStateFreeCur(pCur); + } +#endif + + pOperator->status = OP_RES_TO_RETURN; + closeIntervalWindow(pInfo->aggSup.pResultRowHashTable, &pInfo->twAggSup, &pInfo->interval, NULL, pUpdatedMap, + pInfo->pRecycledPages, pInfo->aggSup.pResultBuf); + + void* pIte = NULL; + while ((pIte = taosHashIterate(pUpdatedMap, pIte)) != NULL) { + taosArrayPush(pUpdated, pIte); + } + taosArraySort(pUpdated, resultrowComparAsc); + + // new disc buf + finalizeUpdatedResult(pOperator->exprSupp.numOfExprs, pInfo->aggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); + initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); + blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); + removeDeleteResults(pUpdatedMap, pInfo->pDelWins); + taosHashCleanup(pUpdatedMap); + doBuildDeleteResult(pInfo->pDelWins, &pInfo->delIndex, pInfo->pDelRes); + if (pInfo->pDelRes->info.rows > 0) { + printDataBlock(pInfo->pDelRes, "single interval"); + return pInfo->pDelRes; + } + + doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); + // new disc buf + // doBuildResult(pOperator, pInfo->binfo.pRes, &pInfo->groupResInfo); + printDataBlock(pInfo->binfo.pRes, "single interval"); + return pInfo->binfo.pRes->info.rows == 0 ? NULL : pInfo->binfo.pRes; +} + +void destroyStreamIntervalOperatorInfo(void* param) { + SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)param; + cleanupBasicInfo(&pInfo->binfo); + cleanupAggSup(&pInfo->aggSup); + pInfo->pRecycledPages = taosArrayDestroy(pInfo->pRecycledPages); + + pInfo->pDelWins = taosArrayDestroy(pInfo->pDelWins); + pInfo->pDelRes = blockDataDestroy(pInfo->pDelRes); + + cleanupGroupResInfo(&pInfo->groupResInfo); + colDataDestroy(&pInfo->twAggSup.timeWindowData); + taosMemoryFreeClear(param); +} + +SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SStreamIntervalOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamIntervalOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL) { + goto _error; + } + SStreamIntervalPhysiNode* pIntervalPhyNode = (SStreamIntervalPhysiNode*)pPhyNode; + + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &numOfCols); + ASSERT(numOfCols > 0); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + SInterval interval = { + .interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision, + }; + STimeWindowAggSupp twAggSupp = { + .waterMark = pIntervalPhyNode->window.watermark, + .calTrigger = pIntervalPhyNode->window.triggerType, + .maxTs = INT64_MIN, + .minTs = INT64_MAX, + }; + ASSERT(twAggSupp.calTrigger != STREAM_TRIGGER_MAX_DELAY); + pOperator->pTaskInfo = pTaskInfo; + pInfo->interval = interval; + pInfo->twAggSup = twAggSupp; + pInfo->ignoreExpiredData = pIntervalPhyNode->window.igExpired; + pInfo->isFinal = false; + + if (pIntervalPhyNode->window.pExprs != NULL) { + int32_t numOfScalar = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pIntervalPhyNode->window.pExprs, NULL, &numOfScalar); + int32_t code = initExprSupp(&pInfo->scalarSupp, pScalarExprInfo, numOfScalar); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } + + pInfo->primaryTsIndex = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + initResultSizeInfo(&pOperator->resultInfo, 4096); + SExprSupp* pSup = &pOperator->exprSupp; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + int32_t code = initAggInfo(pSup, &pInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initBasicInfo(&pInfo->binfo, pResBlock); + initStreamFunciton(pSup->pCtx, pSup->numOfExprs); + initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); + + pInfo->invertible = allInvertible(pSup->pCtx, numOfCols); + pInfo->invertible = false; // Todo(liuyao): Dependent TSDB API + pInfo->pRecycledPages = taosArrayInit(4, sizeof(int32_t)); + pInfo->pDelWins = taosArrayInit(4, sizeof(SWinKey)); + pInfo->delIndex = 0; + pInfo->pDelRes = createSpecialDataBlock(STREAM_DELETE_RESULT); + initResultRowInfo(&pInfo->binfo.resultRowInfo); + + pOperator->name = "StreamIntervalOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doStreamIntervalAgg, NULL, NULL, destroyStreamIntervalOperatorInfo, + aggEncodeResultRow, aggDecodeResultRow, NULL); + + initIntervalDownStream(downstream, pPhyNode->type, &pInfo->aggSup, &pInfo->interval, pInfo->twAggSup.waterMark); + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyStreamIntervalOperatorInfo(pInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} diff --git a/source/libs/executor/src/tsimplehash.c b/source/libs/executor/src/tsimplehash.c index 84b615af7a93aef9fbf86190a2544474b7b2c87b..16fd11f97d330fa3bc2622e2f7671fa532dce61f 100644 --- a/source/libs/executor/src/tsimplehash.c +++ b/source/libs/executor/src/tsimplehash.c @@ -247,8 +247,9 @@ void *tSimpleHashGet(SSHashObj *pHashObj, const void *key, size_t keyLen) { } int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) { + int32_t code = TSDB_CODE_FAILED; if (!pHashObj || !key) { - return TSDB_CODE_FAILED; + return code; } uint32_t hashVal = (*pHashObj->hashFp)(key, (uint32_t)keyLen); @@ -266,13 +267,14 @@ int32_t tSimpleHashRemove(SSHashObj *pHashObj, const void *key, size_t keyLen) { } FREE_HASH_NODE(pNode); atomic_sub_fetch_64(&pHashObj->size, 1); + code = TSDB_CODE_SUCCESS; break; } pPrev = pNode; pNode = pNode->next; } - return TSDB_CODE_SUCCESS; + return code; } int32_t tSimpleHashIterateRemove(SSHashObj *pHashObj, const void *key, size_t keyLen, void **pIter, int32_t *iter) { diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 168cd21c4478d9c1b50053fadf0e9dcdf518d4f4..63fc9d9e1c553bc210c7f525b014cd3d0b4f852b 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -227,9 +227,9 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int continue; } - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); - void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + void* pPage = getBufPage(pHandle->pBuf, *pPgId); code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; @@ -302,9 +302,9 @@ static int32_t adjustMergeTreeForNextTuple(SSortSource *pSource, SMultiwayMergeT pSource->pageIndex = -1; pSource->src.pBlock = blockDataDestroy(pSource->src.pBlock); } else { - SPageInfo* pPgInfo = *(SPageInfo**)taosArrayGet(pSource->pageIdList, pSource->pageIndex); + int32_t* pPgId = taosArrayGet(pSource->pageIdList, pSource->pageIndex); - void* pPage = getBufPage(pHandle->pBuf, getPageId(pPgInfo)); + void* pPage = getBufPage(pHandle->pBuf, *pPgId); int32_t code = blockDataFromBuf(pSource->src.pBlock, pPage); if (code != TSDB_CODE_SUCCESS) { return code; diff --git a/source/libs/function/CMakeLists.txt b/source/libs/function/CMakeLists.txt index ea401e56e5c6585b93344af99280bb450137f98f..dd048a047aeafdddce06dd0841939afe20bfd3d9 100644 --- a/source/libs/function/CMakeLists.txt +++ b/source/libs/function/CMakeLists.txt @@ -14,7 +14,7 @@ target_include_directories( target_link_libraries( function - PRIVATE os util common nodes scalar qcom transport + PRIVATE os util common nodes scalar qcom transport stream PUBLIC uv_a ) diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index 648ae5a538caed9fdb896bc4f1e6fe7537a7948a..f3d3393ac3fa122c60cddcd19373011c5c180495 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -311,22 +311,6 @@ static int32_t translateInOutStr(SFunctionNode* pFunc, char* pErrBuf, int32_t le return TSDB_CODE_SUCCESS; } -static int32_t translateMinMax(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { - return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); - } - - uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_TIMESTAMP_TYPE(paraType) && !IS_NUMERIC_TYPE(paraType) && !IS_NULL_TYPE(paraType)) { - return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); - } else if (IS_NULL_TYPE(paraType)) { - paraType = TSDB_DATA_TYPE_BIGINT; - } - - pFunc->node.resType = (SDataType){.bytes = tDataTypes[paraType].bytes, .type = paraType}; - return TSDB_CODE_SUCCESS; -} - static int32_t translateTrimStr(SFunctionNode* pFunc, char* pErrBuf, int32_t len, bool isLtrim) { if (1 != LIST_LENGTH(pFunc->pParameterList)) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); @@ -2076,7 +2060,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "min", .type = FUNCTION_TYPE_MIN, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, - .translateFunc = translateMinMax, + .translateFunc = translateInOutNum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getMinmaxFuncEnv, .initFunc = minmaxFunctionSetup, @@ -2091,7 +2075,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .name = "max", .type = FUNCTION_TYPE_MAX, .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_SPECIAL_DATA_REQUIRED | FUNC_MGT_SELECT_FUNC, - .translateFunc = translateMinMax, + .translateFunc = translateInOutNum, .dataRequiredFunc = statisDataRequired, .getEnvFunc = getMinmaxFuncEnv, .initFunc = minmaxFunctionSetup, @@ -2175,6 +2159,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .type = FUNCTION_TYPE_AVG_PARTIAL, .classification = FUNC_MGT_AGG_FUNC, .translateFunc = translateAvgPartial, + .dataRequiredFunc = statisDataRequired, .getEnvFunc = getAvgFuncEnv, .initFunc = avgFunctionSetup, .processFunc = avgFunction, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 7160541c13538eb7f8e2a5f7cd0d00aaa237ad8d..9b502eded7d6911c2f8e452e49c1e27c34b9fdcb 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -18,6 +18,7 @@ #include "function.h" #include "query.h" #include "querynodes.h" +#include "streamState.h" #include "tcompare.h" #include "tdatablock.h" #include "tdigest.h" @@ -56,8 +57,13 @@ typedef struct SAvgRes { } SAvgRes; typedef struct STuplePos { - int32_t pageId; - int32_t offset; + union { + struct { + int32_t pageId; + int32_t offset; + }; + STupleKey streamTupleKey; + }; } STuplePos; typedef struct SMinmaxResInfo { @@ -76,11 +82,11 @@ typedef struct STopBotResItem { } STopBotResItem; typedef struct STopBotRes { - int32_t maxSize; - int16_t type; + int32_t maxSize; + int16_t type; - STuplePos nullTuplePos; - bool nullTupleSaved; + STuplePos nullTuplePos; + bool nullTupleSaved; STopBotResItem* pItems; } STopBotRes; @@ -223,14 +229,14 @@ typedef struct SMavgInfo { } SMavgInfo; typedef struct SSampleInfo { - int32_t samples; - int32_t totalPoints; - int32_t numSampled; - uint8_t colType; - int16_t colBytes; + int32_t samples; + int32_t totalPoints; + int32_t numSampled; + uint8_t colType; + int16_t colBytes; - STuplePos nullTuplePos; - bool nullTupleSaved; + STuplePos nullTuplePos; + bool nullTupleSaved; char* data; STuplePos* tuplePos; @@ -1146,8 +1152,9 @@ bool getMinmaxFuncEnv(SFunctionNode* UNUSED_PARAM(pFunc), SFuncExecEnv* pEnv) { return true; } -static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock); -static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); +static STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, + const STupleKey* pKey); +static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos); static const char* loadTupleData(SqlFunctionCtx* pCtx, const STuplePos* pPos); static int32_t findRowIndex(int32_t start, int32_t num, SColumnInfoData* pCol, const char* tval) { @@ -1201,10 +1208,10 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { pBuf->v = *(int64_t*)tval; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); } } else { - if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) { + if (IS_SIGNED_NUMERIC_TYPE(type)) { int64_t prev = 0; GET_TYPED_DATA(prev, int64_t, type, &pBuf->v); @@ -1213,7 +1220,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { *(int64_t*)&pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); } } } else if (IS_UNSIGNED_NUMERIC_TYPE(type)) { @@ -1225,7 +1232,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { *(uint64_t*)&pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); } } } else if (type == TSDB_DATA_TYPE_DOUBLE) { @@ -1237,7 +1244,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { *(double*)&pBuf->v = val; if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); } } } else if (type == TSDB_DATA_TYPE_FLOAT) { @@ -1251,7 +1258,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (pCtx->subsidiaries.num > 0) { index = findRowIndex(pInput->startRowIndex, pInput->numOfRows, pCol, tval); - pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); } } } @@ -1263,7 +1270,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { + if (IS_SIGNED_NUMERIC_TYPE(type) || type == TSDB_DATA_TYPE_BOOL) { if (type == TSDB_DATA_TYPE_TINYINT || type == TSDB_DATA_TYPE_BOOL) { int8_t* pData = (int8_t*)pCol->pData; int8_t* val = (int8_t*)&pBuf->v; @@ -1276,7 +1283,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1307,7 +1314,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1338,7 +1345,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1357,8 +1364,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { numOfElems += 1; } - } else if (type == TSDB_DATA_TYPE_BIGINT || - type == TSDB_DATA_TYPE_TIMESTAMP) { + } else if (type == TSDB_DATA_TYPE_BIGINT) { int64_t* pData = (int64_t*)pCol->pData; int64_t* val = (int64_t*)&pBuf->v; @@ -1370,7 +1376,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1403,7 +1409,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1434,7 +1440,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1465,7 +1471,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1496,7 +1502,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1528,7 +1534,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1559,7 +1565,7 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { if (!pBuf->assign) { *val = pData[i]; if (pCtx->subsidiaries.num > 0) { - pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock); + pBuf->tuplePos = saveTupleData(pCtx, i, pCtx->pSrcBlock, NULL); } pBuf->assign = true; } else { @@ -1581,8 +1587,8 @@ int32_t doMinMaxHelper(SqlFunctionCtx* pCtx, int32_t isMinFunc) { } _min_max_over: - if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved ) { - pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pBuf->nullTupleSaved) { + pBuf->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); pBuf->nullTupleSaved = true; } return numOfElems; @@ -1601,7 +1607,8 @@ int32_t maxFunction(SqlFunctionCtx* pCtx) { } static void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t rowIndex); -static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex); +static void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, + int32_t rowIndex); int32_t minmaxFunctionFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { SResultRowEntryInfo* pEntryInfo = GET_RES_INFO(pCtx); @@ -1651,7 +1658,7 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple if (pCtx->saveHandle.pBuf != NULL) { if (pTuplePos->pageId != -1) { - int32_t numOfCols = pCtx->subsidiaries.num; + int32_t numOfCols = pCtx->subsidiaries.num; const char* p = loadTupleData(pCtx, pTuplePos); bool* nullList = (bool*)p; @@ -1660,7 +1667,7 @@ void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuple // todo set the offset value to optimize the performance. for (int32_t j = 0; j < numOfCols; ++j) { SqlFunctionCtx* pc = pCtx->subsidiaries.pCtx[j]; - int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pBlock->pDataBlock, dstSlotId); ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); @@ -1701,7 +1708,7 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) char* pData = colDataGetData(pSrcCol, rowIndex); // append to dest col - int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; + int32_t dstSlotId = pc->pExpr->base.resSchema.slotId; SColumnInfoData* pDstCol = taosArrayGet(pCtx->pDstBlock->pDataBlock, dstSlotId); ASSERT(pc->pExpr->base.resSchema.bytes == pDstCol->info.bytes); @@ -1712,7 +1719,6 @@ void appendSelectivityValue(SqlFunctionCtx* pCtx, int32_t rowIndex, int32_t pos) colDataAppend(pDstCol, pos, pData, false); } } - } void replaceTupleData(STuplePos* pDestPos, STuplePos* pSourcePos) { @@ -2590,8 +2596,8 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* memcpy(pHisto, pInput->pHisto, sizeof(SHistogramInfo) + sizeof(SHistBin) * (MAX_HISTOGRAM_BIN + 1)); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, - pHisto->numOfEntries, pHisto); + qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, + pHisto); } else { pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); qDebug("%s input histogram, elem:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, @@ -2601,8 +2607,8 @@ static void apercentileTransferInfo(SAPercentileInfo* pInput, SAPercentileInfo* memcpy(pHisto, pRes, sizeof(SHistogramInfo) + sizeof(SHistBin) * MAX_HISTOGRAM_BIN); pHisto->elems = (SHistBin*)((char*)pHisto + sizeof(SHistogramInfo)); - qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, - pHisto->numOfEntries, pHisto); + qDebug("%s merge histo, total:%" PRId64 ", entry:%d, %p", __FUNCTION__, pHisto->numOfElems, pHisto->numOfEntries, + pHisto); tHistogramDestroy(&pRes); } } @@ -2629,8 +2635,8 @@ int32_t apercentileFunctionMerge(SqlFunctionCtx* pCtx) { } if (pInfo->algo != APERCT_ALGO_TDIGEST) { - qDebug("%s after merge, total:%d, numOfEntry:%d, %p", __FUNCTION__, pInfo->pHisto->numOfElems, pInfo->pHisto->numOfEntries, - pInfo->pHisto); + qDebug("%s after merge, total:%d, numOfEntry:%d, %p", __FUNCTION__, pInfo->pHisto->numOfElems, + pInfo->pHisto->numOfEntries, pInfo->pHisto); } SET_VAL(pResInfo, 1, 1); @@ -2709,7 +2715,7 @@ int32_t apercentileCombine(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx) } EFuncDataRequired lastDynDataReq(void* pRes, STimeWindow* pTimeWindow) { - SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*) pRes; + SResultRowEntryInfo* pEntry = (SResultRowEntryInfo*)pRes; // not initialized yet, data is required if (pEntry == NULL) { @@ -2752,13 +2758,14 @@ static FORCE_INLINE TSKEY getRowPTs(SColumnInfoData* pTsColInfo, int32_t rowInde return *(TSKEY*)colDataGetData(pTsColInfo, rowIndex); } -static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, SFirstLastRes* pInfo) { +static void firstlastSaveTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SqlFunctionCtx* pCtx, + SFirstLastRes* pInfo) { if (pCtx->subsidiaries.num <= 0) { return; } if (!pInfo->hasResult) { - pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock); + pInfo->pos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL); } else { updateTupleData(pCtx, rowIndex, pSrcBlock, &pInfo->pos); } @@ -2795,6 +2802,8 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); return 0; } @@ -2871,7 +2880,10 @@ int32_t firstFunction(SqlFunctionCtx* pCtx) { } } #endif - + if (numOfElems == 0) { + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -2892,6 +2904,8 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { // All null data column, return directly. if (pInput->colDataAggIsSet && (pInput->pColumnDataAgg[0]->numOfNull == pInput->totalRows)) { ASSERT(pInputCol->hasNull == true); + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); return 0; } @@ -2952,7 +2966,10 @@ int32_t lastFunction(SqlFunctionCtx* pCtx) { } } #endif - + if (numOfElems == 0) { + // save selectivity value for column consisted of all null values + firstlastSaveTupleData(pCtx->pSrcBlock, pInput->startRowIndex, pCtx, pInfo); + } SET_VAL(pResInfo, numOfElems, 1); return TSDB_CODE_SUCCESS; } @@ -3176,7 +3193,7 @@ bool diffFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { static void doSetPrevVal(SDiffInfo* pDiffInfo, int32_t type, const char* pv) { switch (type) { case TSDB_DATA_TYPE_BOOL: - pDiffInfo->prev.i64 = *(bool*)pv? 1:0; + pDiffInfo->prev.i64 = *(bool*)pv ? 1 : 0; break; case TSDB_DATA_TYPE_TINYINT: pDiffInfo->prev.i64 = *(int8_t*)pv; @@ -3416,7 +3433,7 @@ int32_t topFunction(SqlFunctionCtx* pCtx) { } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { - pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); pRes->nullTupleSaved = true; } return TSDB_CODE_SUCCESS; @@ -3444,7 +3461,7 @@ int32_t bottomFunction(SqlFunctionCtx* pCtx) { } if (numOfElems == 0 && pCtx->subsidiaries.num > 0 && !pRes->nullTupleSaved) { - pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + pRes->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); pRes->nullTupleSaved = true; } @@ -3496,7 +3513,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData // save the data of this tuple if (pCtx->subsidiaries.num > 0) { - pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock); + pItem->tuplePos = saveTupleData(pCtx, rowIndex, pSrcBlock, NULL); } #ifdef BUF_PAGE_DEBUG qDebug("page_saveTuple i:%d, item:%p,pageId:%d, offset:%d\n", pEntryInfo->numOfRes, pItem, pItem->tuplePos.pageId, @@ -3537,7 +3554,8 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData * |(n columns, one bit for each column)| src column #1| src column #2| * +------------------------------------+--------------+--------------+ */ -void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsidiaryResInfo* pSubsidiaryies, char* buf) { +void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsidiaryResInfo* pSubsidiaryies, + char* buf) { char* nullList = buf; char* pStart = (char*)(nullList + sizeof(bool) * pSubsidiaryies->num); @@ -3567,7 +3585,8 @@ void* serializeTupleData(const SSDataBlock* pSrcBlock, int32_t rowIndex, SSubsid return buf; } -static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length) { +static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, + const STupleKey* pKey) { STuplePos p = {0}; if (pHandle->pBuf != NULL) { SFilePage* pPage = NULL; @@ -3585,7 +3604,7 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf } } - p = (STuplePos) {.pageId = pHandle->currentPage, .offset = pPage->num}; + p = (STuplePos){.pageId = pHandle->currentPage, .offset = pPage->num}; memcpy(pPage->data + pPage->num, pBuf, length); pPage->num += length; @@ -3593,12 +3612,16 @@ static STuplePos doSaveTupleData(SSerializeDataHandle* pHandle, const void* pBuf releaseBufPage(pHandle->pBuf, pPage); } else { // other tuple save policy + if (streamStateFuncPut(pHandle->pState, pKey, pBuf, length) < 0) { + ASSERT(0); + } + p.streamTupleKey = *pKey; } return p; } -STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock) { +STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, const STupleKey* pKey) { if (pCtx->subsidiaries.rowLen == 0) { int32_t rowLen = 0; for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -3611,7 +3634,7 @@ STuplePos saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBloc } char* buf = serializeTupleData(pSrcBlock, rowIndex, &pCtx->subsidiaries, pCtx->subsidiaries.buf); - return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen); + return doSaveTupleData(&pCtx->saveHandle, buf, pCtx->subsidiaries.rowLen, pKey); } static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf, size_t length, STuplePos* pPos) { @@ -3621,7 +3644,7 @@ static int32_t doUpdateTupleData(SSerializeDataHandle* pHandle, const void* pBuf setBufPageDirty(pPage, true); releaseBufPage(pHandle->pBuf, pPage); } else { - + streamStateFuncPut(pHandle->pState, &pPos->streamTupleKey, pBuf, length); } return TSDB_CODE_SUCCESS; @@ -3636,11 +3659,14 @@ static int32_t updateTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSD static char* doLoadTupleData(SSerializeDataHandle* pHandle, const STuplePos* pPos) { if (pHandle->pBuf != NULL) { SFilePage* pPage = getBufPage(pHandle->pBuf, pPos->pageId); - char* p = pPage->data + pPos->offset; + char* p = pPage->data + pPos->offset; releaseBufPage(pHandle->pBuf, pPage); return p; } else { - return NULL; + void* value = NULL; + int32_t vLen; + streamStateFuncGet(pHandle->pState, &pPos->streamTupleKey, &value, &vLen); + return (char*)value; } } @@ -3980,8 +4006,8 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { } if (pCtx->end.key == INT64_MIN) { - pInfo->min = (pInfo->min > ptsList[start + pInput->numOfRows - 1]) ? - ptsList[start + pInput->numOfRows - 1] : pInfo->min; + pInfo->min = + (pInfo->min > ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->min; } else { pInfo->min = pCtx->end.key; } @@ -3993,8 +4019,8 @@ int32_t elapsedFunction(SqlFunctionCtx* pCtx) { } if (pCtx->end.key == INT64_MIN) { - pInfo->max = (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? - ptsList[start + pInput->numOfRows - 1] : pInfo->max; + pInfo->max = + (pInfo->max < ptsList[start + pInput->numOfRows - 1]) ? ptsList[start + pInput->numOfRows - 1] : pInfo->max; } else { pInfo->max = pCtx->end.key + 1; } @@ -4701,7 +4727,7 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { colDataAppendNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } continue; } @@ -4714,11 +4740,11 @@ int32_t stateCountFunction(SqlFunctionCtx* pCtx) { } else { pInfo->count = 0; } - colDataAppend(pOutput, i, (char*)&output, false); + colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } } @@ -4754,7 +4780,7 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { colDataAppendNULL(pOutput, i); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } continue; } @@ -4771,11 +4797,11 @@ int32_t stateDurationFunction(SqlFunctionCtx* pCtx) { } else { pInfo->durationStart = 0; } - colDataAppend(pOutput, i, (char*)&output, false); + colDataAppend(pOutput, pCtx->offset + numOfElems - 1, (char*)&output, false); // handle selectivity if (pCtx->subsidiaries.num > 0) { - appendSelectivityValue(pCtx, i, i); + appendSelectivityValue(pCtx, i, pCtx->offset + numOfElems - 1); } } @@ -4971,7 +4997,7 @@ static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* da if (pInfo->numSampled < pInfo->samples) { sampleAssignResult(pInfo, data, pInfo->numSampled); if (pCtx->subsidiaries.num > 0) { - pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock); + pInfo->tuplePos[pInfo->numSampled] = saveTupleData(pCtx, index, pCtx->pSrcBlock, NULL); } pInfo->numSampled++; } else { @@ -5002,7 +5028,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } if (pInfo->numSampled == 0 && pCtx->subsidiaries.num > 0 && !pInfo->nullTupleSaved) { - pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock); + pInfo->nullTuplePos = saveTupleData(pCtx, pInput->startRowIndex, pCtx->pSrcBlock, NULL); pInfo->nullTupleSaved = true; } @@ -5388,8 +5414,8 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { int32_t i = pInput->startRowIndex; if (pCtx->start.key != INT64_MIN) { - ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || - (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); + //ASSERT((pCtx->start.key < tsList[i] && pCtx->order == TSDB_ORDER_ASC) || + // (pCtx->start.key > tsList[i] && pCtx->order == TSDB_ORDER_DESC)); ASSERT(last->key == INT64_MIN); for (; i < pInput->numOfRows + pInput->startRowIndex; ++i) { @@ -5437,6 +5463,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5452,6 +5482,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5466,6 +5500,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5480,6 +5518,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5494,6 +5536,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5508,6 +5554,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5522,6 +5572,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5536,6 +5590,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5550,6 +5608,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } @@ -5564,6 +5626,10 @@ int32_t twaFunction(SqlFunctionCtx* pCtx) { numOfElems++; INIT_INTP_POINT(st, tsList[i], val[i]); + if (pInfo->p.key == st.key) { + return TSDB_CODE_FUNC_DUP_TIMESTAMP; + } + pInfo->dOutput += twa_get_area(pInfo->p, st); pInfo->p = st; } diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index 5de9c52cc1d0f85c268d86a8e54d184a37c55df3..3f472b53a026de186bbab2e70d37997b2b2fda44 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -1183,7 +1183,9 @@ void onUdfcPipeClose(uv_handle_t *handle) { QUEUE_REMOVE(&task->procTaskQueue); uv_sem_post(&task->taskSem); } - conn->session->udfUvPipe = NULL; + if (conn->session != NULL) { + conn->session->udfUvPipe = NULL; + } taosMemoryFree(conn->readBuf.buf); taosMemoryFree(conn); taosMemoryFree((uv_pipe_t *) handle); @@ -1803,6 +1805,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { if (session->udfUvPipe == NULL) { fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName); + taosMemoryFree(session); return TSDB_CODE_UDF_PIPE_NO_PIPE; } @@ -1821,7 +1824,11 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { udfcRunUdfUvTask(task, UV_TASK_DISCONNECT); fnInfo("tear down udf. udf name: %s, udf func handle: %p", session->udfName, handle); - + //TODO: synchronization refactor between libuv event loop and request thread + if (session->udfUvPipe != NULL && session->udfUvPipe->data != NULL) { + SClientUvConn *conn = session->udfUvPipe->data; + conn->session = NULL; + } taosMemoryFree(session); taosMemoryFree(task); diff --git a/source/libs/index/inc/indexInt.h b/source/libs/index/inc/indexInt.h index 065f4acb576263d1f7d5cbe8238273dc325ccb09..9605528ad6ae150fd88f512cdf5344b81d486a99 100644 --- a/source/libs/index/inc/indexInt.h +++ b/source/libs/index/inc/indexInt.h @@ -40,26 +40,31 @@ extern "C" { #define indexTrace(...) do { if (idxDebugFlag & DEBUG_TRACE) { taosPrintLog("IDX", DEBUG_TRACE, idxDebugFlag, __VA_ARGS__);} } while (0) // clang-format on +extern void* indexQhandle; + typedef enum { LT, LE, GT, GE, CONTAINS, EQ } RangeType; typedef enum { kTypeValue, kTypeDeletion } STermValueType; typedef enum { kRebuild, kFinished } SIdxStatus; typedef struct SIndexStat { - int32_t totalAdded; // - int32_t totalDeled; // - int32_t totalUpdated; // - int32_t totalTerms; // - int32_t distinctCol; // distinct column + int32_t total; + int32_t add; // + int32_t del; // + int32_t update; // + int32_t terms; // + int32_t distCol; // distinct column } SIndexStat; struct SIndex { + SIndexOpts opts; + int64_t refId; void* cache; void* tindex; SHashObj* colObj; // < field name, field id> - int64_t suid; // current super table id, -1 is normal table - int32_t cVersion; // current version allocated to cache + int64_t suid; // current super table id, -1 is normal table + int32_t version; // current version allocated to cache SLRUCache* lru; char* path; @@ -68,7 +73,6 @@ struct SIndex { TdThreadMutex mtx; tsem_t sem; bool quit; - SIndexOpts opts; }; struct SIndexMultiTermQuery { @@ -111,14 +115,15 @@ typedef struct Iterate { void iterateValueDestroy(IterateValue* iv, bool destroy); -extern void* indexQhandle; - typedef struct TFileCacheKey { uint64_t suid; uint8_t colType; char* colName; int32_t nColName; } ICacheKey; + +int32_t idxSerialCacheKey(ICacheKey* key, char* buf); + int idxFlushCacheToTFile(SIndex* sIdx, void*, bool quit); int64_t idxAddRef(void* p); @@ -126,10 +131,6 @@ int32_t idxRemoveRef(int64_t ref); void idxAcquireRef(int64_t ref); void idxReleaseRef(int64_t ref); -int32_t idxSerialCacheKey(ICacheKey* key, char* buf); -// int32_t indexSerialKey(ICacheKey* key, char* buf); -// int32_t indexSerialTermKey(SIndexTerm* itm, char* buf); - #define IDX_TYPE_CONTAIN_EXTERN_TYPE(ty, exTy) (((ty >> 4) & (exTy)) != 0) #define IDX_TYPE_GET_TYPE(ty) (ty & 0x0F) diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index be64a8b44d28a76a0b04a78b3940bcb0c86101da..f507e1b3bed918419ca292b6d88ea85311122222 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -25,10 +25,6 @@ #include "tref.h" #include "tsched.h" -#ifdef USE_LUCENE -#include "lucene++/Lucene_c.h" -#endif - #define INDEX_NUM_OF_THREADS 5 #define INDEX_QUEUE_SIZE 200 @@ -74,7 +70,7 @@ void indexCleanup() { typedef struct SIdxColInfo { int colId; // generated by index internal - int cVersion; + int version; } SIdxColInfo; static TdThreadOnce isInit = PTHREAD_ONCE_INIT; @@ -123,7 +119,7 @@ int indexOpen(SIndexOpts* opts, const char* path, SIndex** index) { } idx->colObj = taosHashInit(8, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - idx->cVersion = 1; + idx->version = 1; idx->path = tstrdup(path); taosThreadMutexInit(&idx->mtx, NULL); tsem_init(&idx->sem, 0, 0); @@ -307,7 +303,7 @@ SIndexTerm* indexTermCreate(int64_t suid, SIndexOperOnColumn oper, uint8_t colTy buf = strndup(INDEX_DATA_NULL_STR, (int32_t)strlen(INDEX_DATA_NULL_STR)); len = (int32_t)strlen(INDEX_DATA_NULL_STR); } else { - const char* emptyStr = " "; + static const char* emptyStr = " "; buf = strndup(emptyStr, (int32_t)strlen(emptyStr)); len = (int32_t)strlen(emptyStr); } @@ -589,6 +585,12 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { idxTRsltDestroy(tr); int ret = idxGenTFile(sIdx, pCache, result); + if (ret != 0) { + indexError("failed to merge"); + } else { + int64_t cost = taosGetTimestampUs() - st; + indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); + } idxDestroyFinalRslt(result); idxCacheDestroyImm(pCache); @@ -599,12 +601,6 @@ int idxFlushCacheToTFile(SIndex* sIdx, void* cache, bool quit) { tfileReaderUnRef(pReader); idxCacheUnRef(pCache); - int64_t cost = taosGetTimestampUs() - st; - if (ret != 0) { - indexError("failed to merge, time cost: %" PRId64 "ms", cost / 1000); - } else { - indexInfo("success to merge , time cost: %" PRId64 "ms", cost / 1000); - } atomic_store_32(&pCache->merging, 0); if (quit) { idxPost(sIdx); diff --git a/source/libs/index/src/indexCache.c b/source/libs/index/src/indexCache.c index 794b85d244f875b6874855ac6a36a7500114afd5..7e867db755c658abad341e8fa59cd6687ef9b959 100644 --- a/source/libs/index/src/indexCache.c +++ b/source/libs/index/src/indexCache.c @@ -566,7 +566,6 @@ int idxCachePut(void* cache, SIndexTerm* term, uint64_t uid) { taosThreadMutexUnlock(&pCache->mtx); idxCacheUnRef(pCache); return 0; - // encode end } void idxCacheForceToMerge(void* cache) { IndexCache* pCache = cache; @@ -602,10 +601,10 @@ static int32_t idxQueryMem(MemTable* mem, SIndexTermQuery* query, SIdxTRslt* tr, } } int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STermValueType* s) { - int64_t st = taosGetTimestampUs(); if (cache == NULL) { return 0; } + IndexCache* pCache = cache; MemTable *mem = NULL, *imm = NULL; @@ -616,6 +615,8 @@ int idxCacheSearch(void* cache, SIndexTermQuery* query, SIdxTRslt* result, STerm idxMemRef(imm); taosThreadMutexUnlock(&pCache->mtx); + int64_t st = taosGetTimestampUs(); + int ret = (mem && mem->mem) ? idxQueryMem(mem, query, result, s) : 0; if (ret == 0 && *s != kTypeDeletion) { // continue search in imm diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index cd52d122f781e3210448904af7b9ac0d3b4f9046..ecf91360734e37f0060aeb7758e5c4c5d57d4972 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -81,28 +81,28 @@ __compar_fn_t idxGetCompar(int8_t type) { } return getComparFunc(type, 0); } -static TExeCond tCompareLessThan(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareLessThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_THAN, a, b, type); } -static TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareLessEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_LESS_EQUAL, a, b, type); } -static TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareGreaterThan(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_THAN, a, b, type); } -static TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareGreaterEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_GREATER_EQUAL, a, b, type); } -static TExeCond tCompareContains(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareContains(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } -static TExeCond tCompareEqual(void* a, void* b, int8_t type) { +static FORCE_INLINE TExeCond tCompareEqual(void* a, void* b, int8_t type) { __compar_fn_t func = idxGetCompar(type); return tCompare(func, QUERY_TERM, a, b, type); } @@ -178,9 +178,9 @@ TExeCond tDoCompare(__compar_fn_t func, int8_t comparType, void* a, void* b) { // optime later int32_t ret = func(a, b); switch (comparType) { - case QUERY_LESS_THAN: { + case QUERY_LESS_THAN: if (ret < 0) return MATCH; - } break; + break; case QUERY_LESS_EQUAL: { if (ret <= 0) return MATCH; break; diff --git a/source/libs/index/src/indexFilter.c b/source/libs/index/src/indexFilter.c index 75844ce76f1cb50d6847709309dae1ed3f77bf70..b65acc467215da77019235e5ec44a335b363e344 100644 --- a/source/libs/index/src/indexFilter.c +++ b/source/libs/index/src/indexFilter.c @@ -27,6 +27,44 @@ #define SIF_RET(c) do { int32_t _code = c; if (_code != TSDB_CODE_SUCCESS) { terrno = _code; } return _code; } while (0) #define SIF_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) // clang-format on + +typedef union { + uint8_t u8; + uint16_t u16; + uint32_t u32; + uint64_t u64; + + int8_t i8; + int16_t i16; + int32_t i32; + int64_t i64; + + double d; + float f; +} SDataTypeBuf; + +#define SIF_DATA_CONVERT(type, val, dst) \ + do { \ + if (type == TSDB_DATA_TYPE_DOUBLE) \ + dst = GET_DOUBLE_VAL(val); \ + else if (type == TSDB_DATA_TYPE_BIGINT) \ + dst = *(int64_t *)val; \ + else if (type == TSDB_DATA_TYPE_INT) \ + dst = *(int32_t *)val; \ + else if (type == TSDB_DATA_TYPE_SMALLINT) \ + dst = *(int16_t *)val; \ + else if (type == TSDB_DATA_TYPE_TINYINT) \ + dst = *(int8_t *)val; \ + else if (type == TSDB_DATA_TYPE_UTINYINT) \ + dst = *(uint8_t *)val; \ + else if (type == TSDB_DATA_TYPE_USMALLINT) \ + dst = *(uint16_t *)val; \ + else if (type == TSDB_DATA_TYPE_UINT) \ + dst = *(uint32_t *)val; \ + else if (type == TSDB_DATA_TYPE_UBIGINT) \ + dst = *(uint64_t *)val; \ + } while (0); + typedef struct SIFParam { SHashObj *pFilter; @@ -48,10 +86,9 @@ typedef struct SIFCtx { SHashObj *pRes; /* element is SIFParam */ bool noExec; // true: just iterate condition tree, and add hint to executor plan SIndexMetaArg arg; - // SIdxFltStatus st; } SIFCtx; -static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { +static FORCE_INLINE int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { if (src == OP_TYPE_GREATER_THAN) { *dst = QUERY_GREATER_THAN; } else if (src == OP_TYPE_GREATER_EQUAL) { @@ -73,15 +110,9 @@ static int32_t sifGetFuncFromSql(EOperatorType src, EIndexQueryType *dst) { } typedef int32_t (*sif_func_t)(SIFParam *left, SIFParam *rigth, SIFParam *output); - static sif_func_t sifNullFunc = NULL; -// typedef struct SIFWalkParm -// construct tag filter operator later -// static void destroyTagFilterOperatorInfo(void *param) { -// STagFilterOperatorInfo *pInfo = (STagFilterOperatorInfo *)param; -//} -static void sifFreeParam(SIFParam *param) { +static FORCE_INLINE void sifFreeParam(SIFParam *param) { if (param == NULL) return; taosArrayDestroy(param->result); @@ -91,7 +122,7 @@ static void sifFreeParam(SIFParam *param) { param->pFilter = NULL; } -static int32_t sifGetOperParamNum(EOperatorType ty) { +static FORCE_INLINE int32_t sifGetOperParamNum(EOperatorType ty) { if (OP_TYPE_IS_NULL == ty || OP_TYPE_IS_NOT_NULL == ty || OP_TYPE_IS_TRUE == ty || OP_TYPE_IS_NOT_TRUE == ty || OP_TYPE_IS_FALSE == ty || OP_TYPE_IS_NOT_FALSE == ty || OP_TYPE_IS_UNKNOWN == ty || OP_TYPE_IS_NOT_UNKNOWN == ty || OP_TYPE_MINUS == ty) { @@ -99,14 +130,14 @@ static int32_t sifGetOperParamNum(EOperatorType ty) { } return 2; } -static int32_t sifValidOp(EOperatorType ty) { +static FORCE_INLINE int32_t sifValidOp(EOperatorType ty) { if ((ty >= OP_TYPE_ADD && ty <= OP_TYPE_BIT_OR) || (ty == OP_TYPE_IN || ty == OP_TYPE_NOT_IN) || (ty == OP_TYPE_LIKE || ty == OP_TYPE_NOT_LIKE || ty == OP_TYPE_MATCH || ty == OP_TYPE_NMATCH)) { return -1; } return 0; } -static int32_t sifValidColumn(SColumnNode *cn) { +static FORCE_INLINE int32_t sifValidColumn(SColumnNode *cn) { // add more check if (cn == NULL) { return TSDB_CODE_QRY_INVALID_INPUT; @@ -117,7 +148,7 @@ static int32_t sifValidColumn(SColumnNode *cn) { return TSDB_CODE_SUCCESS; } -static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) { +static FORCE_INLINE SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SIdxFltStatus rs) { // enh rule later if (type == LOGIC_COND_TYPE_AND) { if (ls == SFLT_NOT_INDEX || rs == SFLT_NOT_INDEX) { @@ -135,7 +166,7 @@ static SIdxFltStatus sifMergeCond(ELogicConditionType type, SIdxFltStatus ls, SI return SFLT_NOT_INDEX; } -static int32_t sifGetValueFromNode(SNode *node, char **value) { +static FORCE_INLINE int32_t sifGetValueFromNode(SNode *node, char **value) { // covert data From snode; SValueNode *vn = (SValueNode *)node; @@ -173,7 +204,7 @@ static int32_t sifGetValueFromNode(SNode *node, char **value) { return TSDB_CODE_SUCCESS; } -static int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { +static FORCE_INLINE int32_t sifInitJsonParam(SNode *node, SIFParam *param, SIFCtx *ctx) { SOperatorNode *nd = (SOperatorNode *)node; assert(nodeType(node) == QUERY_NODE_OPERATOR); SColumnNode *l = (SColumnNode *)nd->pLeft; @@ -323,30 +354,30 @@ static int32_t sifExecFunction(SFunctionNode *node, SIFCtx *ctx, SIFParam *outpu return TSDB_CODE_QRY_INVALID_INPUT; } -typedef int (*Filter)(void *a, void *b, int16_t dtype); +typedef int (*FilterFunc)(void *a, void *b, int16_t dtype); -int sifGreaterThan(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifGreaterThan(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_GREATER_THAN, a, b); } -int sifGreaterEqual(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifGreaterEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_GREATER_EQUAL, a, b); } -int sifLessEqual(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifLessEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return tDoCompare(func, QUERY_LESS_EQUAL, a, b); } -int sifLessThan(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifLessThan(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); return (int)tDoCompare(func, QUERY_LESS_THAN, a, b); } -int sifEqual(void *a, void *b, int16_t dtype) { +static FORCE_INLINE int sifEqual(void *a, void *b, int16_t dtype) { __compar_fn_t func = getComparFunc(dtype, 0); //__compar_fn_t func = idxGetCompar(dtype); return (int)tDoCompare(func, QUERY_TERM, a, b); } -static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { +static FORCE_INLINE FilterFunc sifGetFilterFunc(EIndexQueryType type, bool *reverse) { if (type == QUERY_LESS_EQUAL || type == QUERY_LESS_THAN) { *reverse = true; } else { @@ -365,42 +396,6 @@ static Filter sifGetFilterFunc(EIndexQueryType type, bool *reverse) { } return NULL; } -typedef union { - uint8_t u8; - uint16_t u16; - uint32_t u32; - uint64_t u64; - - int8_t i8; - int16_t i16; - int32_t i32; - int64_t i64; - - double d; - float f; -} SDataTypeBuf; - -#define SIF_DATA_CONVERT(type, val, dst) \ - do { \ - if (type == TSDB_DATA_TYPE_DOUBLE) \ - dst = GET_DOUBLE_VAL(val); \ - else if (type == TSDB_DATA_TYPE_BIGINT) \ - dst = *(int64_t *)val; \ - else if (type == TSDB_DATA_TYPE_INT) \ - dst = *(int32_t *)val; \ - else if (type == TSDB_DATA_TYPE_SMALLINT) \ - dst = *(int16_t *)val; \ - else if (type == TSDB_DATA_TYPE_TINYINT) \ - dst = *(int8_t *)val; \ - else if (type == TSDB_DATA_TYPE_UTINYINT) \ - dst = *(uint8_t *)val; \ - else if (type == TSDB_DATA_TYPE_USMALLINT) \ - dst = *(uint16_t *)val; \ - else if (type == TSDB_DATA_TYPE_UINT) \ - dst = *(uint32_t *)val; \ - else if (type == TSDB_DATA_TYPE_UBIGINT) \ - dst = *(uint64_t *)val; \ - } while (0); static void sifSetFltParam(SIFParam *left, SIFParam *right, SDataTypeBuf *typedata, SMetaFltParam *param) { int8_t ltype = left->colValType, rtype = right->colValType; @@ -474,8 +469,8 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP indexMultiTermQueryAdd(mtm, tm, qtype); ret = indexJsonSearch(arg->ivtIdx, mtm, output->result); } else { - bool reverse; - Filter filterFunc = sifGetFilterFunc(qtype, &reverse); + bool reverse; + FilterFunc filterFunc = sifGetFilterFunc(qtype, &reverse); SMetaFltParam param = {.suid = arg->suid, .cid = left->colId, @@ -502,72 +497,72 @@ static int32_t sifDoIndex(SIFParam *left, SIFParam *right, int8_t operType, SIFP return ret; } -static int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifLessThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_THAN; return sifDoIndex(left, right, id, output); } -static int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifLessEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LOWER_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifGreaterThanFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_THAN; return sifDoIndex(left, right, id, output); } -static int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifGreaterEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_GREATER_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotEqualFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_EQUAL; return sifDoIndex(left, right, id, output); } -static int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_IN; return sifDoIndex(left, right, id, output); } -static int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotInFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_IN; return sifDoIndex(left, right, id, output); } -static int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_LIKE; return sifDoIndex(left, right, id, output); } -static int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotLikeFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NOT_LIKE; return sifDoIndex(left, right, id, output); } -static int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_MATCH; return sifDoIndex(left, right, id, output); } -static int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifNotMatchFunc(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_NMATCH; return sifDoIndex(left, right, id, output); } -static int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifJsonContains(SIFParam *left, SIFParam *right, SIFParam *output) { int id = OP_TYPE_JSON_CONTAINS; return sifDoIndex(left, right, id, output); } -static int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { +static FORCE_INLINE int32_t sifJsonGetValue(SIFParam *left, SIFParam *rigth, SIFParam *output) { // return 0 return 0; } -static int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { +static FORCE_INLINE int32_t sifDefaultFunc(SIFParam *left, SIFParam *right, SIFParam *output) { // add more except return TSDB_CODE_QRY_INVALID_INPUT; } -static int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) { +static FORCE_INLINE int32_t sifGetOperFn(int32_t funcId, sif_func_t *func, SIdxFltStatus *status) { // impl later *status = SFLT_ACCURATE_INDEX; switch (funcId) { @@ -693,11 +688,8 @@ static int32_t sifExecLogic(SLogicConditionNode *node, SIFCtx *ctx, SIFParam *ou for (int32_t m = 0; m < node->pParameterList->length; m++) { if (node->condType == LOGIC_COND_TYPE_AND) { taosArrayAddAll(output->result, params[m].result); - // taosArrayDestroy(params[m].result); - // params[m].result = NULL; } else if (node->condType == LOGIC_COND_TYPE_OR) { taosArrayAddAll(output->result, params[m].result); - // params[m].result = NULL; } else if (node->condType == LOGIC_COND_TYPE_NOT) { // taosArrayAddAll(output->result, params[m].result); } diff --git a/source/libs/index/src/indexFst.c b/source/libs/index/src/indexFst.c index 15152cef55c221f8a93bfee533dc6a9750f1db4b..2aa8345e03bb3cec2a86c4240351dc86cb3ec9c7 100644 --- a/source/libs/index/src/indexFst.c +++ b/source/libs/index/src/indexFst.c @@ -19,11 +19,12 @@ #include "tchecksum.h" #include "tcoding.h" -static void fstPackDeltaIn(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, uint8_t nBytes) { +static FORCE_INLINE void fstPackDeltaIn(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr, + uint8_t nBytes) { CompiledAddr deltaAddr = (transAddr == EMPTY_ADDRESS) ? EMPTY_ADDRESS : nodeAddr - transAddr; idxFilePackUintIn(wrt, deltaAddr, nBytes); } -static uint8_t fstPackDetla(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { +static FORCE_INLINE uint8_t fstPackDetla(IdxFstFile* wrt, CompiledAddr nodeAddr, CompiledAddr transAddr) { uint8_t nBytes = packDeltaSize(nodeAddr, transAddr); fstPackDeltaIn(wrt, nodeAddr, transAddr, nBytes); return nBytes; @@ -39,7 +40,7 @@ FstUnFinishedNodes* fstUnFinishedNodesCreate() { fstUnFinishedNodesPushEmpty(nodes, false); return nodes; } -static void unFinishedNodeDestroyElem(void* elem) { +static void FORCE_INLINE unFinishedNodeDestroyElem(void* elem) { FstBuilderNodeUnfinished* b = (FstBuilderNodeUnfinished*)elem; fstBuilderNodeDestroy(b->node); taosMemoryFree(b->last); diff --git a/source/libs/index/src/indexFstFile.c b/source/libs/index/src/indexFstFile.c index 1900e50973ade5278482162df05d1cb528365238..2a33ddd477daa8c1de9e64e958be0d15865f9efb 100644 --- a/source/libs/index/src/indexFstFile.c +++ b/source/libs/index/src/indexFstFile.c @@ -30,14 +30,14 @@ typedef struct { static void deleteDataBlockFromLRU(const void* key, size_t keyLen, void* value) { taosMemoryFree(value); } -static void idxGenLRUKey(char* buf, const char* path, int32_t blockId) { +static FORCE_INLINE void idxGenLRUKey(char* buf, const char* path, int32_t blockId) { char* p = buf; SERIALIZE_STR_VAR_TO_BUF(p, path, strlen(path)); SERIALIZE_VAR_TO_BUF(p, '_', char); idxInt2str(blockId, p, 0); return; } -static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) { +static FORCE_INLINE int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) { if (ctx->type == TFILE) { int nwr = taosWriteFile(ctx->file.pFile, buf, len); assert(nwr == len); @@ -47,7 +47,7 @@ static int idxFileCtxDoWrite(IFileCtx* ctx, uint8_t* buf, int len) { ctx->offset += len; return len; } -static int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) { +static FORCE_INLINE int idxFileCtxDoRead(IFileCtx* ctx, uint8_t* buf, int len) { int nRead = 0; if (ctx->type == TFILE) { #ifdef USE_MMAP @@ -111,7 +111,7 @@ static int idxFileCtxDoReadFrom(IFileCtx* ctx, uint8_t* buf, int len, int32_t of } while (len > 0); return total; } -static int idxFileCtxGetSize(IFileCtx* ctx) { +static FORCE_INLINE int idxFileCtxGetSize(IFileCtx* ctx) { if (ctx->type == TFILE) { int64_t file_size = 0; taosStatFile(ctx->file.buf, &file_size, NULL); @@ -119,7 +119,7 @@ static int idxFileCtxGetSize(IFileCtx* ctx) { } return 0; } -static int idxFileCtxDoFlush(IFileCtx* ctx) { +static FORCE_INLINE int idxFileCtxDoFlush(IFileCtx* ctx) { if (ctx->type == TFILE) { taosFsyncFile(ctx->file.pFile); } else { @@ -211,9 +211,7 @@ IdxFstFile* idxFileCreate(void* wrt) { return cw; } void idxFileDestroy(IdxFstFile* cw) { - // free wrt object: close fd or free mem idxFileFlush(cw); - // idxFileCtxDestroy((IFileCtx *)(cw->wrt)); taosMemoryFree(cw); } @@ -222,10 +220,8 @@ int idxFileWrite(IdxFstFile* write, uint8_t* buf, uint32_t len) { return 0; } // update checksum - // write data to file/socket or mem IFileCtx* ctx = write->wrt; - - int nWrite = ctx->write(ctx, buf, len); + int nWrite = ctx->write(ctx, buf, len); assert(nWrite == len); write->count += len; diff --git a/source/libs/index/src/indexFstRegister.c b/source/libs/index/src/indexFstRegister.c index 34efee0d0db510ea1ce50de26c418ae1fd08761e..e0abcadc78a07b0f69ef92003d4304141551865e 100644 --- a/source/libs/index/src/indexFstRegister.c +++ b/source/libs/index/src/indexFstRegister.c @@ -16,7 +16,7 @@ #include "indexFstRegistry.h" #include "os.h" -uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { +static FORCE_INLINE uint64_t fstRegistryHash(FstRegistry* registry, FstBuilderNode* bNode) { // TODO(yihaoDeng): refactor later const uint64_t FNV_PRIME = 1099511628211; uint64_t h = 14695981039346656037u; diff --git a/source/libs/index/src/indexFstSparse.c b/source/libs/index/src/indexFstSparse.c index ebc0cb3637dc14a1968afe7d9669c7eabdf99427..8746b04eab9c2ea46117e1287ebd934a0a5e4eb9 100644 --- a/source/libs/index/src/indexFstSparse.c +++ b/source/libs/index/src/indexFstSparse.c @@ -15,7 +15,7 @@ #include "indexFstSparse.h" -static void sparSetUtil(int32_t *buf, int32_t cap) { +static FORCE_INLINE void sparSetInitBuf(int32_t *buf, int32_t cap) { for (int32_t i = 0; i < cap; i++) { buf[i] = -1; } @@ -28,8 +28,8 @@ FstSparseSet *sparSetCreate(int32_t sz) { ss->dense = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); ss->sparse = (int32_t *)taosMemoryMalloc(sz * sizeof(int32_t)); - sparSetUtil(ss->dense, sz); - sparSetUtil(ss->sparse, sz); + sparSetInitBuf(ss->dense, sz); + sparSetInitBuf(ss->sparse, sz); ss->cap = sz; @@ -90,7 +90,7 @@ void sparSetClear(FstSparseSet *ss) { if (ss == NULL) { return; } - sparSetUtil(ss->dense, ss->cap); - sparSetUtil(ss->sparse, ss->cap); + sparSetInitBuf(ss->dense, ss->cap); + sparSetInitBuf(ss->sparse, ss->cap); ss->size = 0; } diff --git a/source/libs/index/src/indexTfile.c b/source/libs/index/src/indexTfile.c index 0a47fc0f167a359b35952f0c1e88af03d544c95d..1fc631e9f36c0fb081060831629c7b715a59f978 100644 --- a/source/libs/index/src/indexTfile.c +++ b/source/libs/index/src/indexTfile.c @@ -183,13 +183,14 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) { return NULL; } reader->ctx = ctx; + reader->remove = false; if (0 != tfileReaderVerify(reader)) { indexError("invalid tfile, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); tfileReaderDestroy(reader); return NULL; } - // T_REF_INC(reader); + if (0 != tfileReaderLoadHeader(reader)) { indexError("failed to load index header, suid:%" PRIu64 ", colName:%s", reader->header.suid, reader->header.colName); @@ -203,7 +204,6 @@ TFileReader* tfileReaderCreate(IFileCtx* ctx) { tfileReaderDestroy(reader); return NULL; } - reader->remove = false; return reader; } @@ -211,7 +211,6 @@ void tfileReaderDestroy(TFileReader* reader) { if (reader == NULL) { return; } - // T_REF_INC(reader); fstDestroy(reader->fst); if (reader->remove) { indexInfo("%s is removed", reader->ctx->file.buf); @@ -222,6 +221,7 @@ void tfileReaderDestroy(TFileReader* reader) { taosMemoryFree(reader); } + static int32_t tfSearchTerm(void* reader, SIndexTerm* tem, SIdxTRslt* tr) { int ret = 0; char* p = tem->colVal; @@ -323,10 +323,6 @@ static int32_t tfSearchCompareFunc(void* reader, SIndexTerm* tem, SIdxTRslt* tr, while ((rt = stmStNextWith(st, NULL)) != NULL) { FstSlice* s = &rt->data; char* ch = (char*)fstSliceData(s, NULL); - // if (0 != strncmp(ch, tem->colName, tem->nColName)) { - // swsResultDestroy(rt); - // break; - //} TExeCond cond = cmpFn(ch, p, tem->colType); if (MATCH == cond) { @@ -494,7 +490,6 @@ int tfileReaderSearch(TFileReader* reader, SIndexTermQuery* query, SIdxTRslt* tr TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const char* colName, uint8_t colType) { char fullname[256] = {0}; tfileGenFileFullName(fullname, path, suid, colName, version); - // indexInfo("open write file name %s", fullname); IFileCtx* wcx = idxFileCtxCreate(TFILE, fullname, false, 1024 * 1024 * 64); if (wcx == NULL) { return NULL; @@ -503,8 +498,8 @@ TFileWriter* tfileWriterOpen(char* path, uint64_t suid, int64_t version, const c TFileHeader tfh = {0}; tfh.suid = suid; tfh.version = version; - memcpy(tfh.colName, colName, strlen(colName)); tfh.colType = colType; + memcpy(tfh.colName, colName, strlen(colName)); return tfileWriterCreate(wcx, &tfh); } @@ -706,7 +701,6 @@ static bool tfileIteratorNext(Iterate* iiter) { iv->type = ADD_VALUE; // value in tfile always ADD_VALUE iv->colVal = colVal; return true; - // std::string key(ch, sz); } static IterateValue* tifileIterateGetValue(Iterate* iter) { return &iter->val; } @@ -1036,7 +1030,8 @@ static void tfileGenFileName(char* filename, uint64_t suid, const char* col, int sprintf(filename, "%" PRIu64 "-%s-%" PRId64 ".tindex", suid, col, version); return; } -static void tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, int64_t version) { +static void FORCE_INLINE tfileGenFileFullName(char* fullname, const char* path, uint64_t suid, const char* col, + int64_t version) { char filename[128] = {0}; tfileGenFileName(filename, suid, col, version); sprintf(fullname, "%s/%s", path, filename); diff --git a/source/libs/index/src/indexUtil.c b/source/libs/index/src/indexUtil.c index 3d083c1817f4b8b3930da7d0bed12e278d948d87..cdfb79016f46658e6259cb7d89dc501e386d408f 100644 --- a/source/libs/index/src/indexUtil.c +++ b/source/libs/index/src/indexUtil.c @@ -21,7 +21,7 @@ typedef struct MergeIndex { int len; } MergeIndex; -static int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { +static FORCE_INLINE int iBinarySearch(SArray *arr, int s, int e, uint64_t k) { uint64_t v; int32_t m; while (s <= e) { diff --git a/source/libs/index/test/CMakeLists.txt b/source/libs/index/test/CMakeLists.txt index b3eca280032e56004e649b2d2cef44ec1672d8ac..2bc7353aa51e85cf9c9e1c27607e10d76337ff58 100644 --- a/source/libs/index/test/CMakeLists.txt +++ b/source/libs/index/test/CMakeLists.txt @@ -80,6 +80,11 @@ IF(NOT TD_DARWIN) "${TD_SOURCE_DIR}/include/libs/index" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) + target_include_directories (idxJsonUT + PUBLIC + "${TD_SOURCE_DIR}/include/libs/index" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" + ) target_link_libraries (idxTest os @@ -102,11 +107,7 @@ IF(NOT TD_DARWIN) gtest_main index ) - target_include_directories (idxJsonUT - PUBLIC - "${TD_SOURCE_DIR}/include/libs/index" - "${CMAKE_CURRENT_SOURCE_DIR}/../inc" - ) + target_link_libraries (idxTest os util diff --git a/source/libs/index/test/indexBench.cc b/source/libs/index/test/indexBench.cc new file mode 100644 index 0000000000000000000000000000000000000000..b828be0ffe97ee94c6b19e52c71d049ae023b66a --- /dev/null +++ b/source/libs/index/test/indexBench.cc @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 * or later ("AGPL"), as published by the Free + * Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ +#include +#include +#include +#include +#include +#include "index.h" +#include "indexCache.h" +#include "indexFst.h" +#include "indexFstUtil.h" +#include "indexInt.h" +#include "indexTfile.h" +#include "indexUtil.h" +#include "tskiplist.h" +#include "tutil.h" +using namespace std; + +static std::string logDir = TD_TMP_DIR_PATH "log"; + +static void initLog() { + const char *defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + idxDebugFlag = 143; + strcpy(tsLogDir, logDir.c_str()); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); + } +} + +struct WriteBatch { + SIndexMultiTerm *terms; +}; +class Idx { + public: + Idx(int _cacheSize = 1024 * 1024 * 4, const char *_path = "tindex") { + opts.cacheSize = _cacheSize; + path += TD_TMP_DIR_PATH; + path += _path; + } + int SetUp(bool remove) { + initLog(); + + if (remove) taosRemoveDir(path.c_str()); + + int ret = indexJsonOpen(&opts, path.c_str(), &index); + return ret; + } + int Write(WriteBatch *batch, uint64_t uid) { + // write batch + indexJsonPut(index, batch->terms, uid); + return 0; + } + int Read(const char *json, void *key, int64_t *id) { + // read batch + return 0; + } + + void TearDown() { indexJsonClose(index); } + + std::string path; + + SIndexOpts opts; + SIndex *index; +}; + +SIndexTerm *indexTermCreateT(int64_t suid, SIndexOperOnColumn oper, uint8_t colType, const char *colName, + int32_t nColName, const char *colVal, int32_t nColVal) { + char buf[256] = {0}; + int16_t sz = nColVal; + memcpy(buf, (uint16_t *)&sz, 2); + memcpy(buf + 2, colVal, nColVal); + if (colType == TSDB_DATA_TYPE_BINARY) { + return indexTermCreate(suid, oper, colType, colName, nColName, buf, sizeof(buf)); + } else { + return indexTermCreate(suid, oper, colType, colName, nColName, colVal, nColVal); + } + return NULL; +} +int initWriteBatch(WriteBatch *wb, int batchSize) { + SIndexMultiTerm *terms = indexMultiTermCreate(); + + std::string colName; + std::string colVal; + + for (int i = 0; i < 64; i++) { + colName += '0' + i; + colVal += '0' + i; + } + + for (int i = 0; i < batchSize; i++) { + colVal[i % colVal.size()] = '0' + i % 128; + colName[i % colName.size()] = '0' + i % 128; + SIndexTerm *term = indexTermCreateT(0, ADD_VALUE, TSDB_DATA_TYPE_BINARY, colName.c_str(), colName.size(), + colVal.c_str(), colVal.size()); + indexMultiTermAdd(terms, term); + } + + wb->terms = terms; + return 0; +} + +int BenchWrite(Idx *idx, int batchSize, int limit) { + for (int i = 0; i < limit; i += batchSize) { + WriteBatch wb; + idx->Write(&wb, i); + } + return 0; +} + +int BenchRead(Idx *idx) { return 0; } + +int main() { + // Idx *idx = new Idx; + // if (idx->SetUp(true) != 0) { + // std::cout << "failed to setup index" << std::endl; + // return 0; + // } else { + // std::cout << "succ to setup index" << std::endl; + // } + // BenchWrite(idx, 100, 10000); + return 1; +} diff --git a/source/libs/index/test/indexTests.cc b/source/libs/index/test/indexTests.cc index 5b76de2ef89fdce4780fcf94a1360d68f7684a9e..08bf84ff60fdc07393abf546630c67dd52f6abc1 100644 --- a/source/libs/index/test/indexTests.cc +++ b/source/libs/index/test/indexTests.cc @@ -271,20 +271,20 @@ void validateFst() { } delete m; } -static std::string logDir = TD_TMP_DIR_PATH "log"; - -static void initLog() { - const char* defaultLogFileNamePrefix = "taoslog"; - const int32_t maxLogFileNum = 10; - tsAsyncLog = 0; - idxDebugFlag = 143; - strcpy(tsLogDir, logDir.c_str()); - taosRemoveDir(tsLogDir); - taosMkDir(tsLogDir); - - if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { - printf("failed to open log file in directory:%s\n", tsLogDir); +static std::string logDir = TD_TMP_DIR_PATH "log"; +static void initLog() { + const char* defaultLogFileNamePrefix = "taoslog"; + const int32_t maxLogFileNum = 10; + + tsAsyncLog = 0; + idxDebugFlag = 143; + strcpy(tsLogDir, logDir.c_str()); + taosRemoveDir(tsLogDir); + taosMkDir(tsLogDir); + + if (taosInitLog(defaultLogFileNamePrefix, maxLogFileNum) < 0) { + printf("failed to open log file in directory:%s\n", tsLogDir); } } class IndexEnv : public ::testing::Test { diff --git a/source/libs/monitor/src/monMsg.c b/source/libs/monitor/src/monMsg.c index 8fa7e8860509ca473ad41b1c89efbf430f0c2649..bbee8b1166903bfcafb611baf7b2bf8ed8b48699 100644 --- a/source/libs/monitor/src/monMsg.c +++ b/source/libs/monitor/src/monMsg.c @@ -510,6 +510,7 @@ int32_t tSerializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInfo) SVnodeLoad *pLoad = taosArrayGet(pInfo->pVloads, i); if (tEncodeI32(&encoder, pLoad->vgId) < 0) return -1; if (tEncodeI32(&encoder, pLoad->syncState) < 0) return -1; + if (tEncodeI64(&encoder, pLoad->cacheUsage) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTables) < 0) return -1; if (tEncodeI64(&encoder, pLoad->numOfTimeSeries) < 0) return -1; if (tEncodeI64(&encoder, pLoad->totalStorage) < 0) return -1; @@ -544,6 +545,7 @@ int32_t tDeserializeSMonVloadInfo(void *buf, int32_t bufLen, SMonVloadInfo *pInf SVnodeLoad load = {0}; if (tDecodeI32(&decoder, &load.vgId) < 0) return -1; if (tDecodeI32(&decoder, &load.syncState) < 0) return -1; + if (tDecodeI64(&decoder, &load.cacheUsage) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTables) < 0) return -1; if (tDecodeI64(&decoder, &load.numOfTimeSeries) < 0) return -1; if (tDecodeI64(&decoder, &load.totalStorage) < 0) return -1; @@ -594,7 +596,6 @@ int32_t tDeserializeSMonMloadInfo(void *buf, int32_t bufLen, SMonMloadInfo *pInf return 0; } - int32_t tSerializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -639,5 +640,3 @@ int32_t tDeserializeSQnodeLoad(void *buf, int32_t bufLen, SQnodeLoad *pInfo) { tDecoderClear(&decoder); return 0; } - - diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 904b8f01eb7094f500c418409cefc8f70da4d91a..d2571a395eecdd88ef91305bfb77c2aeb5a0f5c5 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -399,6 +399,8 @@ static int32_t logicVnodeModifCopy(const SVnodeModifyLogicNode* pSrc, SVnodeModi COPY_SCALAR_FIELD(modifyType); COPY_SCALAR_FIELD(msgType); CLONE_NODE_FIELD(pAffectedRows); + CLONE_NODE_FIELD(pStartTs); + CLONE_NODE_FIELD(pEndTs); COPY_SCALAR_FIELD(tableId); COPY_SCALAR_FIELD(stableId); COPY_SCALAR_FIELD(tableType); @@ -778,6 +780,7 @@ SNode* nodesCloneNode(const SNode* pNode) { code = physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst); break; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: code = physiPartitionCopy((const SPartitionPhysiNode*)pNode, (SPartitionPhysiNode*)pDst); break; default: diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index 822bdec365e6f5128d7c36cf6b7d765eeb2488de..afd05eff1d1c6a2c895581b4be0f76e75e9ca631 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -265,6 +265,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiStreamStateWindow"; case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return "PhysiPartition"; + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + return "PhysiStreamPartition"; case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: return "PhysiIndefRowsFunc"; case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: @@ -360,18 +362,14 @@ static int32_t jsonToTableComInfo(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkTableComInfoNumOfTags, pNode->numOfTags, code); - ; if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableComInfoPrecision, pNode->precision, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableComInfoNumOfColumns, pNode->numOfColumns, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableComInfoRowSize, pNode->rowSize, code); - ; } return code; @@ -404,14 +402,11 @@ static int32_t jsonToSchema(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkSchemaType, pNode->type, code); - ; if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSchemaColId, pNode->colId, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkSchemaBytes, pNode->bytes, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkSchemaName, pNode->name); @@ -464,26 +459,20 @@ static int32_t jsonToTableMeta(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkTableMetaVgId, pNode->vgId, code); - ; if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaTableType, pNode->tableType, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaUid, pNode->uid, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaSuid, pNode->suid, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaSversion, pNode->sversion, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkTableMetaTversion, pNode->tversion, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonToObject(pJson, jkTableMetaComInfo, jsonToTableComInfo, &pNode->tableInfo); @@ -924,7 +913,6 @@ static int32_t jsonToLogicFillNode(const SJson* pJson, void* pObj) { int32_t code = jsonToLogicPlanNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkFillLogicPlanMode, pNode->mode, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillLogicPlanWStartTs, &pNode->pWStartTs); @@ -2443,6 +2431,8 @@ static const char* jkDeletePhysiPlanTsColName = "TsColName"; static const char* jkDeletePhysiPlanDeleteTimeRangeStartKey = "DeleteTimeRangeStartKey"; static const char* jkDeletePhysiPlanDeleteTimeRangeEndKey = "DeleteTimeRangeEndKey"; static const char* jkDeletePhysiPlanAffectedRows = "AffectedRows"; +static const char* jkDeletePhysiPlanStartTs = "StartTs"; +static const char* jkDeletePhysiPlanEndTs = "EndTs"; static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; @@ -2469,6 +2459,12 @@ static int32_t physiDeleteNodeToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkDeletePhysiPlanAffectedRows, nodeToJson, pNode->pAffectedRows); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeletePhysiPlanStartTs, nodeToJson, pNode->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkDeletePhysiPlanEndTs, nodeToJson, pNode->pEndTs); + } return code; } @@ -2498,6 +2494,12 @@ static int32_t jsonToPhysiDeleteNode(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkDeletePhysiPlanAffectedRows, &pNode->pAffectedRows); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeletePhysiPlanStartTs, &pNode->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkDeletePhysiPlanEndTs, &pNode->pEndTs); + } return code; } @@ -2813,7 +2815,6 @@ static int32_t jsonToColumnNode(const SJson* pJson, void* pObj) { } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkColumnColType, pNode->colType, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetStringValue(pJson, jkColumnDbName, pNode->dbName); @@ -3116,7 +3117,6 @@ static int32_t jsonToOperatorNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkOperatorType, pNode->opType, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkOperatorLeft, &pNode->pLeft); @@ -3151,7 +3151,6 @@ static int32_t jsonToLogicConditionNode(const SJson* pJson, void* pObj) { int32_t code = jsonToExprNode(pJson, pObj); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkLogicCondType, pNode->condType, code); - ; } if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeList(pJson, jkLogicCondParameters, &pNode->pParameterList); @@ -3440,11 +3439,9 @@ static int32_t jsonToOrderByExprNode(const SJson* pJson, void* pObj) { int32_t code = jsonToNodeObject(pJson, jkOrderByExprExpr, &pNode->pExpr); if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkOrderByExprOrder, pNode->order, code); - ; } if (TSDB_CODE_SUCCESS == code) { tjsonGetNumberValue(pJson, jkOrderByExprNullOrder, pNode->nullOrder, code); - ; } return code; @@ -3622,7 +3619,6 @@ static int32_t jsonToFillNode(const SJson* pJson, void* pObj) { int32_t code; tjsonGetNumberValue(pJson, jkFillMode, pNode->mode, code); - ; if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkFillValues, &pNode->pValues); } @@ -4485,6 +4481,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return physiStateWindowNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: return physiPartitionNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: return physiIndefRowsFuncNodeToJson(pObj, pJson); @@ -4632,6 +4629,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: return jsonToPhysiStateWindowNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: return jsonToPhysiPartitionNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: return jsonToPhysiIndefRowsFuncNode(pJson, pObj); diff --git a/source/libs/nodes/src/nodesMsgFuncs.c b/source/libs/nodes/src/nodesMsgFuncs.c new file mode 100644 index 0000000000000000000000000000000000000000..202ee6717a046a3149a45130189d25308f310648 --- /dev/null +++ b/source/libs/nodes/src/nodesMsgFuncs.c @@ -0,0 +1,3270 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "nodesUtil.h" +#include "plannodes.h" +#include "tdatablock.h" + +#define NODES_MSG_DEFAULT_LEN 1024 + +#define tlvForEach(pDecoder, pTlv, code) \ + while (TSDB_CODE_SUCCESS == code && TSDB_CODE_SUCCESS == (code = tlvGetNextTlv(pDecoder, &pTlv)) && NULL != pTlv) + +typedef struct STlv { + int16_t type; + int32_t len; + char value[0]; +} STlv; + +typedef struct STlvEncoder { + int32_t allocSize; + int32_t offset; + char* pBuf; + int32_t tlvCount; +} STlvEncoder; + +typedef struct STlvDecoder { + int32_t bufSize; + int32_t offset; + const char* pBuf; +} STlvDecoder; + +typedef int32_t (*FToMsg)(const void* pObj, STlvEncoder* pEncoder); +typedef int32_t (*FToObject)(STlvDecoder* pDecoder, void* pObj); +typedef void* (*FMakeObject)(int16_t type); +typedef int32_t (*FSetObject)(STlv* pTlv, void* pObj); + +static int32_t nodeToMsg(const void* pObj, STlvEncoder* pEncoder); +static int32_t nodeListToMsg(const void* pObj, STlvEncoder* pEncoder); +static int32_t msgToNode(STlvDecoder* pDecoder, void** pObj); +static int32_t msgToNodeFromTlv(STlv* pTlv, void** pObj); +static int32_t msgToNodeList(STlvDecoder* pDecoder, void** pObj); +static int32_t msgToNodeListFromTlv(STlv* pTlv, void** pObj); + +static int32_t initTlvEncoder(STlvEncoder* pEncoder) { + pEncoder->allocSize = NODES_MSG_DEFAULT_LEN; + pEncoder->offset = 0; + pEncoder->tlvCount = 0; + pEncoder->pBuf = taosMemoryMalloc(pEncoder->allocSize); + return NULL == pEncoder->pBuf ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS; +} + +static void clearTlvEncoder(STlvEncoder* pEncoder) { taosMemoryFree(pEncoder->pBuf); } + +static void endTlvEncode(STlvEncoder* pEncoder, char** pMsg, int32_t* pLen) { + *pMsg = pEncoder->pBuf; + pEncoder->pBuf = NULL; + *pLen = pEncoder->offset; + // nodesWarn("encode tlv count = %d, tl size = %d", pEncoder->tlvCount, sizeof(STlv) * pEncoder->tlvCount); +} + +static int32_t tlvEncodeImpl(STlvEncoder* pEncoder, int16_t type, const void* pValue, int32_t len) { + int32_t tlvLen = sizeof(STlv) + len; + if (pEncoder->offset + tlvLen > pEncoder->allocSize) { + void* pNewBuf = taosMemoryRealloc(pEncoder->pBuf, pEncoder->allocSize * 2); + if (NULL == pNewBuf) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pEncoder->pBuf = pNewBuf; + pEncoder->allocSize = pEncoder->allocSize * 2; + } + STlv* pTlv = (STlv*)(pEncoder->pBuf + pEncoder->offset); + pTlv->type = type; + pTlv->len = len; + memcpy(pTlv->value, pValue, len); + pEncoder->offset += tlvLen; + ++(pEncoder->tlvCount); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvEncodeI8(STlvEncoder* pEncoder, int16_t type, int8_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeI16(STlvEncoder* pEncoder, int16_t type, int16_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeI32(STlvEncoder* pEncoder, int16_t type, int32_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeI64(STlvEncoder* pEncoder, int16_t type, int64_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeU8(STlvEncoder* pEncoder, int16_t type, uint8_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeU16(STlvEncoder* pEncoder, int16_t type, uint16_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeU64(STlvEncoder* pEncoder, int16_t type, uint64_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeDouble(STlvEncoder* pEncoder, int16_t type, double value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeEnum(STlvEncoder* pEncoder, int16_t type, int32_t value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeBool(STlvEncoder* pEncoder, int16_t type, bool value) { + return tlvEncodeImpl(pEncoder, type, &value, sizeof(value)); +} + +static int32_t tlvEncodeCStr(STlvEncoder* pEncoder, int16_t type, const char* pValue) { + if (NULL == pValue) { + return TSDB_CODE_SUCCESS; + } + return tlvEncodeImpl(pEncoder, type, pValue, strlen(pValue)); +} + +static int32_t tlvEncodeBinary(STlvEncoder* pEncoder, int16_t type, const void* pValue, int32_t len) { + return tlvEncodeImpl(pEncoder, type, pValue, len); +} + +static int32_t tlvEncodeObj(STlvEncoder* pEncoder, int16_t type, FToMsg func, const void* pObj) { + if (NULL == pObj) { + return TSDB_CODE_SUCCESS; + } + + int32_t start = pEncoder->offset; + pEncoder->offset += sizeof(STlv); + int32_t code = func(pObj, pEncoder); + if (TSDB_CODE_SUCCESS == code) { + STlv* pTlv = (STlv*)(pEncoder->pBuf + start); + pTlv->type = type; + pTlv->len = pEncoder->offset - start - sizeof(STlv); + } + return code; +} + +static int32_t tlvEncodeObjArray(STlvEncoder* pEncoder, int16_t type, FToMsg func, const void* pArray, int32_t itemSize, + int32_t num) { + int32_t code = TSDB_CODE_SUCCESS; + if (num > 0) { + int32_t start = pEncoder->offset; + pEncoder->offset += sizeof(STlv); + for (size_t i = 0; TSDB_CODE_SUCCESS == code && i < num; ++i) { + code = tlvEncodeObj(pEncoder, 0, func, (const char*)pArray + i * itemSize); + } + if (TSDB_CODE_SUCCESS == code) { + STlv* pTlv = (STlv*)(pEncoder->pBuf + start); + pTlv->type = type; + pTlv->len = pEncoder->offset - start - sizeof(STlv); + } + } + return code; +} + +static int32_t tlvGetNextTlv(STlvDecoder* pDecoder, STlv** pTlv) { + if (pDecoder->offset == pDecoder->bufSize) { + *pTlv = NULL; + return TSDB_CODE_SUCCESS; + } + + *pTlv = (STlv*)(pDecoder->pBuf + pDecoder->offset); + if ((*pTlv)->len + pDecoder->offset > pDecoder->bufSize) { + return TSDB_CODE_FAILED; + } + pDecoder->offset += sizeof(STlv) + (*pTlv)->len; + return TSDB_CODE_SUCCESS; +} + +static bool tlvDecodeEnd(STlvDecoder* pDecoder) { return pDecoder->offset == pDecoder->bufSize; } + +static int32_t tlvDecodeImpl(STlv* pTlv, void* pValue, int32_t len) { + if (pTlv->len != len) { + return TSDB_CODE_FAILED; + } + memcpy(pValue, pTlv->value, len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeI8(STlv* pTlv, int8_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeI16(STlv* pTlv, int16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeI32(STlv* pTlv, int32_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeI64(STlv* pTlv, int64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeU8(STlv* pTlv, uint8_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeU16(STlv* pTlv, uint16_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeU64(STlv* pTlv, uint64_t* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeDouble(STlv* pTlv, double* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeBool(STlv* pTlv, bool* pValue) { return tlvDecodeImpl(pTlv, pValue, sizeof(*pValue)); } + +static int32_t tlvDecodeEnum(STlv* pTlv, void* pValue, int16_t len) { + int32_t value = 0; + memcpy(&value, pTlv->value, pTlv->len); + switch (len) { + case 1: + *(int8_t*)pValue = value; + break; + case 2: + *(int16_t*)pValue = value; + break; + case 4: + *(int32_t*)pValue = value; + break; + default: + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeCStr(STlv* pTlv, char* pValue) { + memcpy(pValue, pTlv->value, pTlv->len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeCStrP(STlv* pTlv, char** pValue) { + *pValue = strndup(pTlv->value, pTlv->len); + return NULL == *pValue ? TSDB_CODE_OUT_OF_MEMORY : TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeDynBinary(STlv* pTlv, void** pValue) { + *pValue = taosMemoryMalloc(pTlv->len); + if (NULL == *pValue) { + return TSDB_CODE_OUT_OF_MEMORY; + } + memcpy(*pValue, pTlv->value, pTlv->len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeBinary(STlv* pTlv, void* pValue) { + memcpy(pValue, pTlv->value, pTlv->len); + return TSDB_CODE_SUCCESS; +} + +static int32_t tlvDecodeObjFromTlv(STlv* pTlv, FToObject func, void* pObj) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return func(&decoder, pObj); +} + +static int32_t tlvDecodeObj(STlvDecoder* pDecoder, FToObject func, void* pObj) { + STlv* pTlv = NULL; + int32_t code = tlvGetNextTlv(pDecoder, &pTlv); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeObjFromTlv(pTlv, func, pObj); + } + return code; +} + +static int32_t tlvDecodeObjArray(STlvDecoder* pDecoder, FToObject func, void* pArray, int32_t itemSize) { + int32_t code = TSDB_CODE_SUCCESS; + int32_t i = 0; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { code = tlvDecodeObjFromTlv(pTlv, func, (char*)pArray + itemSize * i++); } + return code; +} + +static int32_t tlvDecodeObjArrayFromTlv(STlv* pTlv, FToObject func, void* pArray, int32_t itemSize) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return tlvDecodeObjArray(&decoder, func, pArray, itemSize); +} + +static int32_t tlvDecodeDynObjFromTlv(STlv* pTlv, FMakeObject makeFunc, FToObject toFunc, void** pObj) { + *pObj = makeFunc(pTlv->type); + if (NULL == *pObj) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return tlvDecodeObjFromTlv(pTlv, toFunc, *pObj); +} + +static int32_t tlvDecodeDynObj(STlvDecoder* pDecoder, FMakeObject makeFunc, FToObject toFunc, void** pObj) { + STlv* pTlv = NULL; + int32_t code = tlvGetNextTlv(pDecoder, &pTlv); + if (TSDB_CODE_SUCCESS == code) { + code = tlvDecodeDynObjFromTlv(pTlv, makeFunc, toFunc, pObj); + } + return code; +} + +enum { DATA_TYPE_CODE_TYPE = 1, DATA_TYPE_CODE_PRECISION, DATA_TYPE_CODE_SCALE, DATA_TYPE_CODE_BYTES }; + +static int32_t dataTypeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataType* pNode = (const SDataType*)pObj; + + int32_t code = tlvEncodeI8(pEncoder, DATA_TYPE_CODE_TYPE, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, DATA_TYPE_CODE_PRECISION, pNode->precision); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, DATA_TYPE_CODE_SCALE, pNode->scale); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DATA_TYPE_CODE_BYTES, pNode->bytes); + } + + return code; +} + +static int32_t msgToDataType(STlvDecoder* pDecoder, void* pObj) { + SDataType* pNode = (SDataType*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case DATA_TYPE_CODE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->type); + break; + case DATA_TYPE_CODE_PRECISION: + code = tlvDecodeU8(pTlv, &pNode->precision); + break; + case DATA_TYPE_CODE_SCALE: + code = tlvDecodeU8(pTlv, &pNode->scale); + break; + case DATA_TYPE_CODE_BYTES: + code = tlvDecodeI32(pTlv, &pNode->bytes); + break; + default: + break; + } + } + + return code; +} + +enum { EXPR_CODE_RES_TYPE = 1 }; + +static int32_t exprNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SExprNode* pNode = (const SExprNode*)pObj; + return tlvEncodeObj(pEncoder, EXPR_CODE_RES_TYPE, dataTypeToMsg, &pNode->resType); +} + +static int32_t msgToExprNode(STlvDecoder* pDecoder, void* pObj) { + SExprNode* pNode = (SExprNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case EXPR_CODE_RES_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->resType); + break; + default: + break; + } + } + + return code; +} + +enum { + COLUMN_CODE_EXPR_BASE = 1, + COLUMN_CODE_TABLE_ID, + COLUMN_CODE_TABLE_TYPE, + COLUMN_CODE_COLUMN_ID, + COLUMN_CODE_COLUMN_TYPE, + COLUMN_CODE_DB_NAME, + COLUMN_CODE_TABLE_NAME, + COLUMN_CODE_TABLE_ALIAS, + COLUMN_CODE_COL_NAME, + COLUMN_CODE_DATABLOCK_ID, + COLUMN_CODE_SLOT_ID +}; + +static int32_t columnNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SColumnNode* pNode = (const SColumnNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, COLUMN_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, COLUMN_CODE_TABLE_ID, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, COLUMN_CODE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, COLUMN_CODE_COLUMN_ID, pNode->colId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, COLUMN_CODE_COLUMN_TYPE, pNode->colType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, COLUMN_CODE_DB_NAME, pNode->dbName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, COLUMN_CODE_TABLE_NAME, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, COLUMN_CODE_TABLE_ALIAS, pNode->tableAlias); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, COLUMN_CODE_COL_NAME, pNode->colName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, COLUMN_CODE_DATABLOCK_ID, pNode->dataBlockId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, COLUMN_CODE_SLOT_ID, pNode->slotId); + } + + return code; +} + +static int32_t msgToColumnNode(STlvDecoder* pDecoder, void* pObj) { + SColumnNode* pNode = (SColumnNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case COLUMN_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case COLUMN_CODE_TABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->tableId); + break; + case COLUMN_CODE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case COLUMN_CODE_COLUMN_ID: + code = tlvDecodeI16(pTlv, &pNode->colId); + break; + case COLUMN_CODE_COLUMN_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->colType, sizeof(pNode->colType)); + break; + case COLUMN_CODE_DB_NAME: + code = tlvDecodeCStr(pTlv, pNode->dbName); + break; + case COLUMN_CODE_TABLE_NAME: + code = tlvDecodeCStr(pTlv, pNode->tableName); + break; + case COLUMN_CODE_TABLE_ALIAS: + code = tlvDecodeCStr(pTlv, pNode->tableAlias); + break; + case COLUMN_CODE_COL_NAME: + code = tlvDecodeCStr(pTlv, pNode->colName); + break; + case COLUMN_CODE_DATABLOCK_ID: + code = tlvDecodeI16(pTlv, &pNode->dataBlockId); + break; + case COLUMN_CODE_SLOT_ID: + code = tlvDecodeI16(pTlv, &pNode->slotId); + break; + default: + break; + } + } + + return code; +} + +enum { + VALUE_CODE_EXPR_BASE = 1, + VALUE_CODE_LITERAL, + VALUE_CODE_IS_DURATION, + VALUE_CODE_TRANSLATE, + VALUE_CODE_NOT_RESERVED, + VALUE_CODE_IS_NULL, + VALUE_CODE_DATUM +}; + +static int32_t datumToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SValueNode* pNode = (const SValueNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tlvEncodeBool(pEncoder, VALUE_CODE_DATUM, pNode->datum.b); + break; + case TSDB_DATA_TYPE_TINYINT: + case TSDB_DATA_TYPE_SMALLINT: + case TSDB_DATA_TYPE_INT: + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + code = tlvEncodeI64(pEncoder, VALUE_CODE_DATUM, pNode->datum.i); + break; + case TSDB_DATA_TYPE_UTINYINT: + case TSDB_DATA_TYPE_USMALLINT: + case TSDB_DATA_TYPE_UINT: + case TSDB_DATA_TYPE_UBIGINT: + code = tlvEncodeU64(pEncoder, VALUE_CODE_DATUM, pNode->datum.u); + break; + case TSDB_DATA_TYPE_FLOAT: + case TSDB_DATA_TYPE_DOUBLE: + code = tlvEncodeDouble(pEncoder, VALUE_CODE_DATUM, pNode->datum.d); + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_NCHAR: + code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, varDataTLen(pNode->datum.p)); + break; + case TSDB_DATA_TYPE_JSON: + code = tlvEncodeBinary(pEncoder, VALUE_CODE_DATUM, pNode->datum.p, getJsonValueLen(pNode->datum.p)); + break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + +static int32_t valueNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SValueNode* pNode = (const SValueNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, VALUE_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, VALUE_CODE_LITERAL, pNode->literal); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, VALUE_CODE_IS_DURATION, pNode->isDuration); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, VALUE_CODE_TRANSLATE, pNode->translate); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, VALUE_CODE_NOT_RESERVED, pNode->notReserved); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, VALUE_CODE_IS_NULL, pNode->isNull); + } + if (TSDB_CODE_SUCCESS == code && !pNode->isNull) { + code = datumToMsg(pNode, pEncoder); + } + + return code; +} + +static int32_t msgToDatum(STlv* pTlv, void* pObj) { + SValueNode* pNode = (SValueNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + switch (pNode->node.resType.type) { + case TSDB_DATA_TYPE_NULL: + break; + case TSDB_DATA_TYPE_BOOL: + code = tlvDecodeBool(pTlv, &pNode->datum.b); + *(bool*)&pNode->typeData = pNode->datum.b; + break; + case TSDB_DATA_TYPE_TINYINT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int8_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_SMALLINT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int16_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_INT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int32_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_BIGINT: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_TIMESTAMP: + code = tlvDecodeI64(pTlv, &pNode->datum.i); + *(int64_t*)&pNode->typeData = pNode->datum.i; + break; + case TSDB_DATA_TYPE_UTINYINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint8_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_USMALLINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint16_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint32_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_UBIGINT: + code = tlvDecodeU64(pTlv, &pNode->datum.u); + *(uint64_t*)&pNode->typeData = pNode->datum.u; + break; + case TSDB_DATA_TYPE_FLOAT: + code = tlvDecodeDouble(pTlv, &pNode->datum.d); + *(float*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_DOUBLE: + code = tlvDecodeDouble(pTlv, &pNode->datum.d); + *(double*)&pNode->typeData = pNode->datum.d; + break; + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_VARBINARY: { + pNode->datum.p = taosMemoryCalloc(1, pNode->node.resType.bytes + VARSTR_HEADER_SIZE + 1); + if (NULL == pNode->datum.p) { + code = TSDB_CODE_OUT_OF_MEMORY; + break; + } + code = tlvDecodeBinary(pTlv, pNode->datum.p); + if (TSDB_CODE_SUCCESS == code) { + varDataSetLen(pNode->datum.p, pTlv->len - VARSTR_HEADER_SIZE); + } + break; + } + case TSDB_DATA_TYPE_JSON: + code = tlvDecodeDynBinary(pTlv, (void**)&pNode->datum.p); + break; + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + // todo + default: + break; + } + + return code; +} + +static int32_t msgToValueNode(STlvDecoder* pDecoder, void* pObj) { + SValueNode* pNode = (SValueNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case VALUE_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case VALUE_CODE_LITERAL: + code = tlvDecodeCStrP(pTlv, &pNode->literal); + break; + case VALUE_CODE_IS_DURATION: + code = tlvDecodeBool(pTlv, &pNode->isDuration); + break; + case VALUE_CODE_TRANSLATE: + code = tlvDecodeBool(pTlv, &pNode->translate); + break; + case VALUE_CODE_NOT_RESERVED: + code = tlvDecodeBool(pTlv, &pNode->notReserved); + break; + case VALUE_CODE_IS_NULL: + code = tlvDecodeBool(pTlv, &pNode->isNull); + break; + case VALUE_CODE_DATUM: + code = msgToDatum(pTlv, pNode); + break; + default: + break; + } + } + + return code; +} + +enum { OPERATOR_CODE_EXPR_BASE = 1, OPERATOR_CODE_OP_TYPE, OPERATOR_CODE_LEFT, OPERATOR_CODE_RIGHT }; + +static int32_t operatorNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SOperatorNode* pNode = (const SOperatorNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, OPERATOR_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, OPERATOR_CODE_OP_TYPE, pNode->opType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, OPERATOR_CODE_LEFT, nodeToMsg, pNode->pLeft); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, OPERATOR_CODE_RIGHT, nodeToMsg, pNode->pRight); + } + + return code; +} + +static int32_t msgToOperatorNode(STlvDecoder* pDecoder, void* pObj) { + SOperatorNode* pNode = (SOperatorNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case OPERATOR_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case OPERATOR_CODE_OP_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->opType, sizeof(pNode->opType)); + break; + case OPERATOR_CODE_LEFT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pLeft); + break; + case OPERATOR_CODE_RIGHT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pRight); + break; + default: + break; + } + } + + return code; +} + +enum { LOGIC_COND_CODE_EXPR_BASE = 1, LOGIC_COND_CODE_COND_TYPE, LOGIC_COND_CODE_PARAMETERS }; + +static int32_t logicConditionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SLogicConditionNode* pNode = (const SLogicConditionNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, LOGIC_COND_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, LOGIC_COND_CODE_COND_TYPE, pNode->condType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, LOGIC_COND_CODE_PARAMETERS, nodeListToMsg, pNode->pParameterList); + } + + return code; +} + +static int32_t msgToLogicConditionNode(STlvDecoder* pDecoder, void* pObj) { + SLogicConditionNode* pNode = (SLogicConditionNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case LOGIC_COND_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case LOGIC_COND_CODE_COND_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->condType, sizeof(pNode->condType)); + break; + case LOGIC_COND_CODE_PARAMETERS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pParameterList); + break; + default: + break; + } + } + + return code; +} + +enum { + FUNCTION_CODE_EXPR_BASE = 1, + FUNCTION_CODE_FUNCTION_NAME, + FUNCTION_CODE_FUNCTION_ID, + FUNCTION_CODE_FUNCTION_TYPE, + FUNCTION_CODE_PARAMETERS, + FUNCTION_CODE_UDF_BUF_SIZE +}; + +static int32_t functionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SFunctionNode* pNode = (const SFunctionNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, FUNCTION_CODE_EXPR_BASE, exprNodeToMsg, pNode); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, FUNCTION_CODE_FUNCTION_NAME, pNode->functionName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, FUNCTION_CODE_FUNCTION_ID, pNode->funcId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, FUNCTION_CODE_FUNCTION_TYPE, pNode->funcType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, FUNCTION_CODE_PARAMETERS, nodeListToMsg, pNode->pParameterList); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, FUNCTION_CODE_UDF_BUF_SIZE, pNode->udfBufSize); + } + + return code; +} + +static int32_t msgToFunctionNode(STlvDecoder* pDecoder, void* pObj) { + SFunctionNode* pNode = (SFunctionNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case FUNCTION_CODE_EXPR_BASE: + code = tlvDecodeObjFromTlv(pTlv, msgToExprNode, &pNode->node); + break; + case FUNCTION_CODE_FUNCTION_NAME: + code = tlvDecodeCStr(pTlv, pNode->functionName); + break; + case FUNCTION_CODE_FUNCTION_ID: + code = tlvDecodeI32(pTlv, &pNode->funcId); + break; + case FUNCTION_CODE_FUNCTION_TYPE: + code = tlvDecodeI32(pTlv, &pNode->funcType); + break; + case FUNCTION_CODE_PARAMETERS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pParameterList); + break; + case FUNCTION_CODE_UDF_BUF_SIZE: + code = tlvDecodeI32(pTlv, &pNode->udfBufSize); + break; + default: + break; + } + } + + return code; +} + +enum { ORDER_BY_EXPR_CODE_EXPR = 1, ORDER_BY_EXPR_CODE_ORDER, ORDER_BY_EXPR_CODE_NULL_ORDER }; + +static int32_t orderByExprNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SOrderByExprNode* pNode = (const SOrderByExprNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, ORDER_BY_EXPR_CODE_EXPR, nodeToMsg, pNode->pExpr); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, ORDER_BY_EXPR_CODE_ORDER, pNode->order); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, ORDER_BY_EXPR_CODE_NULL_ORDER, pNode->nullOrder); + } + + return code; +} + +static int32_t msgToOrderByExprNode(STlvDecoder* pDecoder, void* pObj) { + SOrderByExprNode* pNode = (SOrderByExprNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case ORDER_BY_EXPR_CODE_EXPR: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pExpr); + break; + case ORDER_BY_EXPR_CODE_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->order, sizeof(pNode->order)); + break; + case ORDER_BY_EXPR_CODE_NULL_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->nullOrder, sizeof(pNode->nullOrder)); + break; + default: + break; + } + } + + return code; +} + +enum { LIMIT_CODE_LIMIT = 1, LIMIT_CODE_OFFSET }; + +static int32_t limitNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SLimitNode* pNode = (const SLimitNode*)pObj; + + int32_t code = tlvEncodeI64(pEncoder, LIMIT_CODE_LIMIT, pNode->limit); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, LIMIT_CODE_OFFSET, pNode->offset); + } + + return code; +} + +static int32_t msgToLimitNode(STlvDecoder* pDecoder, void* pObj) { + SLimitNode* pNode = (SLimitNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case LIMIT_CODE_LIMIT: + code = tlvDecodeI64(pTlv, &pNode->limit); + break; + case LIMIT_CODE_OFFSET: + code = tlvDecodeI64(pTlv, &pNode->offset); + break; + default: + break; + } + } + + return code; +} + +enum { NAME_CODE_TYPE = 1, NAME_CODE_ACCT_ID, NAME_CODE_DB_NAME, NAME_CODE_TABLE_NAME }; + +static int32_t nameToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SName* pNode = (const SName*)pObj; + + int32_t code = tlvEncodeU8(pEncoder, NAME_CODE_TYPE, pNode->type); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, NAME_CODE_ACCT_ID, pNode->acctId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, NAME_CODE_DB_NAME, pNode->dbname); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, NAME_CODE_TABLE_NAME, pNode->tname); + } + + return code; +} + +static int32_t msgToName(STlvDecoder* pDecoder, void* pObj) { + SName* pNode = (SName*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case NAME_CODE_TYPE: + code = tlvDecodeU8(pTlv, &pNode->type); + break; + case NAME_CODE_ACCT_ID: + code = tlvDecodeI32(pTlv, &pNode->acctId); + break; + case NAME_CODE_DB_NAME: + code = tlvDecodeCStr(pTlv, pNode->dbname); + break; + case NAME_CODE_TABLE_NAME: + code = tlvDecodeCStr(pTlv, pNode->tname); + break; + default: + break; + } + } + + return code; +} + +enum { TIME_WINDOW_CODE_START_KEY = 1, TIME_WINDOW_CODE_END_KEY }; + +static int32_t timeWindowToMsg(const void* pObj, STlvEncoder* pEncoder) { + const STimeWindow* pNode = (const STimeWindow*)pObj; + + int32_t code = tlvEncodeI64(pEncoder, TIME_WINDOW_CODE_START_KEY, pNode->skey); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, TIME_WINDOW_CODE_END_KEY, pNode->ekey); + } + + return code; +} + +static int32_t msgToTimeWindow(STlvDecoder* pDecoder, void* pObj) { + STimeWindow* pNode = (STimeWindow*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case TIME_WINDOW_CODE_START_KEY: + code = tlvDecodeI64(pTlv, &pNode->skey); + break; + case TIME_WINDOW_CODE_END_KEY: + code = tlvDecodeI64(pTlv, &pNode->ekey); + break; + default: + break; + } + } + + return code; +} + +enum { NODE_LIST_CODE_DATA_TYPE = 1, NODE_LIST_CODE_NODE_LIST }; + +static int32_t nodeListNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SNodeListNode* pNode = (const SNodeListNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_DATA_TYPE, dataTypeToMsg, &pNode->dataType); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, NODE_LIST_CODE_NODE_LIST, nodeListToMsg, pNode->pNodeList); + } + + return code; +} + +static int32_t msgToNodeListNode(STlvDecoder* pDecoder, void* pObj) { + SNodeListNode* pNode = (SNodeListNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case NODE_LIST_CODE_DATA_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->dataType); + break; + case NODE_LIST_CODE_NODE_LIST: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pNodeList); + break; + default: + break; + } + } + + return code; +} + +enum { TARGET_CODE_DATA_BLOCK_ID = 1, TARGET_CODE_SLOT_ID, TARGET_CODE_EXPR }; + +static int32_t targetNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const STargetNode* pNode = (const STargetNode*)pObj; + + int32_t code = tlvEncodeI16(pEncoder, TARGET_CODE_DATA_BLOCK_ID, pNode->dataBlockId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI16(pEncoder, TARGET_CODE_SLOT_ID, pNode->slotId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, TARGET_CODE_EXPR, nodeToMsg, pNode->pExpr); + } + + return code; +} + +static int32_t msgToTargetNode(STlvDecoder* pDecoder, void* pObj) { + STargetNode* pNode = (STargetNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case TARGET_CODE_DATA_BLOCK_ID: + code = tlvDecodeI16(pTlv, &pNode->dataBlockId); + break; + case TARGET_CODE_SLOT_ID: + code = tlvDecodeI16(pTlv, &pNode->slotId); + break; + case TARGET_CODE_EXPR: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pExpr); + break; + default: + break; + } + } + + return code; +} + +enum { + DATA_BLOCK_DESC_CODE_DATA_BLOCK_ID = 1, + DATA_BLOCK_DESC_CODE_SLOTS, + DATA_BLOCK_DESC_CODE_TOTAL_ROW_SIZE, + DATA_BLOCK_DESC_CODE_OUTPUT_ROW_SIZE, + DATA_BLOCK_DESC_CODE_PRECISION +}; + +static int32_t dataBlockDescNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataBlockDescNode* pNode = (const SDataBlockDescNode*)pObj; + + int32_t code = tlvEncodeI16(pEncoder, DATA_BLOCK_DESC_CODE_DATA_BLOCK_ID, pNode->dataBlockId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, DATA_BLOCK_DESC_CODE_SLOTS, nodeListToMsg, pNode->pSlots); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DATA_BLOCK_DESC_CODE_TOTAL_ROW_SIZE, pNode->totalRowSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DATA_BLOCK_DESC_CODE_OUTPUT_ROW_SIZE, pNode->outputRowSize); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, DATA_BLOCK_DESC_CODE_PRECISION, pNode->precision); + } + + return code; +} + +static int32_t msgToDataBlockDescNode(STlvDecoder* pDecoder, void* pObj) { + SDataBlockDescNode* pNode = (SDataBlockDescNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case DATA_BLOCK_DESC_CODE_DATA_BLOCK_ID: + code = tlvDecodeI16(pTlv, &pNode->dataBlockId); + break; + case DATA_BLOCK_DESC_CODE_SLOTS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSlots); + break; + case DATA_BLOCK_DESC_CODE_TOTAL_ROW_SIZE: + code = tlvDecodeI32(pTlv, &pNode->totalRowSize); + break; + case DATA_BLOCK_DESC_CODE_OUTPUT_ROW_SIZE: + code = tlvDecodeI32(pTlv, &pNode->outputRowSize); + break; + case DATA_BLOCK_DESC_CODE_PRECISION: + code = tlvDecodeU8(pTlv, &pNode->precision); + break; + default: + break; + } + } + + return code; +} + +enum { + SLOT_DESC_CODE_SLOT_ID = 1, + SLOT_DESC_CODE_DATA_TYPE, + SLOT_DESC_CODE_RESERVE, + SLOT_DESC_CODE_OUTPUT, + SLOT_DESC_CODE_TAG +}; + +static int32_t slotDescNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSlotDescNode* pNode = (const SSlotDescNode*)pObj; + + int32_t code = tlvEncodeI16(pEncoder, SLOT_DESC_CODE_SLOT_ID, pNode->slotId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SLOT_DESC_CODE_DATA_TYPE, dataTypeToMsg, &pNode->dataType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, SLOT_DESC_CODE_RESERVE, pNode->reserve); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, SLOT_DESC_CODE_OUTPUT, pNode->output); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, SLOT_DESC_CODE_TAG, pNode->tag); + } + + return code; +} + +static int32_t msgToSlotDescNode(STlvDecoder* pDecoder, void* pObj) { + SSlotDescNode* pNode = (SSlotDescNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case SLOT_DESC_CODE_SLOT_ID: + code = tlvDecodeI16(pTlv, &pNode->slotId); + break; + case SLOT_DESC_CODE_DATA_TYPE: + code = tlvDecodeObjFromTlv(pTlv, msgToDataType, &pNode->dataType); + break; + case SLOT_DESC_CODE_RESERVE: + code = tlvDecodeBool(pTlv, &pNode->reserve); + break; + case SLOT_DESC_CODE_OUTPUT: + code = tlvDecodeBool(pTlv, &pNode->output); + break; + case SLOT_DESC_CODE_TAG: + code = tlvDecodeBool(pTlv, &pNode->tag); + break; + default: + break; + } + } + + return code; +} + +enum { EP_CODE_FQDN = 1, EP_CODE_port }; + +static int32_t epToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEp* pNode = (const SEp*)pObj; + + int32_t code = tlvEncodeCStr(pEncoder, EP_CODE_FQDN, pNode->fqdn); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU16(pEncoder, EP_CODE_port, pNode->port); + } + + return code; +} + +static int32_t msgToEp(STlvDecoder* pDecoder, void* pObj) { + SEp* pNode = (SEp*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case EP_CODE_FQDN: + code = tlvDecodeCStr(pTlv, pNode->fqdn); + break; + case EP_CODE_port: + code = tlvDecodeU16(pTlv, &pNode->port); + break; + default: + break; + } + } + + return code; +} + +enum { EP_SET_CODE_IN_USE = 1, EP_SET_CODE_NUM_OF_EPS, EP_SET_CODE_EPS }; + +static int32_t epSetToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SEpSet* pNode = (const SEpSet*)pObj; + + int32_t code = tlvEncodeI8(pEncoder, EP_SET_CODE_IN_USE, pNode->inUse); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, EP_SET_CODE_NUM_OF_EPS, pNode->numOfEps); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObjArray(pEncoder, EP_SET_CODE_EPS, epToMsg, pNode->eps, sizeof(SEp), pNode->numOfEps); + } + + return code; +} + +static int32_t msgToEpSet(STlvDecoder* pDecoder, void* pObj) { + SEpSet* pNode = (SEpSet*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case EP_SET_CODE_IN_USE: + code = tlvDecodeI8(pTlv, &pNode->inUse); + break; + case EP_SET_CODE_NUM_OF_EPS: + code = tlvDecodeI8(pTlv, &pNode->numOfEps); + break; + case EP_SET_CODE_EPS: + code = tlvDecodeObjArrayFromTlv(pTlv, msgToEp, pNode->eps, sizeof(SEp)); + break; + default: + break; + } + } + + return code; +} + +enum { QUERY_NODE_ADDR_CODE_NODE_ID = 1, QUERY_NODE_ADDR_CODE_EP_SET }; + +static int32_t queryNodeAddrToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SQueryNodeAddr* pNode = (const SQueryNodeAddr*)pObj; + + int32_t code = tlvEncodeI32(pEncoder, QUERY_NODE_ADDR_CODE_NODE_ID, pNode->nodeId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, QUERY_NODE_ADDR_CODE_EP_SET, epSetToMsg, &pNode->epSet); + } + + return code; +} + +static int32_t msgToQueryNodeAddr(STlvDecoder* pDecoder, void* pObj) { + SQueryNodeAddr* pNode = (SQueryNodeAddr*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case QUERY_NODE_ADDR_CODE_NODE_ID: + code = tlvDecodeI32(pTlv, &pNode->nodeId); + break; + case QUERY_NODE_ADDR_CODE_EP_SET: + code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->epSet); + break; + } + } + + return code; +} + +enum { + DOWNSTREAM_SOURCE_CODE_ADDR = 1, + DOWNSTREAM_SOURCE_CODE_TASK_ID, + DOWNSTREAM_SOURCE_CODE_SCHED_ID, + DOWNSTREAM_SOURCE_CODE_EXEC_ID, + DOWNSTREAM_SOURCE_CODE_FETCH_MSG_TYPE +}; + +static int32_t downstreamSourceNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDownstreamSourceNode* pNode = (const SDownstreamSourceNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, DOWNSTREAM_SOURCE_CODE_ADDR, queryNodeAddrToMsg, &pNode->addr); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, DOWNSTREAM_SOURCE_CODE_TASK_ID, pNode->taskId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, DOWNSTREAM_SOURCE_CODE_SCHED_ID, pNode->schedId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DOWNSTREAM_SOURCE_CODE_EXEC_ID, pNode->execId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, DOWNSTREAM_SOURCE_CODE_FETCH_MSG_TYPE, pNode->fetchMsgType); + } + + return code; +} + +static int32_t msgToDownstreamSourceNode(STlvDecoder* pDecoder, void* pObj) { + SDownstreamSourceNode* pNode = (SDownstreamSourceNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case DOWNSTREAM_SOURCE_CODE_ADDR: + code = tlvDecodeObjFromTlv(pTlv, msgToQueryNodeAddr, &pNode->addr); + break; + case DOWNSTREAM_SOURCE_CODE_TASK_ID: + code = tlvDecodeU64(pTlv, &pNode->taskId); + break; + case DOWNSTREAM_SOURCE_CODE_SCHED_ID: + code = tlvDecodeU64(pTlv, &pNode->schedId); + break; + case DOWNSTREAM_SOURCE_CODE_EXEC_ID: + code = tlvDecodeI32(pTlv, &pNode->execId); + break; + case DOWNSTREAM_SOURCE_CODE_FETCH_MSG_TYPE: + code = tlvDecodeI32(pTlv, &pNode->fetchMsgType); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_NODE_CODE_OUTPUT_DESC = 1, + PHY_NODE_CODE_CONDITIONS, + PHY_NODE_CODE_CHILDREN, + PHY_NODE_CODE_LIMIT, + PHY_NODE_CODE_SLIMIT +}; + +static int32_t physiNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SPhysiNode* pNode = (const SPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_OUTPUT_DESC, nodeToMsg, pNode->pOutputDataBlockDesc); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_CONDITIONS, nodeToMsg, pNode->pConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_CHILDREN, nodeListToMsg, pNode->pChildren); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_LIMIT, nodeToMsg, pNode->pLimit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_NODE_CODE_SLIMIT, nodeToMsg, pNode->pSlimit); + } + + return code; +} + +static int32_t msgToPhysiNode(STlvDecoder* pDecoder, void* pObj) { + SPhysiNode* pNode = (SPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_NODE_CODE_OUTPUT_DESC: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOutputDataBlockDesc); + break; + case PHY_NODE_CODE_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pConditions); + break; + case PHY_NODE_CODE_CHILDREN: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pChildren); + break; + case PHY_NODE_CODE_LIMIT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pLimit); + break; + case PHY_NODE_CODE_SLIMIT: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pSlimit); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_SCAN_CODE_BASE_NODE = 1, + PHY_SCAN_CODE_SCAN_COLS, + PHY_SCAN_CODE_SCAN_PSEUDO_COLS, + PHY_SCAN_CODE_BASE_UID, + PHY_SCAN_CODE_BASE_SUID, + PHY_SCAN_CODE_BASE_TABLE_TYPE, + PHY_SCAN_CODE_BASE_TABLE_NAME +}; + +static int32_t physiScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SScanPhysiNode* pNode = (const SScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_SCAN_COLS, nodeListToMsg, pNode->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_SCAN_PSEUDO_COLS, nodeListToMsg, pNode->pScanPseudoCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_SCAN_CODE_BASE_UID, pNode->uid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_SCAN_CODE_BASE_SUID, pNode->suid); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_SCAN_CODE_BASE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SCAN_CODE_BASE_TABLE_NAME, nameToMsg, &pNode->tableName); + } + + return code; +} + +static int32_t msgToPhysiScanNode(STlvDecoder* pDecoder, void* pObj) { + SScanPhysiNode* pNode = (SScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SCAN_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_SCAN_CODE_SCAN_COLS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pScanCols); + break; + case PHY_SCAN_CODE_SCAN_PSEUDO_COLS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pScanPseudoCols); + break; + case PHY_SCAN_CODE_BASE_UID: + code = tlvDecodeU64(pTlv, &pNode->uid); + break; + case PHY_SCAN_CODE_BASE_SUID: + code = tlvDecodeU64(pTlv, &pNode->suid); + break; + case PHY_SCAN_CODE_BASE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case PHY_SCAN_CODE_BASE_TABLE_NAME: + code = tlvDecodeObjFromTlv(pTlv, msgToName, &pNode->tableName); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_LAST_ROW_SCAN_CODE_SCAN = 1, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT }; + +static int32_t physiLastRowScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SLastRowScanPhysiNode* pNode = (const SLastRowScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS, nodeListToMsg, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_LAST_ROW_SCAN_CODE_GROUP_SORT, pNode->groupSort); + } + + return code; +} + +static int32_t msgToPhysiLastRowScanNode(STlvDecoder* pDecoder, void* pObj) { + SLastRowScanPhysiNode* pNode = (SLastRowScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_LAST_ROW_SCAN_CODE_SCAN: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan); + break; + case PHY_LAST_ROW_SCAN_CODE_GROUP_TAGS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupTags); + break; + case PHY_LAST_ROW_SCAN_CODE_GROUP_SORT: + code = tlvDecodeBool(pTlv, &pNode->groupSort); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_TABLE_SCAN_CODE_SCAN = 1, + PHY_TABLE_SCAN_CODE_SCAN_COUNT, + PHY_TABLE_SCAN_CODE_REVERSE_SCAN_COUNT, + PHY_TABLE_SCAN_CODE_SCAN_RANGE, + PHY_TABLE_SCAN_CODE_RATIO, + PHY_TABLE_SCAN_CODE_DATA_REQUIRED, + PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS, + PHY_TABLE_SCAN_CODE_GROUP_TAGS, + PHY_TABLE_SCAN_CODE_GROUP_SORT, + PHY_TABLE_SCAN_CODE_INTERVAL, + PHY_TABLE_SCAN_CODE_OFFSET, + PHY_TABLE_SCAN_CODE_SLIDING, + PHY_TABLE_SCAN_CODE_INTERVAL_UNIT, + PHY_TABLE_SCAN_CODE_SLIDING_UNIT, + PHY_TABLE_SCAN_CODE_TRIGGER_TYPE, + PHY_TABLE_SCAN_CODE_WATERMARK, + PHY_TABLE_SCAN_CODE_IG_EXPIRED, + PHY_TABLE_SCAN_CODE_ASSIGN_BLOCK_UID, +}; + +static int32_t physiTableScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const STableScanPhysiNode* pNode = (const STableScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, PHY_TABLE_SCAN_CODE_SCAN_COUNT, pNode->scanSeq[0]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU8(pEncoder, PHY_TABLE_SCAN_CODE_REVERSE_SCAN_COUNT, pNode->scanSeq[1]); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_SCAN_RANGE, timeWindowToMsg, &pNode->scanRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeDouble(pEncoder, PHY_TABLE_SCAN_CODE_RATIO, pNode->ratio); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_TABLE_SCAN_CODE_DATA_REQUIRED, pNode->dataRequired); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS, nodeListToMsg, pNode->pDynamicScanFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_TABLE_SCAN_CODE_GROUP_TAGS, nodeListToMsg, pNode->pGroupTags); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_TABLE_SCAN_CODE_GROUP_SORT, pNode->groupSort); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_INTERVAL, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_OFFSET, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_SLIDING, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_INTERVAL_UNIT, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_SLIDING_UNIT, pNode->slidingUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_TRIGGER_TYPE, pNode->triggerType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_TABLE_SCAN_CODE_WATERMARK, pNode->watermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_TABLE_SCAN_CODE_IG_EXPIRED, pNode->igExpired); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_TABLE_SCAN_CODE_ASSIGN_BLOCK_UID, pNode->assignBlockUid); + } + + return code; +} + +static int32_t msgToPhysiTableScanNode(STlvDecoder* pDecoder, void* pObj) { + STableScanPhysiNode* pNode = (STableScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_TABLE_SCAN_CODE_SCAN: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan); + break; + case PHY_TABLE_SCAN_CODE_SCAN_COUNT: + code = tlvDecodeU8(pTlv, pNode->scanSeq); + break; + case PHY_TABLE_SCAN_CODE_REVERSE_SCAN_COUNT: + code = tlvDecodeU8(pTlv, pNode->scanSeq + 1); + break; + case PHY_TABLE_SCAN_CODE_SCAN_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->scanRange); + break; + case PHY_TABLE_SCAN_CODE_RATIO: + code = tlvDecodeDouble(pTlv, &pNode->ratio); + break; + case PHY_TABLE_SCAN_CODE_DATA_REQUIRED: + code = tlvDecodeI32(pTlv, &pNode->dataRequired); + break; + case PHY_TABLE_SCAN_CODE_DYN_SCAN_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pDynamicScanFuncs); + break; + case PHY_TABLE_SCAN_CODE_GROUP_TAGS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupTags); + break; + case PHY_TABLE_SCAN_CODE_GROUP_SORT: + code = tlvDecodeBool(pTlv, &pNode->groupSort); + break; + case PHY_TABLE_SCAN_CODE_INTERVAL: + code = tlvDecodeI64(pTlv, &pNode->interval); + break; + case PHY_TABLE_SCAN_CODE_OFFSET: + code = tlvDecodeI64(pTlv, &pNode->offset); + break; + case PHY_TABLE_SCAN_CODE_SLIDING: + code = tlvDecodeI64(pTlv, &pNode->sliding); + break; + case PHY_TABLE_SCAN_CODE_INTERVAL_UNIT: + code = tlvDecodeI8(pTlv, &pNode->intervalUnit); + break; + case PHY_TABLE_SCAN_CODE_SLIDING_UNIT: + code = tlvDecodeI8(pTlv, &pNode->slidingUnit); + break; + case PHY_TABLE_SCAN_CODE_TRIGGER_TYPE: + code = tlvDecodeI8(pTlv, &pNode->triggerType); + break; + case PHY_TABLE_SCAN_CODE_WATERMARK: + code = tlvDecodeI64(pTlv, &pNode->watermark); + break; + case PHY_TABLE_SCAN_CODE_IG_EXPIRED: + code = tlvDecodeI8(pTlv, &pNode->igExpired); + break; + case PHY_TABLE_SCAN_CODE_ASSIGN_BLOCK_UID: + code = tlvDecodeBool(pTlv, &pNode->assignBlockUid); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_SYSTABLE_SCAN_CODE_SCAN = 1, + PHY_SYSTABLE_SCAN_CODE_MGMT_EP_SET, + PHY_SYSTABLE_SCAN_CODE_SHOW_REWRITE, + PHY_SYSTABLE_SCAN_CODE_ACCOUNT_ID, + PHY_SYSTABLE_SCAN_CODE_SYS_INFO +}; + +static int32_t physiSysTableScanNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSystemTableScanPhysiNode* pNode = (const SSystemTableScanPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SYSTABLE_SCAN_CODE_SCAN, physiScanNodeToMsg, &pNode->scan); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SYSTABLE_SCAN_CODE_MGMT_EP_SET, epSetToMsg, &pNode->mgmtEpSet); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SYSTABLE_SCAN_CODE_SHOW_REWRITE, pNode->showRewrite); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_SYSTABLE_SCAN_CODE_ACCOUNT_ID, pNode->accountId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_SYSTABLE_SCAN_CODE_SYS_INFO, pNode->sysInfo); + } + + return code; +} + +static int32_t msgToPhysiSysTableScanNode(STlvDecoder* pDecoder, void* pObj) { + SSystemTableScanPhysiNode* pNode = (SSystemTableScanPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SYSTABLE_SCAN_CODE_SCAN: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiScanNode, &pNode->scan); + break; + case PHY_SYSTABLE_SCAN_CODE_MGMT_EP_SET: + code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->mgmtEpSet); + break; + case PHY_SYSTABLE_SCAN_CODE_SHOW_REWRITE: + code = tlvDecodeBool(pTlv, &pNode->showRewrite); + break; + case PHY_SYSTABLE_SCAN_CODE_ACCOUNT_ID: + code = tlvDecodeI32(pTlv, &pNode->accountId); + break; + case PHY_SYSTABLE_SCAN_CODE_SYS_INFO: + code = tlvDecodeBool(pTlv, &pNode->sysInfo); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_PROJECT_CODE_BASE_NODE = 1, + PHY_PROJECT_CODE_PROJECTIONS, + PHY_PROJECT_CODE_MERGE_DATA_BLOCK, + PHY_PROJECT_CODE_IGNORE_GROUP_ID +}; + +static int32_t physiProjectNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SProjectPhysiNode* pNode = (const SProjectPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_PROJECT_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PROJECT_CODE_PROJECTIONS, nodeListToMsg, pNode->pProjections); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_PROJECT_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_PROJECT_CODE_IGNORE_GROUP_ID, pNode->ignoreGroupId); + } + + return code; +} + +static int32_t msgToPhysiProjectNode(STlvDecoder* pDecoder, void* pObj) { + SProjectPhysiNode* pNode = (SProjectPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_PROJECT_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_PROJECT_CODE_PROJECTIONS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pProjections); + break; + case PHY_PROJECT_CODE_MERGE_DATA_BLOCK: + code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); + break; + case PHY_PROJECT_CODE_IGNORE_GROUP_ID: + code = tlvDecodeBool(pTlv, &pNode->ignoreGroupId); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_SORT_MERGE_JOIN_CODE_BASE_NODE = 1, + PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, + PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, + PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, + PHY_SORT_MERGE_JOIN_CODE_TARGETS, + PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER +}; + +static int32_t physiJoinNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSortMergeJoinPhysiNode* pNode = (const SSortMergeJoinPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE, pNode->joinType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION, nodeToMsg, pNode->pMergeCondition); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS, nodeToMsg, pNode->pOnConditions); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_MERGE_JOIN_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + + return code; +} + +static int32_t msgToPhysiJoinNode(STlvDecoder* pDecoder, void* pObj) { + SSortMergeJoinPhysiNode* pNode = (SSortMergeJoinPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SORT_MERGE_JOIN_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_SORT_MERGE_JOIN_CODE_JOIN_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->joinType, sizeof(pNode->joinType)); + break; + case PHY_SORT_MERGE_JOIN_CODE_MERGE_CONDITION: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pMergeCondition); + break; + case PHY_SORT_MERGE_JOIN_CODE_ON_CONDITIONS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pOnConditions); + break; + case PHY_SORT_MERGE_JOIN_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + case PHY_SORT_MERGE_JOIN_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_AGG_CODE_BASE_NODE = 1, + PHY_AGG_CODE_EXPR, + PHY_AGG_CODE_GROUP_KEYS, + PHY_AGG_CODE_AGG_FUNCS, + PHY_AGG_CODE_MERGE_DATA_BLOCK +}; + +static int32_t physiAggNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SAggPhysiNode* pNode = (const SAggPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_GROUP_KEYS, nodeListToMsg, pNode->pGroupKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_AGG_CODE_AGG_FUNCS, nodeListToMsg, pNode->pAggFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_AGG_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); + } + + return code; +} + +static int32_t msgToPhysiAggNode(STlvDecoder* pDecoder, void* pObj) { + SAggPhysiNode* pNode = (SAggPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_AGG_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_AGG_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_AGG_CODE_GROUP_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pGroupKeys); + break; + case PHY_AGG_CODE_AGG_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pAggFuncs); + break; + case PHY_AGG_CODE_MERGE_DATA_BLOCK: + code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_EXCHANGE_CODE_BASE_NODE = 1, + PHY_EXCHANGE_CODE_SRC_GROUP_ID, + PHY_EXCHANGE_CODE_SINGLE_CHANNEL, + PHY_EXCHANGE_CODE_SRC_ENDPOINTS +}; + +static int32_t physiExchangeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SExchangePhysiNode* pNode = (const SExchangePhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_EXCHANGE_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_EXCHANGE_CODE_SRC_GROUP_ID, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_EXCHANGE_CODE_SINGLE_CHANNEL, pNode->singleChannel); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_EXCHANGE_CODE_SRC_ENDPOINTS, nodeListToMsg, pNode->pSrcEndPoints); + } + + return code; +} + +static int32_t msgToPhysiExchangeNode(STlvDecoder* pDecoder, void* pObj) { + SExchangePhysiNode* pNode = (SExchangePhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_EXCHANGE_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_EXCHANGE_CODE_SRC_GROUP_ID: + code = tlvDecodeI32(pTlv, &pNode->srcGroupId); + break; + case PHY_EXCHANGE_CODE_SINGLE_CHANNEL: + code = tlvDecodeBool(pTlv, &pNode->singleChannel); + break; + case PHY_EXCHANGE_CODE_SRC_ENDPOINTS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSrcEndPoints); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_MERGE_CODE_BASE_NODE = 1, + PHY_MERGE_CODE_MERGE_KEYS, + PHY_MERGE_CODE_TARGETS, + PHY_MERGE_CODE_NUM_OF_CHANNELS, + PHY_MERGE_CODE_SRC_GROUP_ID, + PHY_MERGE_CODE_GROUP_SORT +}; + +static int32_t physiMergeNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SMergePhysiNode* pNode = (const SMergePhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_MERGE_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_MERGE_CODE_MERGE_KEYS, nodeListToMsg, pNode->pMergeKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_MERGE_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_MERGE_CODE_NUM_OF_CHANNELS, pNode->numOfChannels); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_MERGE_CODE_SRC_GROUP_ID, pNode->srcGroupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_MERGE_CODE_GROUP_SORT, pNode->groupSort); + } + + return code; +} + +static int32_t msgToPhysiMergeNode(STlvDecoder* pDecoder, void* pObj) { + SMergePhysiNode* pNode = (SMergePhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_MERGE_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_MERGE_CODE_MERGE_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pMergeKeys); + break; + case PHY_MERGE_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + case PHY_MERGE_CODE_NUM_OF_CHANNELS: + code = tlvDecodeI32(pTlv, &pNode->numOfChannels); + break; + case PHY_MERGE_CODE_SRC_GROUP_ID: + code = tlvDecodeI32(pTlv, &pNode->srcGroupId); + break; + case PHY_MERGE_CODE_GROUP_SORT: + code = tlvDecodeBool(pTlv, &pNode->groupSort); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_SORT_CODE_BASE_NODE = 1, PHY_SORT_CODE_EXPR, PHY_SORT_CODE_SORT_KEYS, PHY_SORT_CODE_TARGETS }; + +static int32_t physiSortNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSortPhysiNode* pNode = (const SSortPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_SORT_KEYS, nodeListToMsg, pNode->pSortKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_SORT_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + + return code; +} + +static int32_t msgToPhysiSortNode(STlvDecoder* pDecoder, void* pObj) { + SSortPhysiNode* pNode = (SSortPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SORT_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_SORT_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_SORT_CODE_SORT_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSortKeys); + break; + case PHY_SORT_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_WINDOW_CODE_BASE_NODE = 1, + PHY_WINDOW_CODE_EXPR, + PHY_WINDOW_CODE_FUNCS, + PHY_WINDOW_CODE_TS_PK, + PHY_WINDOW_CODE_TS_END, + PHY_WINDOW_CODE_TRIGGER_TYPE, + PHY_WINDOW_CODE_WATERMARK, + PHY_WINDOW_CODE_IG_EXPIRED, + PHY_WINDOW_CODE_INPUT_TS_ORDER, + PHY_WINDOW_CODE_OUTPUT_TS_ORDER, + PHY_WINDOW_CODE_MERGE_DATA_BLOCK +}; + +static int32_t physiWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SWinodwPhysiNode* pNode = (const SWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_FUNCS, nodeListToMsg, pNode->pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_TS_PK, nodeToMsg, pNode->pTspk); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_WINDOW_CODE_TS_END, nodeToMsg, pNode->pTsEnd); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_WINDOW_CODE_TRIGGER_TYPE, pNode->triggerType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_WINDOW_CODE_WATERMARK, pNode->watermark); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_WINDOW_CODE_IG_EXPIRED, pNode->igExpired); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_WINDOW_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_WINDOW_CODE_OUTPUT_TS_ORDER, pNode->outputTsOrder); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeBool(pEncoder, PHY_WINDOW_CODE_MERGE_DATA_BLOCK, pNode->mergeDataBlock); + } + + return code; +} + +static int32_t msgToPhysiWindowNode(STlvDecoder* pDecoder, void* pObj) { + SWinodwPhysiNode* pNode = (SWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_WINDOW_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_WINDOW_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_WINDOW_CODE_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs); + break; + case PHY_WINDOW_CODE_TS_PK: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTspk); + break; + case PHY_WINDOW_CODE_TS_END: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTsEnd); + break; + case PHY_WINDOW_CODE_TRIGGER_TYPE: + code = tlvDecodeI8(pTlv, &pNode->triggerType); + break; + case PHY_WINDOW_CODE_WATERMARK: + code = tlvDecodeI64(pTlv, &pNode->watermark); + break; + case PHY_WINDOW_CODE_IG_EXPIRED: + code = tlvDecodeI8(pTlv, &pNode->igExpired); + break; + case PHY_WINDOW_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + case PHY_WINDOW_CODE_OUTPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->outputTsOrder, sizeof(pNode->outputTsOrder)); + break; + case PHY_WINDOW_CODE_MERGE_DATA_BLOCK: + code = tlvDecodeBool(pTlv, &pNode->mergeDataBlock); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_INTERVAL_CODE_WINDOW = 1, + PHY_INTERVAL_CODE_INTERVAL, + PHY_INTERVAL_CODE_OFFSET, + PHY_INTERVAL_CODE_SLIDING, + PHY_INTERVAL_CODE_INTERVAL_UNIT, + PHY_INTERVAL_CODE_SLIDING_UNIT +}; + +static int32_t physiIntervalNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SIntervalPhysiNode* pNode = (const SIntervalPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_INTERVAL_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_INTERVAL, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_OFFSET, pNode->offset); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INTERVAL_CODE_SLIDING, pNode->sliding); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_INTERVAL_UNIT, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_INTERVAL_CODE_SLIDING_UNIT, pNode->slidingUnit); + } + + return code; +} + +static int32_t msgToPhysiIntervalNode(STlvDecoder* pDecoder, void* pObj) { + SIntervalPhysiNode* pNode = (SIntervalPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_INTERVAL_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_INTERVAL_CODE_INTERVAL: + code = tlvDecodeI64(pTlv, &pNode->interval); + break; + case PHY_INTERVAL_CODE_OFFSET: + code = tlvDecodeI64(pTlv, &pNode->offset); + break; + case PHY_INTERVAL_CODE_SLIDING: + code = tlvDecodeI64(pTlv, &pNode->sliding); + break; + case PHY_INTERVAL_CODE_INTERVAL_UNIT: + code = tlvDecodeI8(pTlv, &pNode->intervalUnit); + break; + case PHY_INTERVAL_CODE_SLIDING_UNIT: + code = tlvDecodeI8(pTlv, &pNode->slidingUnit); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_FILL_CODE_BASE_NODE = 1, + PHY_FILL_CODE_MODE, + PHY_FILL_CODE_FILL_EXPRS, + PHY_FILL_CODE_NOT_FILL_EXPRS, + PHY_FILL_CODE_WSTART, + PHY_FILL_CODE_VALUES, + PHY_FILL_CODE_TIME_RANGE, + PHY_FILL_CODE_INPUT_TS_ORDER +}; + +static int32_t physiFillNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SFillPhysiNode* pNode = (const SFillPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_FILL_CODE_MODE, pNode->mode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_FILL_EXPRS, nodeListToMsg, pNode->pFillExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_NOT_FILL_EXPRS, nodeListToMsg, pNode->pNotFillExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_WSTART, nodeToMsg, pNode->pWStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_VALUES, nodeToMsg, pNode->pValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_FILL_CODE_TIME_RANGE, timeWindowToMsg, &pNode->timeRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_FILL_CODE_INPUT_TS_ORDER, pNode->inputTsOrder); + } + + return code; +} + +static int32_t msgToPhysiFillNode(STlvDecoder* pDecoder, void* pObj) { + SFillPhysiNode* pNode = (SFillPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_FILL_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_FILL_CODE_MODE: + code = tlvDecodeEnum(pTlv, &pNode->mode, sizeof(pNode->mode)); + break; + case PHY_FILL_CODE_FILL_EXPRS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFillExprs); + break; + case PHY_FILL_CODE_NOT_FILL_EXPRS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pNotFillExprs); + break; + case PHY_FILL_CODE_WSTART: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pWStartTs); + break; + case PHY_FILL_CODE_VALUES: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pValues); + break; + case PHY_FILL_CODE_TIME_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, (void**)&pNode->timeRange); + break; + case PHY_FILL_CODE_INPUT_TS_ORDER: + code = tlvDecodeEnum(pTlv, &pNode->inputTsOrder, sizeof(pNode->inputTsOrder)); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_SESSION_CODE_WINDOW = 1, PHY_SESSION_CODE_GAP }; + +static int32_t physiSessionWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSessionWinodwPhysiNode* pNode = (const SSessionWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_SESSION_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_SESSION_CODE_GAP, pNode->gap); + } + + return code; +} + +static int32_t msgToPhysiSessionWindowNode(STlvDecoder* pDecoder, void* pObj) { + SSessionWinodwPhysiNode* pNode = (SSessionWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_SESSION_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_SESSION_CODE_GAP: + code = tlvDecodeI64(pTlv, &pNode->gap); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_STATE_CODE_WINDOW = 1, PHY_STATE_CODE_KEY }; + +static int32_t physiStateWindowNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SStateWinodwPhysiNode* pNode = (const SStateWinodwPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_STATE_CODE_WINDOW, physiWindowNodeToMsg, &pNode->window); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_STATE_CODE_KEY, nodeToMsg, pNode->pStateKey); + } + + return code; +} + +static int32_t msgToPhysiStateWindowNode(STlvDecoder* pDecoder, void* pObj) { + SStateWinodwPhysiNode* pNode = (SStateWinodwPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_STATE_CODE_WINDOW: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiWindowNode, &pNode->window); + break; + case PHY_STATE_CODE_KEY: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStateKey); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_PARTITION_CODE_BASE_NODE = 1, PHY_PARTITION_CODE_EXPR, PHY_PARTITION_CODE_KEYS, PHY_PARTITION_CODE_TARGETS }; + +static int32_t physiPartitionNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SPartitionPhysiNode* pNode = (const SPartitionPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_KEYS, nodeListToMsg, pNode->pPartitionKeys); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_PARTITION_CODE_TARGETS, nodeListToMsg, pNode->pTargets); + } + + return code; +} + +static int32_t msgToPhysiPartitionNode(STlvDecoder* pDecoder, void* pObj) { + SPartitionPhysiNode* pNode = (SPartitionPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_PARTITION_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_PARTITION_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_PARTITION_CODE_KEYS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pPartitionKeys); + break; + case PHY_PARTITION_CODE_TARGETS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pTargets); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE = 1, PHY_INDEF_ROWS_FUNC_CODE_EXPRS, PHY_INDEF_ROWS_FUNC_CODE_FUNCS }; + +static int32_t physiIndefRowsFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SIndefRowsFuncPhysiNode* pNode = (const SIndefRowsFuncPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INDEF_ROWS_FUNC_CODE_EXPRS, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INDEF_ROWS_FUNC_CODE_FUNCS, nodeListToMsg, pNode->pFuncs); + } + + return code; +} + +static int32_t msgToPhysiIndefRowsFuncNode(STlvDecoder* pDecoder, void* pObj) { + SIndefRowsFuncPhysiNode* pNode = (SIndefRowsFuncPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_INDEF_ROWS_FUNC_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_INDEF_ROWS_FUNC_CODE_EXPRS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_INDEF_ROWS_FUNC_CODE_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_INERP_FUNC_CODE_BASE_NODE = 1, + PHY_INERP_FUNC_CODE_EXPR, + PHY_INERP_FUNC_CODE_FUNCS, + PHY_INERP_FUNC_CODE_TIME_RANGE, + PHY_INERP_FUNC_CODE_INTERVAL, + PHY_INERP_FUNC_CODE_INTERVAL_UNIT, + PHY_INERP_FUNC_CODE_FILL_MODE, + PHY_INERP_FUNC_CODE_FILL_VALUES, + PHY_INERP_FUNC_CODE_TIME_SERIES +}; + +static int32_t physiInterpFuncNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SInterpFuncPhysiNode* pNode = (const SInterpFuncPhysiNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_BASE_NODE, physiNodeToMsg, &pNode->node); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_EXPR, nodeListToMsg, pNode->pExprs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_FUNCS, nodeListToMsg, pNode->pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_RANGE, timeWindowToMsg, &pNode->timeRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI64(pEncoder, PHY_INERP_FUNC_CODE_INTERVAL, pNode->interval); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_INERP_FUNC_CODE_INTERVAL_UNIT, pNode->intervalUnit); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, PHY_INERP_FUNC_CODE_FILL_MODE, pNode->fillMode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_FILL_VALUES, nodeToMsg, pNode->pFillValues); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_INERP_FUNC_CODE_TIME_SERIES, nodeToMsg, pNode->pTimeSeries); + } + + return code; +} + +static int32_t msgToPhysiInterpFuncNode(STlvDecoder* pDecoder, void* pObj) { + SInterpFuncPhysiNode* pNode = (SInterpFuncPhysiNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_INERP_FUNC_CODE_BASE_NODE: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysiNode, &pNode->node); + break; + case PHY_INERP_FUNC_CODE_EXPR: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pExprs); + break; + case PHY_INERP_FUNC_CODE_FUNCS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pFuncs); + break; + case PHY_INERP_FUNC_CODE_TIME_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->timeRange); + break; + case PHY_INERP_FUNC_CODE_INTERVAL: + code = tlvDecodeI64(pTlv, &pNode->interval); + break; + case PHY_INERP_FUNC_CODE_INTERVAL_UNIT: + code = tlvDecodeI8(pTlv, &pNode->intervalUnit); + break; + case PHY_INERP_FUNC_CODE_FILL_MODE: + code = tlvDecodeEnum(pTlv, &pNode->fillMode, sizeof(pNode->fillMode)); + break; + case PHY_INERP_FUNC_CODE_FILL_VALUES: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pFillValues); + break; + case PHY_INERP_FUNC_CODE_TIME_SERIES: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTimeSeries); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_DATA_SINK_CODE_INPUT_DESC = 1 }; + +static int32_t physicDataSinkNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataSinkNode* pNode = (const SDataSinkNode*)pObj; + return tlvEncodeObj(pEncoder, PHY_DATA_SINK_CODE_INPUT_DESC, nodeToMsg, pNode->pInputDataBlockDesc); +} + +static int32_t msgToPhysicDataSinkNode(STlvDecoder* pDecoder, void* pObj) { + SDataSinkNode* pNode = (SDataSinkNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_DATA_SINK_CODE_INPUT_DESC: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pInputDataBlockDesc); + break; + default: + break; + } + } + + return code; +} + +enum { PHY_DISPATCH_CODE_SINK = 1 }; + +static int32_t physiDispatchNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataDispatcherNode* pNode = (const SDataDispatcherNode*)pObj; + return tlvEncodeObj(pEncoder, PHY_DISPATCH_CODE_SINK, physicDataSinkNodeToMsg, &pNode->sink); +} + +static int32_t msgToPhysiDispatchNode(STlvDecoder* pDecoder, void* pObj) { + SDataDispatcherNode* pNode = (SDataDispatcherNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_DISPATCH_CODE_SINK: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysicDataSinkNode, &pNode->sink); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_QUERY_INSERT_CODE_SINK = 1, + PHY_QUERY_INSERT_CODE_COLS, + PHY_QUERY_INSERT_CODE_TABLE_ID, + PHY_QUERY_INSERT_CODE_STABLE_ID, + PHY_QUERY_INSERT_CODE_TABLE_TYPE, + PHY_QUERY_INSERT_CODE_TABLE_NAME, + PHY_QUERY_INSERT_CODE_VG_ID, + PHY_QUERY_INSERT_CODE_EP_SET +}; + +static int32_t physiQueryInsertNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SQueryInserterNode* pNode = (const SQueryInserterNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_SINK, physicDataSinkNodeToMsg, &pNode->sink); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_COLS, nodeListToMsg, pNode->pCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_QUERY_INSERT_CODE_TABLE_ID, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_QUERY_INSERT_CODE_STABLE_ID, pNode->stableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_QUERY_INSERT_CODE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, PHY_QUERY_INSERT_CODE_TABLE_NAME, pNode->tableName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, PHY_QUERY_INSERT_CODE_VG_ID, pNode->vgId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_QUERY_INSERT_CODE_EP_SET, epSetToMsg, &pNode->epSet); + } + + return code; +} + +static int32_t msgToPhysiQueryInsertNode(STlvDecoder* pDecoder, void* pObj) { + SQueryInserterNode* pNode = (SQueryInserterNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_QUERY_INSERT_CODE_SINK: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysicDataSinkNode, &pNode->sink); + break; + case PHY_QUERY_INSERT_CODE_COLS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pCols); + break; + case PHY_QUERY_INSERT_CODE_TABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->tableId); + break; + case PHY_QUERY_INSERT_CODE_STABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->stableId); + break; + case PHY_QUERY_INSERT_CODE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case PHY_QUERY_INSERT_CODE_TABLE_NAME: + code = tlvDecodeCStr(pTlv, pNode->tableName); + break; + case PHY_QUERY_INSERT_CODE_VG_ID: + code = tlvDecodeI32(pTlv, &pNode->vgId); + break; + case PHY_QUERY_INSERT_CODE_EP_SET: + code = tlvDecodeObjFromTlv(pTlv, msgToEpSet, &pNode->epSet); + break; + default: + break; + } + } + + return code; +} + +enum { + PHY_DELETER_CODE_SINK = 1, + PHY_DELETER_CODE_TABLE_ID, + PHY_DELETER_CODE_TABLE_TYPE, + PHY_DELETER_CODE_TABLE_FNAME, + PHY_DELETER_CODE_TS_COL_NAME, + PHY_DELETER_CODE_DELETE_TIME_RANGE, + PHY_DELETER_CODE_AFFECTED_ROWS, + PHY_DELETER_CODE_START_TS, + PHY_DELETER_CODE_END_TS +}; + +static int32_t physiDeleteNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SDataDeleterNode* pNode = (const SDataDeleterNode*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_SINK, physicDataSinkNodeToMsg, &pNode->sink); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeU64(pEncoder, PHY_DELETER_CODE_TABLE_ID, pNode->tableId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI8(pEncoder, PHY_DELETER_CODE_TABLE_TYPE, pNode->tableType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, PHY_DELETER_CODE_TABLE_FNAME, pNode->tableFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, PHY_DELETER_CODE_TS_COL_NAME, pNode->tsColName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_DELETE_TIME_RANGE, timeWindowToMsg, &pNode->deleteTimeRange); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_AFFECTED_ROWS, nodeToMsg, pNode->pAffectedRows); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_START_TS, nodeToMsg, pNode->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, PHY_DELETER_CODE_END_TS, nodeToMsg, pNode->pEndTs); + } + + return code; +} + +static int32_t msgToPhysiDeleteNode(STlvDecoder* pDecoder, void* pObj) { + SDataDeleterNode* pNode = (SDataDeleterNode*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case PHY_DELETER_CODE_SINK: + code = tlvDecodeObjFromTlv(pTlv, msgToPhysicDataSinkNode, &pNode->sink); + break; + case PHY_DELETER_CODE_TABLE_ID: + code = tlvDecodeU64(pTlv, &pNode->tableId); + break; + case PHY_DELETER_CODE_TABLE_TYPE: + code = tlvDecodeI8(pTlv, &pNode->tableType); + break; + case PHY_DELETER_CODE_TABLE_FNAME: + code = tlvDecodeCStr(pTlv, pNode->tableFName); + break; + case PHY_DELETER_CODE_TS_COL_NAME: + code = tlvDecodeCStr(pTlv, pNode->tsColName); + break; + case PHY_DELETER_CODE_DELETE_TIME_RANGE: + code = tlvDecodeObjFromTlv(pTlv, msgToTimeWindow, &pNode->deleteTimeRange); + break; + case PHY_DELETER_CODE_AFFECTED_ROWS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pAffectedRows); + break; + case PHY_DELETER_CODE_START_TS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pStartTs); + break; + case PHY_DELETER_CODE_END_TS: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pEndTs); + break; + default: + break; + } + } + + return code; +} + +enum { SUBPLAN_ID_CODE_QUERY_ID = 1, SUBPLAN_ID_CODE_GROUP_ID, SUBPLAN_ID_CODE_SUBPLAN_ID }; + +static int32_t subplanIdToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSubplanId* pNode = (const SSubplanId*)pObj; + + int32_t code = tlvEncodeU64(pEncoder, SUBPLAN_ID_CODE_QUERY_ID, pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_ID_CODE_GROUP_ID, pNode->groupId); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_ID_CODE_SUBPLAN_ID, pNode->subplanId); + } + + return code; +} + +static int32_t msgToSubplanId(STlvDecoder* pDecoder, void* pObj) { + SSubplanId* pNode = (SSubplanId*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case SUBPLAN_ID_CODE_QUERY_ID: + code = tlvDecodeU64(pTlv, &pNode->queryId); + break; + case SUBPLAN_ID_CODE_GROUP_ID: + code = tlvDecodeI32(pTlv, &pNode->groupId); + break; + case SUBPLAN_ID_CODE_SUBPLAN_ID: + code = tlvDecodeI32(pTlv, &pNode->subplanId); + break; + default: + break; + } + } + + return code; +} + +enum { + SUBPLAN_CODE_SUBPLAN_ID = 1, + SUBPLAN_CODE_SUBPLAN_TYPE, + SUBPLAN_CODE_MSG_TYPE, + SUBPLAN_CODE_LEVEL, + SUBPLAN_CODE_DBFNAME, + SUBPLAN_CODE_USER, + SUBPLAN_CODE_EXECNODE, + SUBPLAN_CODE_ROOT_NODE, + SUBPLAN_CODE_DATA_SINK, + SUBPLAN_CODE_TAG_COND, + SUBPLAN_CODE_TAG_INDEX_COND +}; + +static int32_t subplanToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SSubplan* pNode = (const SSubplan*)pObj; + + int32_t code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_SUBPLAN_ID, subplanIdToMsg, &pNode->id); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeEnum(pEncoder, SUBPLAN_CODE_SUBPLAN_TYPE, pNode->subplanType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_CODE_MSG_TYPE, pNode->msgType); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, SUBPLAN_CODE_LEVEL, pNode->level); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, SUBPLAN_CODE_DBFNAME, pNode->dbFName); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeCStr(pEncoder, SUBPLAN_CODE_USER, pNode->user); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_EXECNODE, queryNodeAddrToMsg, &pNode->execNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_ROOT_NODE, nodeToMsg, pNode->pNode); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_DATA_SINK, nodeToMsg, pNode->pDataSink); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_TAG_COND, nodeToMsg, pNode->pTagCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, SUBPLAN_CODE_TAG_INDEX_COND, nodeToMsg, pNode->pTagIndexCond); + } + + return code; +} + +static int32_t msgToSubplan(STlvDecoder* pDecoder, void* pObj) { + SSubplan* pNode = (SSubplan*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case SUBPLAN_CODE_SUBPLAN_ID: + code = tlvDecodeObjFromTlv(pTlv, msgToSubplanId, &pNode->id); + break; + case SUBPLAN_CODE_SUBPLAN_TYPE: + code = tlvDecodeEnum(pTlv, &pNode->subplanType, sizeof(pNode->subplanType)); + break; + case SUBPLAN_CODE_MSG_TYPE: + code = tlvDecodeI32(pTlv, &pNode->msgType); + break; + case SUBPLAN_CODE_LEVEL: + code = tlvDecodeI32(pTlv, &pNode->level); + break; + case SUBPLAN_CODE_DBFNAME: + code = tlvDecodeCStr(pTlv, pNode->dbFName); + break; + case SUBPLAN_CODE_USER: + code = tlvDecodeCStr(pTlv, pNode->user); + break; + case SUBPLAN_CODE_EXECNODE: + code = tlvDecodeObjFromTlv(pTlv, msgToQueryNodeAddr, &pNode->execNode); + break; + case SUBPLAN_CODE_ROOT_NODE: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pNode); + break; + case SUBPLAN_CODE_DATA_SINK: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pDataSink); + break; + case SUBPLAN_CODE_TAG_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagCond); + break; + case SUBPLAN_CODE_TAG_INDEX_COND: + code = msgToNodeFromTlv(pTlv, (void**)&pNode->pTagIndexCond); + break; + default: + break; + } + } + + return code; +} + +enum { QUERY_PLAN_CODE_QUERY_ID = 1, QUERY_PLAN_CODE_NUM_OF_SUBPLANS, QUERY_PLAN_CODE_SUBPLANS }; + +static int32_t queryPlanToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SQueryPlan* pNode = (const SQueryPlan*)pObj; + + int32_t code = tlvEncodeU64(pEncoder, QUERY_PLAN_CODE_QUERY_ID, pNode->queryId); + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeI32(pEncoder, QUERY_PLAN_CODE_NUM_OF_SUBPLANS, pNode->numOfSubplans); + } + if (TSDB_CODE_SUCCESS == code) { + code = tlvEncodeObj(pEncoder, QUERY_PLAN_CODE_SUBPLANS, nodeListToMsg, pNode->pSubplans); + } + + return code; +} + +static int32_t msgToQueryPlan(STlvDecoder* pDecoder, void* pObj) { + SQueryPlan* pNode = (SQueryPlan*)pObj; + + int32_t code = TSDB_CODE_SUCCESS; + STlv* pTlv = NULL; + tlvForEach(pDecoder, pTlv, code) { + switch (pTlv->type) { + case QUERY_PLAN_CODE_QUERY_ID: + code = tlvDecodeU64(pTlv, &pNode->queryId); + break; + case QUERY_PLAN_CODE_NUM_OF_SUBPLANS: + code = tlvDecodeI32(pTlv, &pNode->numOfSubplans); + break; + case QUERY_PLAN_CODE_SUBPLANS: + code = msgToNodeListFromTlv(pTlv, (void**)&pNode->pSubplans); + break; + default: + break; + } + } + + return code; +} + +static int32_t specificNodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pObj)) { + case QUERY_NODE_COLUMN: + code = columnNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_VALUE: + code = valueNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_OPERATOR: + code = operatorNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_LOGIC_CONDITION: + code = logicConditionNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_FUNCTION: + code = functionNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_ORDER_BY_EXPR: + code = orderByExprNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_LIMIT: + code = limitNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_NODE_LIST: + code = nodeListNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_TARGET: + code = targetNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_DATABLOCK_DESC: + code = dataBlockDescNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_SLOT_DESC: + code = slotDescNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_DOWNSTREAM_SOURCE: + return downstreamSourceNodeToMsg(pObj, pEncoder); + case QUERY_NODE_LEFT_VALUE: + break; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + code = physiScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + code = physiLastRowScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + code = physiTableScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + code = physiSysTableScanNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + code = physiProjectNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: + code = physiJoinNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: + code = physiAggNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + code = physiExchangeNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + code = physiMergeNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + code = physiSortNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + code = physiIntervalNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + code = physiFillNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + code = physiSessionWindowNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: + code = physiStateWindowNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + code = physiPartitionNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + code = physiIndefRowsFuncNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: + code = physiInterpFuncNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + code = physiDispatchNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + code = physiQueryInsertNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + code = physiDeleteNodeToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_SUBPLAN: + code = subplanToMsg(pObj, pEncoder); + break; + case QUERY_NODE_PHYSICAL_PLAN: + code = queryPlanToMsg(pObj, pEncoder); + break; + default: + nodesWarn("specificNodeToMsg unknown node = %s", nodesNodeName(nodeType(pObj))); + break; + } + if (TSDB_CODE_SUCCESS != code) { + nodesError("specificNodeToMsg error node = %s", nodesNodeName(nodeType(pObj))); + } + return code; +} + +static int32_t msgToSpecificNode(STlvDecoder* pDecoder, void* pObj) { + int32_t code = TSDB_CODE_SUCCESS; + switch (nodeType(pObj)) { + case QUERY_NODE_COLUMN: + code = msgToColumnNode(pDecoder, pObj); + break; + case QUERY_NODE_VALUE: + code = msgToValueNode(pDecoder, pObj); + break; + case QUERY_NODE_OPERATOR: + code = msgToOperatorNode(pDecoder, pObj); + break; + case QUERY_NODE_LOGIC_CONDITION: + code = msgToLogicConditionNode(pDecoder, pObj); + break; + case QUERY_NODE_FUNCTION: + code = msgToFunctionNode(pDecoder, pObj); + break; + case QUERY_NODE_ORDER_BY_EXPR: + code = msgToOrderByExprNode(pDecoder, pObj); + break; + case QUERY_NODE_LIMIT: + code = msgToLimitNode(pDecoder, pObj); + break; + case QUERY_NODE_NODE_LIST: + code = msgToNodeListNode(pDecoder, pObj); + break; + case QUERY_NODE_TARGET: + code = msgToTargetNode(pDecoder, pObj); + break; + case QUERY_NODE_DATABLOCK_DESC: + code = msgToDataBlockDescNode(pDecoder, pObj); + break; + case QUERY_NODE_SLOT_DESC: + code = msgToSlotDescNode(pDecoder, pObj); + break; + case QUERY_NODE_DOWNSTREAM_SOURCE: + return msgToDownstreamSourceNode(pDecoder, pObj); + case QUERY_NODE_LEFT_VALUE: + break; + case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + code = msgToPhysiScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + code = msgToPhysiLastRowScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: + code = msgToPhysiTableScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: + code = msgToPhysiSysTableScanNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_PROJECT: + code = msgToPhysiProjectNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: + code = msgToPhysiJoinNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_AGG: + code = msgToPhysiAggNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_EXCHANGE: + code = msgToPhysiExchangeNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE: + code = msgToPhysiMergeNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_SORT: + case QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT: + code = msgToPhysiSortNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: + code = msgToPhysiIntervalNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_FILL: + code = msgToPhysiFillNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: + code = msgToPhysiSessionWindowNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_STATE: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE: + code = msgToPhysiStateWindowNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + code = msgToPhysiPartitionNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: + code = msgToPhysiIndefRowsFuncNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: + code = msgToPhysiInterpFuncNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_DISPATCH: + code = msgToPhysiDispatchNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: + code = msgToPhysiQueryInsertNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN_DELETE: + code = msgToPhysiDeleteNode(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_SUBPLAN: + code = msgToSubplan(pDecoder, pObj); + break; + case QUERY_NODE_PHYSICAL_PLAN: + code = msgToQueryPlan(pDecoder, pObj); + break; + default: + nodesWarn("msgToSpecificNode unknown node = %s", nodesNodeName(nodeType(pObj))); + break; + } + if (TSDB_CODE_SUCCESS != code) { + nodesError("msgToSpecificNode error node = %s", nodesNodeName(nodeType(pObj))); + } + return code; +} + +static int32_t nodeToMsg(const void* pObj, STlvEncoder* pEncoder) { + return tlvEncodeObj(pEncoder, nodeType(pObj), specificNodeToMsg, pObj); +} + +static int32_t msgToNode(STlvDecoder* pDecoder, void** pObj) { + return tlvDecodeDynObj(pDecoder, (FMakeObject)nodesMakeNode, msgToSpecificNode, pObj); +} + +static int32_t msgToNodeFromTlv(STlv* pTlv, void** pObj) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return msgToNode(&decoder, pObj); +} + +static int32_t nodeListToMsg(const void* pObj, STlvEncoder* pEncoder) { + const SNodeList* pList = (const SNodeList*)pObj; + + SNode* pNode = NULL; + FOREACH(pNode, pList) { + int32_t code = nodeToMsg(pNode, pEncoder); + if (TSDB_CODE_SUCCESS != code) { + return code; + } + } + + return TSDB_CODE_SUCCESS; +} + +static int32_t msgToNodeList(STlvDecoder* pDecoder, void** pObj) { + SNodeList* pList = nodesMakeList(); + + int32_t code = TSDB_CODE_SUCCESS; + while (TSDB_CODE_SUCCESS == code && !tlvDecodeEnd(pDecoder)) { + SNode* pNode = NULL; + code = msgToNode(pDecoder, (void**)&pNode); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListAppend(pList, pNode); + } + } + if (TSDB_CODE_SUCCESS == code) { + *pObj = pList; + } else { + nodesDestroyList(pList); + } + return code; +} + +static int32_t msgToNodeListFromTlv(STlv* pTlv, void** pObj) { + STlvDecoder decoder = {.bufSize = pTlv->len, .offset = 0, .pBuf = pTlv->value}; + return msgToNodeList(&decoder, pObj); +} + +int32_t nodesNodeToMsg(const SNode* pNode, char** pMsg, int32_t* pLen) { + if (NULL == pNode || NULL == pMsg || NULL == pLen) { + terrno = TSDB_CODE_FAILED; + return TSDB_CODE_FAILED; + } + + STlvEncoder encoder; + int32_t code = initTlvEncoder(&encoder); + if (TSDB_CODE_SUCCESS == code) { + code = nodeToMsg(pNode, &encoder); + } + if (TSDB_CODE_SUCCESS == code) { + endTlvEncode(&encoder, pMsg, pLen); + } + clearTlvEncoder(&encoder); + + terrno = code; + return code; +} + +int32_t nodesMsgToNode(const char* pMsg, int32_t len, SNode** pNode) { + if (NULL == pMsg || NULL == pNode) { + return TSDB_CODE_SUCCESS; + } + + STlvDecoder decoder = {.bufSize = len, .offset = 0, .pBuf = pMsg}; + int32_t code = msgToNode(&decoder, (void**)pNode); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(*pNode); + *pNode = NULL; + } + + terrno = code; + return code; +} diff --git a/source/libs/nodes/src/nodesTraverseFuncs.c b/source/libs/nodes/src/nodesTraverseFuncs.c index 2e23998aad9463fb7a4a9b6834ceab2f7ea51e55..728e173ff85e87d553d118f0baf0022a99c58f5d 100644 --- a/source/libs/nodes/src/nodesTraverseFuncs.c +++ b/source/libs/nodes/src/nodesTraverseFuncs.c @@ -537,7 +537,8 @@ static EDealRes dispatchPhysiPlan(SNode* pNode, ETraversalOrder order, FNodeWalk } break; } - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: { SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)pNode; res = walkPhysiNode((SPhysiNode*)pNode, order, walker, pContext); if (DEAL_RES_ERROR != res && DEAL_RES_END != res) { diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index d13057a93e824c2b94d94a006664b4cbc4c2f870..805ddb9e422db5a42c6aed1594985d5a233c4ccd 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -322,6 +322,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SStreamStateWinodwPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PARTITION: return makeNode(type, sizeof(SPartitionPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: + return makeNode(type, sizeof(SStreamPartitionPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC: return makeNode(type, sizeof(SIndefRowsFuncPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC: @@ -725,6 +727,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pWhere); nodesDestroyNode(pStmt->pCountFunc); + nodesDestroyNode(pStmt->pFirstFunc); + nodesDestroyNode(pStmt->pLastFunc); nodesDestroyNode(pStmt->pTagCond); break; } @@ -789,6 +793,8 @@ void nodesDestroyNode(SNode* pNode) { destroyVgDataBlockArray(pLogicNode->pDataBlocks); // pVgDataBlocks is weak reference nodesDestroyNode(pLogicNode->pAffectedRows); + nodesDestroyNode(pLogicNode->pStartTs); + nodesDestroyNode(pLogicNode->pEndTs); taosMemoryFreeClear(pLogicNode->pVgroupList); nodesDestroyList(pLogicNode->pInsertCols); break; @@ -951,7 +957,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pPhyNode->pStateKey); break; } - case QUERY_NODE_PHYSICAL_PLAN_PARTITION: { + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + case QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION: { SPartitionPhysiNode* pPhyNode = (SPartitionPhysiNode*)pNode; destroyPhysiNode((SPhysiNode*)pPhyNode); nodesDestroyList(pPhyNode->pExprs); @@ -994,6 +1001,8 @@ void nodesDestroyNode(SNode* pNode) { SDataDeleterNode* pSink = (SDataDeleterNode*)pNode; destroyDataSinkNode((SDataSinkNode*)pSink); nodesDestroyNode(pSink->pAffectedRows); + nodesDestroyNode(pSink->pStartTs); + nodesDestroyNode(pSink->pEndTs); break; } case QUERY_NODE_PHYSICAL_SUBPLAN: { diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 2a4f4c194d3e46c9ff4d052187431287dce87b3e..898e4bf7328f873688f1ed6c8c31690de28f74c5 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -48,6 +48,7 @@ typedef enum EDatabaseOptionType { DB_OPTION_KEEP, DB_OPTION_PAGES, DB_OPTION_PAGESIZE, + DB_OPTION_TSDB_PAGESIZE, DB_OPTION_PRECISION, DB_OPTION_REPLICA, DB_OPTION_STRICT, @@ -59,7 +60,10 @@ typedef enum EDatabaseOptionType { DB_OPTION_WAL_RETENTION_PERIOD, DB_OPTION_WAL_RETENTION_SIZE, DB_OPTION_WAL_ROLL_PERIOD, - DB_OPTION_WAL_SEGMENT_SIZE + DB_OPTION_WAL_SEGMENT_SIZE, + DB_OPTION_STT_TRIGGER, + DB_OPTION_TABLE_PREFIX, + DB_OPTION_TABLE_SUFFIX } EDatabaseOptionType; typedef enum ETableOptionType { @@ -170,6 +174,7 @@ SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable); SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId); +SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint); SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo); SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal); SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 9bff061d02fbfa8d5795dff82c9ec93b7093f96d..bef2ed98ec92e07431d765ac8275ab9e6c111a2f 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -184,6 +184,7 @@ db_options(A) ::= db_options(B) KEEP integer_list(C). db_options(A) ::= db_options(B) KEEP variable_list(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_KEEP, C); } db_options(A) ::= db_options(B) PAGES NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PAGES, &C); } db_options(A) ::= db_options(B) PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PAGESIZE, &C); } +db_options(A) ::= db_options(B) TSDB_PAGESIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TSDB_PAGESIZE, &C); } db_options(A) ::= db_options(B) PRECISION NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_PRECISION, &C); } db_options(A) ::= db_options(B) REPLICA NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_REPLICA, &C); } db_options(A) ::= db_options(B) STRICT NK_STRING(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STRICT, &C); } @@ -207,6 +208,9 @@ db_options(A) ::= db_options(B) WAL_RETENTION_SIZE NK_MINUS(D) NK_INTEGER(C). } db_options(A) ::= db_options(B) WAL_ROLL_PERIOD NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_ROLL_PERIOD, &C); } db_options(A) ::= db_options(B) WAL_SEGMENT_SIZE NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_WAL_SEGMENT_SIZE, &C); } +db_options(A) ::= db_options(B) STT_TRIGGER NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_STT_TRIGGER, &C); } +db_options(A) ::= db_options(B) TABLE_PREFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_PREFIX, &C); } +db_options(A) ::= db_options(B) TABLE_SUFFIX NK_INTEGER(C). { A = setDatabaseOption(pCxt, B, DB_OPTION_TABLE_SUFFIX, &C); } alter_db_options(A) ::= alter_db_option(B). { A = createAlterDatabaseOptions(pCxt); A = setAlterDatabaseOption(pCxt, A, &B); } alter_db_options(A) ::= alter_db_options(B) alter_db_option(C). { A = setAlterDatabaseOption(pCxt, B, &C); } @@ -223,6 +227,7 @@ alter_db_option(A) ::= KEEP variable_list(B). //alter_db_option(A) ::= REPLICA NK_INTEGER(B). { A.type = DB_OPTION_REPLICA; A.val = B; } //alter_db_option(A) ::= STRICT NK_STRING(B). { A.type = DB_OPTION_STRICT; A.val = B; } alter_db_option(A) ::= WAL_LEVEL NK_INTEGER(B). { A.type = DB_OPTION_WAL; A.val = B; } +alter_db_option(A) ::= STT_TRIGGER NK_INTEGER(B). { A.type = DB_OPTION_STT_TRIGGER; A.val = B; } %type integer_list { SNodeList* } %destructor integer_list { nodesDestroyList($$); } @@ -410,6 +415,8 @@ cmd ::= SHOW TABLE DISTRIBUTED full_table_name(A). cmd ::= SHOW CONSUMERS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } cmd ::= SHOW SUBSCRIPTIONS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } cmd ::= SHOW TAGS FROM table_name_cond(A) from_db_opt(B). { pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, B, A, OP_TYPE_EQUAL); } +cmd ::= SHOW VNODES NK_INTEGER(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A), NULL); } +cmd ::= SHOW VNODES NK_STRING(A). { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &A)); } db_name_cond_opt(A) ::= . { A = createDefaultDatabaseCondValue(pCxt); } db_name_cond_opt(A) ::= db_name(B) NK_DOT. { A = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &B); } @@ -997,4 +1004,4 @@ null_ordering_opt(A) ::= NULLS LAST. %fallback ABORT AFTER ATTACH BEFORE BEGIN BITAND BITNOT BITOR BLOCKS CHANGE COMMA COMPACT CONCAT CONFLICT COPY DEFERRED DELIMITERS DETACH DIVIDE DOT EACH END FAIL FILE FOR GLOB ID IMMEDIATE IMPORT INITIALLY INSTEAD ISNULL KEY NK_BITNOT NK_SEMI NOTNULL OF PLUS PRIVILEGE RAISE REPLACE RESTRICT ROW SEMI STAR STATEMENT STRING - TIMES UPDATE VALUES VARIABLE VIEW VNODES WAL. + TIMES UPDATE VALUES VARIABLE VIEW WAL. diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 9eae29f5ec54c69d0f15100713c2686e67ebacfc..6f11c653a4a853744922c9ad17464aad71d77142 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -795,6 +795,20 @@ SNode* createSetOperator(SAstCreateContext* pCxt, ESetOperatorType type, SNode* return (SNode*)setOp; } +static void updateWalOptionsDefault(SDatabaseOptions* pOptions) { + if (!pOptions->walRetentionPeriodIsSet) { + pOptions->walRetentionPeriod = + pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_RET_PERIOD : TSDB_REP_DEF_DB_WAL_RET_PERIOD; + } + if (!pOptions->walRetentionSizeIsSet) { + pOptions->walRetentionSize = pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_RET_SIZE : TSDB_REP_DEF_DB_WAL_RET_SIZE; + } + if (!pOptions->walRollPeriodIsSet) { + pOptions->walRollPeriod = + pOptions->replica > 1 ? TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD : TSDB_REP_DEF_DB_WAL_ROLL_PERIOD; + } +} + SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { CHECK_PARSER_STATUS(pCxt); SDatabaseOptions* pOptions = (SDatabaseOptions*)nodesMakeNode(QUERY_NODE_DATABASE_OPTIONS); @@ -812,6 +826,7 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->keep[2] = TSDB_DEFAULT_KEEP; pOptions->pages = TSDB_DEFAULT_PAGES_PER_VNODE; pOptions->pagesize = TSDB_DEFAULT_PAGESIZE_PER_VNODE; + pOptions->tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; pOptions->precision = TSDB_DEFAULT_PRECISION; pOptions->replica = TSDB_DEFAULT_DB_REPLICA; pOptions->strict = TSDB_DEFAULT_DB_STRICT; @@ -819,10 +834,11 @@ SNode* createDefaultDatabaseOptions(SAstCreateContext* pCxt) { pOptions->numOfVgroups = TSDB_DEFAULT_VN_PER_DB; pOptions->singleStable = TSDB_DEFAULT_DB_SINGLE_STABLE; pOptions->schemaless = TSDB_DEFAULT_DB_SCHEMALESS; - pOptions->walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD; - pOptions->walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE; - pOptions->walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD; + updateWalOptionsDefault(pOptions); pOptions->walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; + pOptions->sstTrigger = TSDB_DEFAULT_SST_TRIGGER; + pOptions->tablePrefix = TSDB_DEFAULT_HASH_PREFIX; + pOptions->tableSuffix = TSDB_DEFAULT_HASH_SUFFIX; return (SNode*)pOptions; } @@ -843,6 +859,7 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->keep[2] = -1; pOptions->pages = -1; pOptions->pagesize = -1; + pOptions->tsdbPageSize = -1; pOptions->precision = -1; pOptions->replica = -1; pOptions->strict = -1; @@ -854,83 +871,103 @@ SNode* createAlterDatabaseOptions(SAstCreateContext* pCxt) { pOptions->walRetentionSize = -1; pOptions->walRollPeriod = -1; pOptions->walSegmentSize = -1; + pOptions->sstTrigger = -1; + pOptions->tablePrefix = -1; + pOptions->tableSuffix = -1; return (SNode*)pOptions; } SNode* setDatabaseOption(SAstCreateContext* pCxt, SNode* pOptions, EDatabaseOptionType type, void* pVal) { CHECK_PARSER_STATUS(pCxt); + SDatabaseOptions* pDbOptions = (SDatabaseOptions*)pOptions; switch (type) { case DB_OPTION_BUFFER: - ((SDatabaseOptions*)pOptions)->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->buffer = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_CACHEMODEL: - COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->cacheModelStr, (SToken*)pVal); + COPY_STRING_FORM_STR_TOKEN(pDbOptions->cacheModelStr, (SToken*)pVal); break; case DB_OPTION_CACHESIZE: - ((SDatabaseOptions*)pOptions)->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->cacheLastSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_COMP: - ((SDatabaseOptions*)pOptions)->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + pDbOptions->compressionLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_DAYS: { SToken* pToken = pVal; if (TK_NK_INTEGER == pToken->type) { - ((SDatabaseOptions*)pOptions)->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440; + pDbOptions->daysPerFile = taosStr2Int32(pToken->z, NULL, 10) * 1440; } else { - ((SDatabaseOptions*)pOptions)->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); + pDbOptions->pDaysPerFile = (SValueNode*)createDurationValueNode(pCxt, pToken); } break; } case DB_OPTION_FSYNC: - ((SDatabaseOptions*)pOptions)->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->fsyncPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MAXROWS: - ((SDatabaseOptions*)pOptions)->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->maxRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_MINROWS: - ((SDatabaseOptions*)pOptions)->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->minRowsPerBlock = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_KEEP: - ((SDatabaseOptions*)pOptions)->pKeep = pVal; + pDbOptions->pKeep = pVal; break; case DB_OPTION_PAGES: - ((SDatabaseOptions*)pOptions)->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->pages = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PAGESIZE: - ((SDatabaseOptions*)pOptions)->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->pagesize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + break; + case DB_OPTION_TSDB_PAGESIZE: + pDbOptions->tsdbPageSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_PRECISION: - COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->precisionStr, (SToken*)pVal); + COPY_STRING_FORM_STR_TOKEN(pDbOptions->precisionStr, (SToken*)pVal); break; case DB_OPTION_REPLICA: - ((SDatabaseOptions*)pOptions)->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + pDbOptions->replica = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + updateWalOptionsDefault(pDbOptions); break; case DB_OPTION_STRICT: - COPY_STRING_FORM_STR_TOKEN(((SDatabaseOptions*)pOptions)->strictStr, (SToken*)pVal); + COPY_STRING_FORM_STR_TOKEN(pDbOptions->strictStr, (SToken*)pVal); break; case DB_OPTION_WAL: - ((SDatabaseOptions*)pOptions)->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walLevel = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_VGROUPS: - ((SDatabaseOptions*)pOptions)->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->numOfVgroups = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_SINGLE_STABLE: - ((SDatabaseOptions*)pOptions)->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); + pDbOptions->singleStable = taosStr2Int8(((SToken*)pVal)->z, NULL, 10); break; case DB_OPTION_RETENTIONS: - ((SDatabaseOptions*)pOptions)->pRetentions = pVal; + pDbOptions->pRetentions = pVal; break; case DB_OPTION_WAL_RETENTION_PERIOD: - ((SDatabaseOptions*)pOptions)->walRetentionPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walRetentionPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walRetentionPeriodIsSet = true; break; case DB_OPTION_WAL_RETENTION_SIZE: - ((SDatabaseOptions*)pOptions)->walRetentionSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walRetentionSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walRetentionSizeIsSet = true; break; case DB_OPTION_WAL_ROLL_PERIOD: - ((SDatabaseOptions*)pOptions)->walRollPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walRollPeriod = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walRollPeriodIsSet = true; break; case DB_OPTION_WAL_SEGMENT_SIZE: - ((SDatabaseOptions*)pOptions)->walSegmentSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + pDbOptions->walSegmentSize = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + break; + case DB_OPTION_STT_TRIGGER: + pDbOptions->sstTrigger = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + break; + case DB_OPTION_TABLE_PREFIX: + pDbOptions->tablePrefix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); + break; + case DB_OPTION_TABLE_SUFFIX: + pDbOptions->tableSuffix = taosStr2Int32(((SToken*)pVal)->z, NULL, 10); break; default: break; @@ -1266,7 +1303,7 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD EOperatorType tableCondType) { CHECK_PARSER_STATUS(pCxt); if (needDbShowStmt(type) && NULL == pDbName) { - snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "db not specified"); + snprintf(pCxt->pQueryCxt->pMsg, pCxt->pQueryCxt->msgLen, "database not specified"); pCxt->errCode = TSDB_CODE_PAR_SYNTAX_ERROR; return NULL; } @@ -1317,6 +1354,15 @@ SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId) { return (SNode*)pStmt; } +SNode* createShowVnodesStmt(SAstCreateContext* pCxt, SNode* pDnodeId, SNode* pDnodeEndpoint) { + CHECK_PARSER_STATUS(pCxt); + SShowVnodesStmt* pStmt = (SShowVnodesStmt*)nodesMakeNode(QUERY_NODE_SHOW_VNODES_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->pDnodeId = pDnodeId; + pStmt->pDnodeEndpoint = pDnodeEndpoint; + return (SNode*)pStmt; +} + SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo) { CHECK_PARSER_STATUS(pCxt); char password[TSDB_USET_PASSWORD_LEN] = {0}; @@ -1741,10 +1787,10 @@ SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDb return (SNode*)pStmt; } -SNode* createCountFuncForDelete(SAstCreateContext* pCxt) { +SNode* createFuncForDelete(SAstCreateContext* pCxt, const char* pFuncName) { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); CHECK_OUT_OF_MEM(pFunc); - strcpy(pFunc->functionName, "count"); + strcpy(pFunc->functionName, pFuncName); if (TSDB_CODE_SUCCESS != nodesListMakeStrictAppend(&pFunc->pParameterList, createPrimaryKeyCol(pCxt, NULL))) { nodesDestroyNode((SNode*)pFunc); CHECK_OUT_OF_MEM(NULL); @@ -1758,8 +1804,10 @@ SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere) { CHECK_OUT_OF_MEM(pStmt); pStmt->pFromTable = pTable; pStmt->pWhere = pWhere; - pStmt->pCountFunc = createCountFuncForDelete(pCxt); - if (NULL == pStmt->pCountFunc) { + pStmt->pCountFunc = createFuncForDelete(pCxt, "count"); + pStmt->pFirstFunc = createFuncForDelete(pCxt, "first"); + pStmt->pLastFunc = createFuncForDelete(pCxt, "last"); + if (NULL == pStmt->pCountFunc || NULL == pStmt->pFirstFunc || NULL == pStmt->pLastFunc) { nodesDestroyNode((SNode*)pStmt); CHECK_OUT_OF_MEM(NULL); } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index aa9ead2e278c96daeff7cff8a7f44ba9fa5aa2a6..ec5f6c4e570206f1a08d537f735605d0d54d53ef 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -464,6 +464,11 @@ static int32_t collectMetaKeyFromShowDnodeVariables(SCollectMetaKeyCxt* pCxt, SS return code; } +static int32_t collectMetaKeyFromShowVnodes(SCollectMetaKeyCxt* pCxt, SShowVnodesStmt* pStmt) { + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VNODES, + pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) { return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } @@ -604,6 +609,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowVariables(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: return collectMetaKeyFromShowDnodeVariables(pCxt, (SShowDnodeVariablesStmt*)pStmt); + case QUERY_NODE_SHOW_VNODES_STMT: + return collectMetaKeyFromShowVnodes(pCxt, (SShowVnodesStmt*)pStmt); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index bd8491a59b98a3bd9dd44e23eecab02a2e4a729a..1af61b0ca833d33f36983bce9f4898da369e9bfb 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -1129,11 +1129,14 @@ static int32_t parseTableOptions(SInsertParseContext* pCxt) { NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); if (TK_TTL == sToken.type) { pCxt->pSql += index; - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); if (TK_NK_INTEGER != sToken.type) { return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z); } pCxt->createTblReq.ttl = taosStr2Int32(sToken.z, NULL, 10); + if (pCxt->createTblReq.ttl < 0) { + return buildSyntaxErrMsg(&pCxt->msg, "Invalid option ttl", sToken.z); + } } else if (TK_COMMENT == sToken.type) { pCxt->pSql += index; NEXT_TOKEN(pCxt->pSql, sToken); @@ -1420,9 +1423,7 @@ static int32_t parseDataFromFile(SInsertParseContext* pCxt, SToken filePath, STa } static void destroyInsertParseContextForTable(SInsertParseContext* pCxt) { - if (!pCxt->pComCxt->async) { - taosMemoryFreeClear(pCxt->pTableMeta); - } + taosMemoryFreeClear(pCxt->pTableMeta); destroyBoundColumnInfo(&pCxt->tags); tdDestroySVCreateTbReq(&pCxt->createTblReq); } @@ -1742,7 +1743,7 @@ static int32_t skipTableOptions(SInsertParseSyntaxCxt* pCxt) { NEXT_TOKEN_KEEP_SQL(pCxt->pSql, sToken, index); if (TK_TTL == sToken.type || TK_COMMENT == sToken.type) { pCxt->pSql += index; - NEXT_TOKEN(pCxt->pSql, sToken); + NEXT_TOKEN_WITH_PREV(pCxt->pSql, sToken); } else { break; } @@ -2312,7 +2313,7 @@ static int32_t smlBoundColumnData(SArray* cols, SParsedDataColInfo* pColList, SS SToken sToken = {.n = kv->keyLen, .z = (char*)kv->key}; col_id_t t = lastColIdx + 1; col_id_t index = ((t == 0 && !isTag) ? 0 : findCol(&sToken, t, nCols, pSchema)); - uDebug("SML, index:%d, t:%d, ncols:%d, kv->name:%s", index, t, nCols, kv->key); + uDebug("SML, index:%d, t:%d, ncols:%d", index, t, nCols); if (index < 0 && t > 0) { index = findCol(&sToken, 0, t, pSchema); isOrdered = false; @@ -2533,9 +2534,7 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols if (p) kv = *p; } - if (!kv || kv->length == 0) { - MemRowAppend(&pBuf, NULL, 0, ¶m); - } else { + if (kv){ int32_t colLen = kv->length; if (pColSchema->type == TSDB_DATA_TYPE_TIMESTAMP) { // uError("SML:data before:%" PRId64 ", precision:%d", kv->i, pTableMeta->tableInfo.precision); @@ -2548,6 +2547,8 @@ int32_t smlBindData(void* handle, SArray* tags, SArray* colsSchema, SArray* cols } else { MemRowAppend(&pBuf, &(kv->value), colLen, ¶m); } + }else{ + pBuilder->hasNone = true; } if (PRIMARYKEY_TIMESTAMP_COL_ID == pColSchema->colId) { diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 80ec447f66157dd00e1081ee87e431a58b5998cd..68eb3e6fb1985dda6a8db9106fc2a6bcc8cd3583 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -187,6 +187,7 @@ static SKeyword keywordTable[] = { {"SNODES", TK_SNODES}, {"SOFFSET", TK_SOFFSET}, {"SPLIT", TK_SPLIT}, + {"STT_TRIGGER", TK_STT_TRIGGER}, {"STABLE", TK_STABLE}, {"STABLES", TK_STABLES}, {"STATE", TK_STATE}, @@ -199,6 +200,8 @@ static SKeyword keywordTable[] = { {"SYSINFO", TK_SYSINFO}, {"TABLE", TK_TABLE}, {"TABLES", TK_TABLES}, + {"TABLE_PREFIX", TK_TABLE_PREFIX}, + {"TABLE_SUFFIX", TK_TABLE_SUFFIX}, {"TAG", TK_TAG}, {"TAGS", TK_TAGS}, {"TBNAME", TK_TBNAME}, @@ -213,6 +216,7 @@ static SKeyword keywordTable[] = { {"TRANSACTIONS", TK_TRANSACTIONS}, {"TRIGGER", TK_TRIGGER}, {"TRIM", TK_TRIM}, + {"TSDB_PAGESIZE", TK_TSDB_PAGESIZE}, {"TSERIES", TK_TSERIES}, {"TTL", TK_TTL}, {"UNION", TK_UNION}, @@ -228,6 +232,7 @@ static SKeyword keywordTable[] = { {"VERBOSE", TK_VERBOSE}, {"VGROUP", TK_VGROUP}, {"VGROUPS", TK_VGROUPS}, + {"VNODES", TK_VNODES}, {"WAL_FSYNC_PERIOD", TK_WAL_FSYNC_PERIOD}, {"WAL_LEVEL", TK_WAL_LEVEL}, {"WAL_RETENTION_PERIOD", TK_WAL_RETENTION_PERIOD}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a88b7b417fc3d7ce807ae6f263db52cab9bcf918..ea9eefbb2999da482202e99de1075cd3588f2b26 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -245,6 +245,12 @@ static const SSysTableShowAdapter sysTableShowAdapter[] = { .numOfShowCols = 1, .pShowCols = {"*"} }, + { .showType = QUERY_NODE_SHOW_VNODES_STMT, + .pDbName = TSDB_INFORMATION_SCHEMA_DB, + .pTableName = TSDB_INS_TABLE_VNODES, + .numOfShowCols = 1, + .pShowCols = {"*"} + }, }; // clang-format on @@ -1277,6 +1283,36 @@ static int32_t rewriteCountStar(STranslateContext* pCxt, SFunctionNode* pCount) return code; } +static bool isCountTbname(SFunctionNode* pFunc) { + if (FUNCTION_TYPE_COUNT != pFunc->funcType || 1 != LIST_LENGTH(pFunc->pParameterList)) { + return false; + } + SNode* pPara = nodesListGetNode(pFunc->pParameterList, 0); + return (QUERY_NODE_FUNCTION == nodeType(pPara) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPara)->funcType); +} + +// count(tbname) is rewritten as count(ts) for scannning optimization +static int32_t rewriteCountTbname(STranslateContext* pCxt, SFunctionNode* pCount) { + SFunctionNode* pTbname = (SFunctionNode*)nodesListGetNode(pCount->pParameterList, 0); + const char* pTableAlias = NULL; + if (LIST_LENGTH(pTbname->pParameterList) > 0) { + pTableAlias = ((SValueNode*)nodesListGetNode(pTbname->pParameterList, 0))->literal; + } + STableNode* pTable = NULL; + int32_t code = findTable(pCxt, pTableAlias, &pTable); + if (TSDB_CODE_SUCCESS == code) { + SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); + if (NULL == pCol) { + code = TSDB_CODE_OUT_OF_MEMORY; + } else { + setColumnInfoBySchema((SRealTableNode*)pTable, ((SRealTableNode*)pTable)->pMeta->schema, -1, pCol); + NODES_DESTORY_LIST(pCount->pParameterList); + code = nodesListMakeAppend(&pCount->pParameterList, (SNode*)pCol); + } + } + return code; +} + static bool hasInvalidFuncNesting(SNodeList* pParameterList) { bool hasInvalidFunc = false; nodesWalkExprs(pParameterList, haveVectorFunction, &hasInvalidFunc); @@ -1312,6 +1348,9 @@ static int32_t translateAggFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (isCountStar(pFunc)) { return rewriteCountStar(pCxt, pFunc); } + if (isCountTbname(pFunc)) { + return rewriteCountTbname(pCxt, pFunc); + } return TSDB_CODE_SUCCESS; } @@ -3308,10 +3347,16 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { if (TSDB_CODE_SUCCESS == code) { code = translateDeleteWhere(pCxt, pDelete); } + pCxt->currClause = SQL_CLAUSE_SELECT; if (TSDB_CODE_SUCCESS == code) { - pCxt->currClause = SQL_CLAUSE_SELECT; code = translateExpr(pCxt, &pDelete->pCountFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, &pDelete->pFirstFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, &pDelete->pLastFunc); + } return code; } @@ -3476,6 +3521,10 @@ static int32_t buildCreateDbReq(STranslateContext* pCxt, SCreateDatabaseStmt* pS pReq->walRetentionSize = pStmt->pOptions->walRetentionSize; pReq->walRollPeriod = pStmt->pOptions->walRollPeriod; pReq->walSegmentSize = pStmt->pOptions->walSegmentSize; + pReq->sstTrigger = pStmt->pOptions->sstTrigger; + pReq->hashPrefix = pStmt->pOptions->tablePrefix; + pReq->hashSuffix = pStmt->pOptions->tableSuffix; + pReq->tsdbPageSize = pStmt->pOptions->tsdbPageSize; pReq->ignoreExist = pStmt->ignoreExists; return buildCreateDbRetentions(pStmt->pOptions->pRetentions, pReq); } @@ -3720,6 +3769,10 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "pagesize", pOptions->pagesize, TSDB_MIN_PAGESIZE_PER_VNODE, TSDB_MAX_PAGESIZE_PER_VNODE); } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbRangeOption(pCxt, "tsdbPagesize", pOptions->tsdbPageSize, TSDB_MIN_TSDB_PAGESIZE, + TSDB_MAX_TSDB_PAGESIZE); + } if (TSDB_CODE_SUCCESS == code) { code = checkDbPrecisionOption(pCxt, pOptions); } @@ -3760,6 +3813,15 @@ static int32_t checkDatabaseOptions(STranslateContext* pCxt, const char* pDbName code = checkDbRangeOption(pCxt, "walSegmentSize", pOptions->walSegmentSize, TSDB_DB_MIN_WAL_SEGMENT_SIZE, INT32_MAX); } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbRangeOption(pCxt, "sstTrigger", pOptions->sstTrigger, TSDB_MIN_STT_TRIGGER, TSDB_MAX_STT_TRIGGER); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbRangeOption(pCxt, "tablePrefix", pOptions->tablePrefix, TSDB_MIN_HASH_PREFIX, TSDB_MAX_HASH_PREFIX); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkDbRangeOption(pCxt, "tableSuffix", pOptions->tableSuffix, TSDB_MIN_HASH_SUFFIX, TSDB_MAX_HASH_SUFFIX); + } if (TSDB_CODE_SUCCESS == code) { code = checkOptionsDependency(pCxt, pDbName, pOptions); } @@ -3833,6 +3895,7 @@ static void buildAlterDbReq(STranslateContext* pCxt, SAlterDatabaseStmt* pStmt, pReq->cacheLast = pStmt->pOptions->cacheModel; pReq->cacheLastSize = pStmt->pOptions->cacheLastSize; pReq->replications = pStmt->pOptions->replica; + pReq->sstTrigger = pStmt->pOptions->sstTrigger; return; } @@ -5837,6 +5900,25 @@ static int32_t rewriteShowDnodeVariables(STranslateContext* pCxt, SQuery* pQuery return code; } +static int32_t rewriteShowVnodes(STranslateContext* pCxt, SQuery* pQuery) { + SShowVnodesStmt* pShow = (SShowVnodesStmt*)(pQuery->pRoot); + SSelectStmt* pStmt = NULL; + int32_t code = createSelectStmtForShow(QUERY_NODE_SHOW_VNODES_STMT, &pStmt); + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pShow->pDnodeId) { + code = createOperatorNode(OP_TYPE_EQUAL, "dnode_id", pShow->pDnodeId, &pStmt->pWhere); + } else { + code = createOperatorNode(OP_TYPE_EQUAL, "dnode_ep", pShow->pDnodeEndpoint, &pStmt->pWhere); + } + } + if (TSDB_CODE_SUCCESS == code) { + pQuery->showRewrite = true; + nodesDestroyNode(pQuery->pRoot); + pQuery->pRoot = (SNode*)pStmt; + } + return code; +} + static SNode* createBlockDistInfoFunc() { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { @@ -5884,12 +5966,6 @@ typedef struct SVgroupCreateTableBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupCreateTableBatch; -static void destroyCreateTbReq(SVCreateTbReq* pReq) { - taosMemoryFreeClear(pReq->name); - taosMemoryFreeClear(pReq->comment); - taosMemoryFreeClear(pReq->ntb.schemaRow.pSchema); -} - static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* pStmt, const SVgroupInfo* pVgroupInfo, SVgroupCreateTableBatch* pBatch) { char dbFName[TSDB_DB_FNAME_LEN] = {0}; @@ -5904,7 +5980,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* if (pStmt->pOptions->commentNull == false) { req.comment = strdup(pStmt->pOptions->comment); if (NULL == req.comment) { - destroyCreateTbReq(&req); + tdDestroySVCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } req.commentLen = strlen(pStmt->pOptions->comment); @@ -5915,7 +5991,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* req.ntb.schemaRow.version = 1; req.ntb.schemaRow.pSchema = taosMemoryCalloc(req.ntb.schemaRow.nCols, sizeof(SSchema)); if (NULL == req.name || NULL == req.ntb.schemaRow.pSchema) { - destroyCreateTbReq(&req); + tdDestroySVCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } if (pStmt->ignoreExists) { @@ -5931,7 +6007,7 @@ static int32_t buildNormalTableBatchReq(int32_t acctId, const SCreateTableStmt* strcpy(pBatch->dbName, pStmt->dbName); pBatch->req.pArray = taosArrayInit(1, sizeof(struct SVCreateTbReq)); if (NULL == pBatch->req.pArray) { - destroyCreateTbReq(&req); + tdDestroySVCreateTbReq(&req); return TSDB_CODE_OUT_OF_MEMORY; } taosArrayPush(pBatch->req.pArray, &req); @@ -5976,16 +6052,7 @@ static void destroyCreateTbReqBatch(void* data) { size_t size = taosArrayGetSize(pTbBatch->req.pArray); for (int32_t i = 0; i < size; ++i) { SVCreateTbReq* pTableReq = taosArrayGet(pTbBatch->req.pArray, i); - taosMemoryFreeClear(pTableReq->name); - taosMemoryFreeClear(pTableReq->comment); - - if (pTableReq->type == TSDB_NORMAL_TABLE) { - taosMemoryFreeClear(pTableReq->ntb.schemaRow.pSchema); - } else if (pTableReq->type == TSDB_CHILD_TABLE) { - taosMemoryFreeClear(pTableReq->ctb.pTag); - taosMemoryFreeClear(pTableReq->ctb.name); - taosArrayDestroy(pTableReq->ctb.tagName); - } + tdDestroySVCreateTbReq(pTableReq); } taosArrayDestroy(pTbBatch->req.pArray); @@ -6346,6 +6413,8 @@ static int32_t rewriteCreateSubTable(STranslateContext* pCxt, SCreateSubTableCla if (TSDB_CODE_SUCCESS == code) { addCreateTbReqIntoVgroup(pCxt->pParseCxt->acctId, pVgroupHashmap, pStmt, pTag, pSuperTableMeta->uid, pStmt->useTableName, &info, tagName, pSuperTableMeta->tableInfo.numOfTags); + } else { + taosMemoryFree(pTag); } taosArrayDestroy(tagName); @@ -6408,7 +6477,8 @@ typedef struct SVgroupDropTableBatch { char dbName[TSDB_DB_NAME_LEN]; } SVgroupDropTableBatch; -static void addDropTbReqIntoVgroup(SHashObj* pVgroupHashmap, SDropTableClause* pClause, SVgroupInfo* pVgInfo, uint64_t suid) { +static void addDropTbReqIntoVgroup(SHashObj* pVgroupHashmap, SDropTableClause* pClause, SVgroupInfo* pVgInfo, + uint64_t suid) { SVDropTbReq req = {.name = pClause->tableName, .suid = suid, .igNotExists = pClause->ignoreNotExists}; SVgroupDropTableBatch* pTableBatch = taosHashGet(pVgroupHashmap, &pVgInfo->vgId, sizeof(pVgInfo->vgId)); if (NULL == pTableBatch) { @@ -6565,7 +6635,17 @@ static int32_t buildUpdateTagValReq(STranslateContext* pCxt, SAlterTableStmt* pS pReq->colId = pSchema->colId; SDataType targetDt = schemaToDataType(pTableMeta->tableInfo.precision, pSchema); - if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { + + if (QUERY_NODE_VALUE != pStmt->pVal->node.type) { + SValueNode* pVal = NULL; + pCxt->errCode = createTagValFromExpr(pCxt, targetDt, (SNode*)pStmt->pVal, &pVal); + if (pCxt->errCode) { + return pCxt->errCode; + } + + nodesDestroyNode((SNode*)pStmt->pVal); + pStmt->pVal = pVal; + } else if (DEAL_RES_ERROR == translateValueImpl(pCxt, pStmt->pVal, targetDt, true)) { return pCxt->errCode; } @@ -6946,6 +7026,9 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: code = rewriteShowDnodeVariables(pCxt, pQuery); break; + case QUERY_NODE_SHOW_VNODES_STMT: + code = rewriteShowVnodes(pCxt, pQuery); + break; case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: code = rewriteShowTableDist(pCxt, pQuery); break; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 32513fd0b6f56097b2b7f08ae03725ce39498a37..daab80667ca88b6f1d1929813d98b2941fad2c57 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -1124,7 +1124,7 @@ int32_t getTableMetaFromCacheForInsert(SArray* pTableMetaPos, SParseMetaCache* p int32_t reqIndex = *(int32_t*)taosArrayGet(pTableMetaPos, tableNo); SMetaRes* pRes = taosArrayGet(pMetaCache->pTableMetaData, reqIndex); if (TSDB_CODE_SUCCESS == pRes->code) { - *pMeta = pRes->pRes; + *pMeta = tableMetaDup(pRes->pRes); if (NULL == *pMeta) { return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index c4bd1aff044a491edede232eff74b8dea1feeadb..a2a52aa0e0b507864b1d1c9cd82656392d678aa5 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 426 +#define YYNOCODE 430 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SAlterOption yy5; - int8_t yy59; - int64_t yy69; - EJoinType yy156; - SNodeList* yy172; - EFillMode yy186; - SToken yy209; - int32_t yy232; - SNode* yy272; - bool yy293; - EOperatorType yy392; - ENullOrder yy493; - SDataType yy616; - EOrder yy818; + int64_t yy49; + SDataType yy84; + EFillMode yy134; + SToken yy149; + EOrder yy158; + int32_t yy160; + SNode* yy312; + EOperatorType yy320; + int8_t yy363; + SAlterOption yy405; + ENullOrder yy417; + bool yy497; + SNodeList* yy824; + EJoinType yy832; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,17 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 667 -#define YYNRULE 489 -#define YYNTOKEN 305 -#define YY_MAX_SHIFT 666 -#define YY_MIN_SHIFTREDUCE 972 -#define YY_MAX_SHIFTREDUCE 1460 -#define YY_ERROR_ACTION 1461 -#define YY_ACCEPT_ACTION 1462 -#define YY_NO_ACTION 1463 -#define YY_MIN_REDUCE 1464 -#define YY_MAX_REDUCE 1952 +#define YYNSTATE 675 +#define YYNRULE 496 +#define YYNTOKEN 309 +#define YY_MAX_SHIFT 674 +#define YY_MIN_SHIFTREDUCE 985 +#define YY_MAX_SHIFTREDUCE 1480 +#define YY_ERROR_ACTION 1481 +#define YY_ACCEPT_ACTION 1482 +#define YY_NO_ACTION 1483 +#define YY_MIN_REDUCE 1484 +#define YY_MAX_REDUCE 1979 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,650 +216,690 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2259) +#define YY_ACTTAB_COUNT (2512) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 433, 1930, 434, 1499, 1593, 441, 526, 434, 1499, 513, - /* 10 */ 30, 260, 39, 37, 1929, 326, 325, 117, 1927, 1702, - /* 20 */ 339, 1465, 1261, 146, 471, 40, 38, 36, 35, 34, - /* 30 */ 1786, 552, 1606, 1337, 1604, 1259, 344, 552, 1287, 1649, - /* 40 */ 1651, 378, 107, 1774, 526, 106, 105, 104, 103, 102, - /* 50 */ 101, 100, 99, 98, 1770, 117, 1332, 432, 1804, 64, - /* 60 */ 436, 14, 476, 36, 35, 34, 553, 148, 1267, 1476, - /* 70 */ 450, 1756, 1604, 577, 39, 37, 1400, 1595, 1766, 1772, - /* 80 */ 328, 1930, 339, 1526, 1261, 1804, 217, 1286, 1770, 1, - /* 90 */ 571, 1005, 1656, 542, 164, 1337, 1818, 1259, 1927, 327, - /* 100 */ 95, 1787, 580, 1789, 1790, 576, 43, 571, 1654, 158, - /* 110 */ 1864, 663, 1766, 1772, 330, 1860, 159, 513, 1332, 63, - /* 120 */ 1930, 78, 1643, 14, 571, 1339, 1340, 1703, 163, 541, - /* 130 */ 1267, 1009, 1010, 165, 33, 32, 1890, 1927, 40, 38, - /* 140 */ 36, 35, 34, 543, 63, 63, 640, 639, 638, 637, - /* 150 */ 349, 2, 636, 635, 128, 630, 629, 628, 627, 626, - /* 160 */ 625, 624, 139, 620, 619, 618, 348, 347, 615, 614, - /* 170 */ 1262, 107, 1260, 663, 106, 105, 104, 103, 102, 101, - /* 180 */ 100, 99, 98, 440, 1775, 1287, 436, 1339, 1340, 223, - /* 190 */ 224, 11, 10, 1265, 1266, 1770, 1314, 1315, 1317, 1318, - /* 200 */ 1319, 1320, 1321, 1322, 573, 569, 1330, 1331, 1333, 1334, - /* 210 */ 1335, 1336, 1338, 1341, 1464, 496, 1431, 33, 32, 1766, - /* 220 */ 1772, 40, 38, 36, 35, 34, 526, 167, 494, 150, - /* 230 */ 492, 571, 1262, 1564, 1260, 1261, 210, 55, 116, 115, - /* 240 */ 114, 113, 112, 111, 110, 109, 108, 305, 1259, 63, - /* 250 */ 516, 1701, 22, 300, 1604, 1265, 1266, 167, 1314, 1315, - /* 260 */ 1317, 1318, 1319, 1320, 1321, 1322, 573, 569, 1330, 1331, - /* 270 */ 1333, 1334, 1335, 1336, 1338, 1341, 39, 37, 1650, 1651, - /* 280 */ 1373, 1267, 167, 167, 339, 552, 1261, 613, 1286, 49, - /* 290 */ 1160, 1161, 76, 305, 1786, 1487, 516, 1337, 1421, 1259, - /* 300 */ 1118, 602, 601, 600, 1122, 599, 1124, 1125, 598, 1127, - /* 310 */ 595, 342, 1133, 592, 1135, 1136, 589, 586, 1285, 146, - /* 320 */ 1332, 1581, 1804, 173, 663, 14, 479, 478, 1606, 377, - /* 330 */ 553, 376, 1267, 1656, 1930, 1756, 1756, 577, 39, 37, - /* 340 */ 312, 535, 1419, 1420, 1422, 1423, 339, 1928, 1261, 1654, - /* 350 */ 1705, 1927, 84, 2, 42, 71, 1656, 63, 70, 1337, - /* 360 */ 1818, 1259, 1267, 343, 95, 1787, 580, 1789, 1790, 576, - /* 370 */ 605, 571, 1654, 1597, 1864, 663, 345, 1589, 330, 1860, - /* 380 */ 159, 1288, 1332, 1262, 146, 1260, 1022, 167, 1021, 1339, - /* 390 */ 1340, 33, 32, 1606, 1267, 40, 38, 36, 35, 34, - /* 400 */ 1891, 1930, 384, 39, 37, 1342, 1265, 1266, 1486, 1485, - /* 410 */ 611, 339, 1786, 1261, 165, 8, 1023, 438, 1927, 634, - /* 420 */ 632, 1080, 611, 1284, 1337, 622, 1259, 167, 549, 137, - /* 430 */ 136, 608, 607, 606, 1262, 1695, 1260, 663, 1484, 303, - /* 440 */ 1804, 137, 136, 608, 607, 606, 172, 1332, 575, 1756, - /* 450 */ 1756, 1339, 1340, 1756, 1082, 577, 127, 1265, 1266, 1267, - /* 460 */ 1314, 1315, 1317, 1318, 1319, 1320, 1321, 1322, 573, 569, - /* 470 */ 1330, 1331, 1333, 1334, 1335, 1336, 1338, 1341, 1818, 1756, - /* 480 */ 9, 1591, 293, 1787, 580, 1789, 1790, 576, 574, 571, - /* 490 */ 568, 1836, 167, 74, 125, 167, 1262, 222, 1260, 33, - /* 500 */ 32, 1529, 663, 40, 38, 36, 35, 34, 551, 160, - /* 510 */ 1872, 1873, 1587, 1877, 1483, 1600, 1339, 1340, 252, 1265, - /* 520 */ 1266, 1579, 1314, 1315, 1317, 1318, 1319, 1320, 1321, 1322, - /* 530 */ 573, 569, 1330, 1331, 1333, 1334, 1335, 1336, 1338, 1341, - /* 540 */ 1700, 526, 300, 33, 32, 1457, 91, 40, 38, 36, - /* 550 */ 35, 34, 169, 167, 316, 1756, 1241, 1242, 623, 124, - /* 560 */ 1576, 1262, 1879, 1260, 26, 482, 481, 1596, 1462, 1604, - /* 570 */ 33, 32, 123, 1582, 40, 38, 36, 35, 34, 213, - /* 580 */ 1786, 1411, 477, 480, 1265, 1266, 1876, 1314, 1315, 1317, - /* 590 */ 1318, 1319, 1320, 1321, 1322, 573, 569, 1330, 1331, 1333, - /* 600 */ 1334, 1335, 1336, 1338, 1341, 39, 37, 475, 1804, 317, - /* 610 */ 146, 315, 314, 339, 473, 1261, 578, 1361, 475, 1607, - /* 620 */ 549, 1756, 611, 577, 28, 1299, 1337, 354, 1259, 474, - /* 630 */ 33, 32, 1456, 450, 40, 38, 36, 35, 34, 538, - /* 640 */ 474, 137, 136, 608, 607, 606, 1818, 1695, 127, 1332, - /* 650 */ 96, 1787, 580, 1789, 1790, 576, 572, 571, 175, 74, - /* 660 */ 1864, 1267, 526, 609, 1863, 1860, 1647, 1930, 554, 512, - /* 670 */ 33, 32, 122, 382, 40, 38, 36, 35, 34, 27, - /* 680 */ 164, 1599, 9, 1482, 1927, 1022, 125, 1021, 7, 1366, - /* 690 */ 1604, 33, 32, 1481, 1565, 40, 38, 36, 35, 34, - /* 700 */ 469, 250, 1872, 548, 663, 547, 33, 32, 1930, 1930, - /* 710 */ 40, 38, 36, 35, 34, 1023, 544, 539, 1339, 1340, - /* 720 */ 526, 166, 164, 307, 1756, 1927, 1927, 135, 487, 1404, - /* 730 */ 526, 383, 39, 37, 1756, 1286, 1480, 302, 1879, 1284, - /* 740 */ 339, 389, 1261, 497, 307, 526, 412, 604, 1604, 424, - /* 750 */ 526, 549, 1299, 1337, 1477, 1259, 404, 209, 1604, 61, - /* 760 */ 1359, 405, 1875, 1262, 373, 1260, 397, 1479, 425, 1478, - /* 770 */ 399, 490, 255, 1604, 54, 484, 1332, 1756, 1604, 127, - /* 780 */ 208, 1359, 1475, 375, 371, 419, 1265, 1266, 1267, 1314, - /* 790 */ 1315, 1317, 1318, 1319, 1320, 1321, 1322, 573, 569, 1330, - /* 800 */ 1331, 1333, 1334, 1335, 1336, 1338, 1341, 390, 1756, 2, - /* 810 */ 1756, 1397, 58, 526, 1360, 57, 1879, 125, 505, 386, - /* 820 */ 1289, 33, 32, 1756, 448, 40, 38, 36, 35, 34, - /* 830 */ 1474, 663, 161, 1872, 1873, 1360, 1877, 177, 176, 1505, - /* 840 */ 1874, 1604, 1347, 1009, 1010, 1339, 1340, 423, 1286, 1580, - /* 850 */ 418, 417, 416, 415, 414, 411, 410, 409, 408, 407, - /* 860 */ 403, 402, 401, 400, 394, 393, 392, 391, 549, 388, - /* 870 */ 387, 1756, 1656, 616, 1473, 1393, 29, 337, 1354, 1355, - /* 880 */ 1356, 1357, 1358, 1362, 1363, 1364, 1365, 658, 1655, 468, - /* 890 */ 1262, 610, 1260, 1286, 1647, 1068, 127, 29, 337, 1354, - /* 900 */ 1355, 1356, 1357, 1358, 1362, 1363, 1364, 1365, 272, 613, - /* 910 */ 536, 1634, 1316, 1265, 1266, 1756, 1314, 1315, 1317, 1318, - /* 920 */ 1319, 1320, 1321, 1322, 573, 569, 1330, 1331, 1333, 1334, - /* 930 */ 1335, 1336, 1338, 1341, 125, 147, 1472, 1786, 561, 352, - /* 940 */ 278, 351, 1884, 1393, 1743, 482, 481, 1516, 1774, 162, - /* 950 */ 1872, 1873, 123, 1877, 276, 60, 1805, 232, 59, 1770, - /* 960 */ 526, 526, 477, 480, 1471, 1804, 44, 4, 244, 483, - /* 970 */ 145, 449, 1601, 578, 180, 429, 427, 1756, 1756, 1930, - /* 980 */ 577, 1930, 1500, 1766, 1772, 334, 1470, 1786, 1604, 1604, - /* 990 */ 526, 361, 164, 554, 164, 571, 1927, 216, 1927, 498, - /* 1000 */ 556, 499, 1469, 1818, 1468, 1756, 63, 94, 1787, 580, - /* 1010 */ 1789, 1790, 576, 526, 571, 1804, 558, 1864, 1604, 336, - /* 1020 */ 335, 306, 1860, 578, 506, 1316, 526, 1756, 1756, 1275, - /* 1030 */ 577, 201, 77, 1930, 199, 1396, 1644, 510, 526, 1930, - /* 1040 */ 1337, 1604, 1268, 1756, 93, 1756, 166, 1511, 1467, 227, - /* 1050 */ 1927, 350, 164, 1818, 1604, 1786, 1927, 95, 1787, 580, - /* 1060 */ 1789, 1790, 576, 1332, 571, 526, 1604, 1864, 41, 485, - /* 1070 */ 1316, 330, 1860, 1943, 526, 1267, 522, 53, 509, 68, - /* 1080 */ 67, 381, 1898, 1804, 171, 524, 221, 203, 526, 1756, - /* 1090 */ 202, 578, 205, 1604, 207, 204, 1756, 206, 577, 525, - /* 1100 */ 301, 1509, 1604, 369, 1894, 367, 363, 359, 356, 353, - /* 1110 */ 1270, 554, 131, 526, 1786, 1212, 1604, 1269, 566, 567, - /* 1120 */ 526, 1818, 134, 488, 261, 94, 1787, 580, 1789, 1790, - /* 1130 */ 576, 346, 571, 225, 135, 1864, 51, 550, 666, 306, - /* 1140 */ 1860, 1604, 1804, 562, 167, 236, 51, 323, 1604, 41, - /* 1150 */ 578, 1930, 267, 90, 1786, 1756, 617, 577, 41, 519, - /* 1160 */ 1777, 11, 10, 87, 164, 249, 156, 3, 1927, 229, - /* 1170 */ 254, 656, 652, 648, 644, 265, 584, 1276, 1066, 1271, - /* 1180 */ 1818, 1111, 1804, 1418, 294, 1787, 580, 1789, 1790, 576, - /* 1190 */ 578, 571, 239, 1367, 1786, 1756, 1323, 577, 1459, 1460, - /* 1200 */ 1279, 1281, 257, 1779, 259, 271, 134, 92, 135, 5, - /* 1210 */ 230, 1049, 569, 1330, 1331, 1333, 1334, 1335, 1336, 559, - /* 1220 */ 1818, 360, 1804, 1139, 95, 1787, 580, 1789, 1790, 576, - /* 1230 */ 578, 571, 268, 355, 1864, 1756, 119, 577, 330, 1860, - /* 1240 */ 1943, 134, 549, 523, 1050, 313, 1228, 1273, 174, 1921, - /* 1250 */ 385, 1351, 1284, 1143, 1272, 1150, 406, 413, 1697, 421, - /* 1260 */ 1818, 420, 422, 1786, 95, 1787, 580, 1789, 1790, 576, - /* 1270 */ 127, 571, 426, 428, 1864, 219, 430, 1290, 330, 1860, - /* 1280 */ 1943, 431, 439, 1148, 1292, 442, 183, 443, 138, 1883, - /* 1290 */ 554, 1804, 1291, 1235, 185, 212, 444, 1293, 188, 578, - /* 1300 */ 445, 190, 447, 72, 1756, 73, 577, 451, 125, 194, - /* 1310 */ 470, 472, 1594, 198, 118, 1590, 304, 1786, 200, 554, - /* 1320 */ 140, 269, 141, 250, 1872, 548, 1592, 547, 1588, 1818, - /* 1330 */ 1930, 142, 143, 285, 1787, 580, 1789, 1790, 576, 211, - /* 1340 */ 571, 500, 1736, 164, 214, 1804, 507, 1927, 504, 511, - /* 1350 */ 218, 322, 534, 578, 514, 520, 501, 1735, 1756, 1930, - /* 1360 */ 577, 132, 1707, 517, 324, 1289, 81, 1786, 521, 133, - /* 1370 */ 270, 83, 166, 554, 537, 1605, 1927, 530, 1905, 234, - /* 1380 */ 1895, 238, 6, 1818, 1786, 532, 533, 285, 1787, 580, - /* 1390 */ 1789, 1790, 576, 329, 571, 1804, 546, 531, 540, 529, - /* 1400 */ 528, 248, 1288, 578, 1393, 126, 563, 560, 1756, 48, - /* 1410 */ 577, 1880, 1804, 1930, 1904, 85, 1648, 331, 1577, 659, - /* 1420 */ 578, 582, 264, 660, 243, 1756, 164, 577, 153, 1886, - /* 1430 */ 1927, 247, 245, 1818, 1786, 246, 253, 96, 1787, 580, - /* 1440 */ 1789, 1790, 576, 1845, 571, 273, 662, 1864, 299, 275, - /* 1450 */ 1818, 565, 1860, 256, 149, 1787, 580, 1789, 1790, 576, - /* 1460 */ 1786, 571, 1804, 52, 1946, 1926, 557, 286, 296, 258, - /* 1470 */ 578, 564, 295, 1750, 277, 1756, 1749, 577, 65, 1748, - /* 1480 */ 1747, 66, 1744, 357, 358, 1253, 1254, 170, 1804, 362, - /* 1490 */ 1742, 364, 365, 527, 366, 1741, 578, 368, 555, 1944, - /* 1500 */ 1818, 1756, 1740, 577, 96, 1787, 580, 1789, 1790, 576, - /* 1510 */ 1786, 571, 370, 1739, 1864, 372, 1738, 1230, 374, 1861, - /* 1520 */ 1231, 1718, 1786, 379, 380, 1716, 1818, 1717, 1715, 1690, - /* 1530 */ 294, 1787, 580, 1789, 1790, 576, 1786, 571, 1804, 1689, - /* 1540 */ 1200, 129, 1688, 1687, 69, 1686, 578, 395, 1681, 396, - /* 1550 */ 1804, 1756, 1685, 577, 1684, 1683, 1682, 398, 578, 1680, - /* 1560 */ 1679, 1678, 1677, 1756, 1804, 577, 1676, 1675, 1674, 1673, - /* 1570 */ 1672, 1671, 575, 1670, 1669, 1668, 1818, 1756, 1667, 577, - /* 1580 */ 289, 1787, 580, 1789, 1790, 576, 130, 571, 1818, 1786, - /* 1590 */ 1666, 1665, 149, 1787, 580, 1789, 1790, 576, 1664, 571, - /* 1600 */ 1663, 1662, 1818, 1202, 1660, 1659, 293, 1787, 580, 1789, - /* 1610 */ 1790, 576, 1661, 571, 1658, 1837, 1657, 1804, 545, 1531, - /* 1620 */ 178, 1530, 338, 120, 181, 578, 196, 1528, 179, 1496, - /* 1630 */ 1756, 157, 577, 435, 1012, 437, 1011, 1945, 1495, 182, - /* 1640 */ 152, 121, 1786, 452, 453, 467, 463, 459, 455, 195, - /* 1650 */ 1731, 1725, 1714, 189, 1786, 1818, 187, 1713, 1699, 294, - /* 1660 */ 1787, 580, 1789, 1790, 576, 1583, 571, 1527, 1786, 1042, - /* 1670 */ 1804, 1525, 454, 1523, 456, 340, 458, 457, 578, 1521, - /* 1680 */ 460, 75, 1804, 1756, 193, 577, 462, 461, 1519, 464, - /* 1690 */ 578, 465, 466, 1508, 1507, 1756, 1804, 577, 1492, 1585, - /* 1700 */ 1153, 1154, 197, 1584, 578, 1079, 1074, 50, 1818, 1756, - /* 1710 */ 1517, 577, 294, 1787, 580, 1789, 1790, 576, 631, 571, - /* 1720 */ 1818, 1076, 1786, 633, 279, 1787, 580, 1789, 1790, 576, - /* 1730 */ 1075, 571, 1512, 318, 1818, 319, 1786, 1510, 280, 1787, - /* 1740 */ 580, 1789, 1790, 576, 320, 571, 192, 186, 1786, 191, - /* 1750 */ 1804, 486, 489, 446, 1491, 491, 1490, 1489, 578, 493, - /* 1760 */ 495, 97, 1730, 1756, 1804, 577, 1237, 56, 1724, 184, - /* 1770 */ 502, 1712, 578, 1710, 508, 503, 1804, 1756, 215, 577, - /* 1780 */ 1711, 1709, 321, 1708, 578, 15, 144, 220, 1818, 1756, - /* 1790 */ 1245, 577, 281, 1787, 580, 1789, 1790, 576, 1706, 571, - /* 1800 */ 1698, 226, 1818, 518, 79, 1786, 288, 1787, 580, 1789, - /* 1810 */ 1790, 576, 228, 571, 1818, 1786, 515, 80, 290, 1787, - /* 1820 */ 580, 1789, 1790, 576, 82, 571, 87, 41, 231, 23, - /* 1830 */ 47, 1786, 1433, 1804, 233, 241, 235, 1415, 237, 242, - /* 1840 */ 1417, 578, 16, 1804, 25, 1777, 1756, 151, 577, 240, - /* 1850 */ 24, 578, 46, 1410, 86, 1786, 1756, 17, 577, 1804, - /* 1860 */ 1390, 251, 1389, 1776, 154, 1450, 45, 578, 18, 1439, - /* 1870 */ 1445, 1818, 1756, 13, 577, 282, 1787, 580, 1789, 1790, - /* 1880 */ 576, 1818, 571, 1804, 1444, 291, 1787, 580, 1789, 1790, - /* 1890 */ 576, 578, 571, 332, 1449, 1448, 1756, 1818, 577, 333, - /* 1900 */ 10, 283, 1787, 580, 1789, 1790, 576, 1277, 571, 1352, - /* 1910 */ 19, 1786, 1821, 1307, 1327, 570, 155, 1325, 31, 581, - /* 1920 */ 1324, 1818, 12, 20, 168, 292, 1787, 580, 1789, 1790, - /* 1930 */ 576, 1786, 571, 21, 583, 1140, 341, 585, 579, 1804, - /* 1940 */ 1137, 587, 588, 590, 1134, 591, 593, 578, 596, 1132, - /* 1950 */ 594, 1786, 1756, 1128, 577, 1126, 1131, 597, 1117, 1804, - /* 1960 */ 1130, 88, 1149, 603, 1129, 89, 62, 578, 262, 1145, - /* 1970 */ 612, 1786, 1756, 1071, 577, 1070, 1040, 1818, 1069, 1804, - /* 1980 */ 1067, 284, 1787, 580, 1789, 1790, 576, 578, 571, 1065, - /* 1990 */ 1064, 1786, 1756, 1063, 577, 263, 1086, 1818, 1061, 1804, - /* 2000 */ 1060, 297, 1787, 580, 1789, 1790, 576, 578, 571, 621, - /* 2010 */ 1059, 1058, 1756, 1057, 577, 1056, 1055, 1818, 1083, 1804, - /* 2020 */ 1081, 298, 1787, 580, 1789, 1790, 576, 578, 571, 1052, - /* 2030 */ 1051, 1786, 1756, 1048, 577, 1047, 1046, 1818, 1045, 1524, - /* 2040 */ 641, 1798, 1787, 580, 1789, 1790, 576, 1786, 571, 642, - /* 2050 */ 643, 1522, 645, 646, 647, 1520, 649, 1818, 651, 1804, - /* 2060 */ 650, 1797, 1787, 580, 1789, 1790, 576, 578, 571, 1518, - /* 2070 */ 653, 654, 1756, 655, 577, 1804, 1506, 657, 1002, 1488, - /* 2080 */ 266, 661, 664, 578, 1263, 274, 665, 1463, 1756, 1463, - /* 2090 */ 577, 1463, 1463, 1463, 1463, 1463, 1463, 1818, 1786, 1463, - /* 2100 */ 1463, 1796, 1787, 580, 1789, 1790, 576, 1463, 571, 1463, - /* 2110 */ 1463, 1463, 1463, 1818, 1786, 1463, 1463, 310, 1787, 580, - /* 2120 */ 1789, 1790, 576, 1463, 571, 1463, 1804, 1463, 1463, 1463, - /* 2130 */ 1463, 1463, 1463, 1463, 578, 1463, 1463, 1463, 1463, 1756, - /* 2140 */ 1463, 577, 1804, 1463, 1463, 1463, 1463, 1463, 1463, 1463, - /* 2150 */ 578, 1463, 1463, 1463, 1463, 1756, 1463, 577, 1463, 1463, - /* 2160 */ 1463, 1463, 1463, 1463, 1818, 1786, 1463, 1463, 309, 1787, - /* 2170 */ 580, 1789, 1790, 576, 1463, 571, 1463, 1463, 1463, 1463, - /* 2180 */ 1818, 1786, 1463, 1463, 311, 1787, 580, 1789, 1790, 576, - /* 2190 */ 1463, 571, 1463, 1804, 1463, 1463, 1463, 1463, 1463, 1463, - /* 2200 */ 1463, 578, 1463, 1463, 1463, 1463, 1756, 1463, 577, 1804, - /* 2210 */ 1463, 1463, 1463, 1463, 1463, 1463, 1463, 578, 1463, 1463, - /* 2220 */ 1463, 1463, 1756, 1463, 577, 1463, 1463, 1463, 1463, 1463, - /* 2230 */ 1463, 1818, 1463, 1463, 1463, 308, 1787, 580, 1789, 1790, - /* 2240 */ 576, 1463, 571, 1463, 1463, 1463, 1463, 1818, 1463, 1463, - /* 2250 */ 1463, 287, 1787, 580, 1789, 1790, 576, 1463, 571, + /* 0 */ 436, 1813, 437, 1519, 444, 345, 437, 1519, 1674, 1676, + /* 10 */ 33, 32, 39, 37, 40, 38, 36, 35, 34, 1620, + /* 20 */ 340, 530, 1281, 1801, 40, 38, 36, 35, 34, 1831, + /* 30 */ 1797, 1681, 117, 1357, 1797, 1279, 1770, 582, 312, 474, + /* 40 */ 30, 260, 1783, 1525, 581, 385, 148, 1679, 1496, 1629, + /* 50 */ 1035, 1813, 1034, 1801, 1793, 1799, 1352, 558, 1793, 1799, + /* 60 */ 329, 14, 1675, 1676, 1797, 472, 575, 1845, 1287, 1546, + /* 70 */ 575, 94, 1814, 584, 1816, 1817, 580, 1485, 575, 1831, + /* 80 */ 1036, 1891, 303, 362, 1802, 306, 1887, 582, 1793, 1799, + /* 90 */ 335, 666, 1783, 1, 581, 1797, 1451, 1957, 107, 1957, + /* 100 */ 575, 106, 105, 104, 103, 102, 101, 100, 99, 98, + /* 110 */ 166, 1681, 165, 1307, 1954, 671, 1954, 1845, 328, 1793, + /* 120 */ 1799, 149, 1814, 584, 1816, 1817, 580, 1679, 575, 1359, + /* 130 */ 1360, 575, 648, 647, 646, 645, 350, 1607, 644, 643, + /* 140 */ 128, 638, 637, 636, 635, 634, 633, 632, 631, 139, + /* 150 */ 627, 626, 625, 349, 348, 622, 621, 620, 619, 618, + /* 160 */ 36, 35, 34, 33, 32, 559, 1971, 40, 38, 36, + /* 170 */ 35, 34, 556, 556, 1306, 1282, 556, 1280, 1136, 606, + /* 180 */ 605, 604, 1140, 603, 1142, 1143, 602, 1145, 599, 63, + /* 190 */ 1151, 596, 1153, 1154, 593, 590, 453, 453, 1285, 1286, + /* 200 */ 553, 1334, 1335, 1337, 1338, 1339, 1340, 1341, 1342, 577, + /* 210 */ 573, 1350, 1351, 1353, 1354, 1355, 1356, 1358, 1361, 39, + /* 220 */ 37, 1420, 1178, 1179, 1307, 1831, 500, 340, 127, 1281, + /* 230 */ 33, 32, 167, 546, 40, 38, 36, 35, 34, 498, + /* 240 */ 1357, 496, 1279, 486, 485, 43, 42, 63, 558, 107, + /* 250 */ 123, 379, 106, 105, 104, 103, 102, 101, 100, 99, + /* 260 */ 98, 481, 484, 1352, 491, 22, 125, 480, 14, 545, + /* 270 */ 378, 84, 377, 1018, 1813, 1287, 39, 37, 530, 501, + /* 280 */ 1417, 250, 1899, 552, 340, 551, 1281, 478, 1957, 117, + /* 290 */ 1813, 1957, 1622, 209, 7, 327, 479, 1357, 1507, 1279, + /* 300 */ 2, 166, 1831, 146, 164, 1954, 1629, 494, 1954, 477, + /* 310 */ 582, 488, 1631, 1022, 1023, 1783, 208, 581, 1831, 150, + /* 320 */ 1352, 1957, 671, 1588, 435, 14, 579, 439, 167, 1441, + /* 330 */ 558, 1783, 1287, 581, 1956, 422, 1359, 1360, 1954, 1783, + /* 340 */ 1845, 135, 173, 517, 94, 1814, 584, 1816, 1817, 580, + /* 350 */ 326, 575, 58, 1729, 1891, 57, 1845, 2, 306, 1887, + /* 360 */ 293, 1814, 584, 1816, 1817, 580, 578, 575, 572, 1863, + /* 370 */ 1957, 1484, 539, 1439, 1440, 1442, 1443, 222, 71, 671, + /* 380 */ 146, 70, 1282, 164, 1280, 167, 167, 1954, 1506, 1632, + /* 390 */ 177, 176, 54, 1359, 1360, 116, 115, 114, 113, 112, + /* 400 */ 111, 110, 109, 108, 63, 1285, 1286, 1098, 1334, 1335, + /* 410 */ 1337, 1338, 1339, 1340, 1341, 1342, 577, 573, 1350, 1351, + /* 420 */ 1353, 1354, 1355, 1356, 1358, 1361, 33, 32, 1681, 1783, + /* 430 */ 40, 38, 36, 35, 34, 344, 26, 1261, 1262, 1282, + /* 440 */ 1100, 1280, 33, 32, 1679, 28, 40, 38, 36, 35, + /* 450 */ 34, 33, 32, 217, 562, 40, 38, 36, 35, 34, + /* 460 */ 1813, 317, 1285, 1286, 542, 1334, 1335, 1337, 1338, 1339, + /* 470 */ 1340, 1341, 1342, 577, 573, 1350, 1351, 1353, 1354, 1355, + /* 480 */ 1356, 1358, 1361, 39, 37, 302, 1413, 1304, 1831, 530, + /* 490 */ 63, 340, 78, 1281, 415, 210, 557, 427, 223, 224, + /* 500 */ 55, 1783, 1957, 581, 1357, 1308, 1279, 441, 516, 1416, + /* 510 */ 353, 1957, 1431, 1304, 400, 165, 428, 1629, 402, 1954, + /* 520 */ 318, 49, 316, 315, 1955, 476, 1845, 1352, 1954, 478, + /* 530 */ 95, 1814, 584, 1816, 1817, 580, 1722, 575, 1306, 1287, + /* 540 */ 1891, 548, 543, 167, 331, 1887, 159, 172, 1957, 1281, + /* 550 */ 1957, 477, 76, 305, 674, 64, 520, 305, 163, 393, + /* 560 */ 520, 164, 1279, 164, 8, 1954, 1917, 1954, 267, 553, + /* 570 */ 547, 389, 158, 33, 32, 343, 346, 40, 38, 36, + /* 580 */ 35, 34, 156, 146, 146, 1668, 671, 664, 660, 656, + /* 590 */ 652, 265, 1631, 1631, 1482, 1287, 1035, 127, 1034, 426, + /* 600 */ 1359, 1360, 421, 420, 419, 418, 417, 414, 413, 412, + /* 610 */ 411, 410, 406, 405, 404, 403, 397, 396, 395, 394, + /* 620 */ 1505, 391, 390, 314, 617, 1424, 1036, 92, 1367, 167, + /* 630 */ 230, 1306, 33, 32, 1306, 125, 40, 38, 36, 35, + /* 640 */ 34, 1305, 671, 1381, 63, 615, 1282, 252, 1280, 555, + /* 650 */ 160, 1899, 1900, 355, 1904, 352, 11, 10, 483, 482, + /* 660 */ 167, 1783, 563, 527, 137, 136, 612, 611, 610, 1285, + /* 670 */ 1286, 1732, 1334, 1335, 1337, 1338, 1339, 1340, 1341, 1342, + /* 680 */ 577, 573, 1350, 1351, 1353, 1354, 1355, 1356, 1358, 1361, + /* 690 */ 39, 37, 1362, 1957, 1287, 1957, 219, 502, 340, 374, + /* 700 */ 1281, 74, 1282, 167, 1280, 27, 164, 74, 164, 91, + /* 710 */ 1954, 1357, 1954, 1279, 1255, 1386, 212, 1504, 376, 372, + /* 720 */ 122, 1605, 124, 1625, 1813, 1285, 1286, 1722, 1503, 1624, + /* 730 */ 1621, 1502, 1477, 530, 1352, 33, 32, 1957, 175, 40, + /* 740 */ 38, 36, 35, 34, 169, 530, 1287, 39, 37, 1393, + /* 750 */ 164, 530, 1831, 1549, 1954, 340, 383, 1281, 1783, 1501, + /* 760 */ 582, 1629, 384, 443, 553, 1783, 439, 581, 1357, 1783, + /* 770 */ 1279, 9, 1783, 1629, 530, 642, 640, 517, 530, 1629, + /* 780 */ 558, 617, 61, 167, 1306, 392, 145, 1730, 1500, 407, + /* 790 */ 1845, 1352, 127, 671, 285, 1814, 584, 1816, 1817, 580, + /* 800 */ 1783, 575, 1629, 1287, 337, 336, 1629, 1359, 1360, 609, + /* 810 */ 1499, 530, 530, 1336, 1295, 530, 1336, 486, 485, 1476, + /* 820 */ 1957, 1319, 408, 451, 123, 1357, 452, 1288, 9, 1783, + /* 830 */ 125, 530, 1498, 166, 1495, 481, 484, 1954, 629, 1629, + /* 840 */ 1629, 480, 1626, 1629, 1309, 161, 1899, 1900, 1352, 1904, + /* 850 */ 671, 1783, 1728, 1282, 300, 1280, 630, 1618, 1601, 1629, + /* 860 */ 1287, 90, 33, 32, 1359, 1360, 40, 38, 36, 35, + /* 870 */ 34, 87, 1727, 1783, 300, 1783, 1285, 1286, 1494, 1334, + /* 880 */ 1335, 1337, 1338, 1339, 1340, 1341, 1342, 577, 573, 1350, + /* 890 */ 1351, 1353, 1354, 1355, 1356, 1358, 1361, 530, 530, 553, + /* 900 */ 530, 201, 530, 530, 199, 1906, 509, 570, 347, 503, + /* 910 */ 1282, 510, 1280, 514, 227, 307, 1614, 1493, 613, 1783, + /* 920 */ 1492, 1672, 1491, 1022, 1023, 1629, 1629, 127, 1629, 1903, + /* 930 */ 1629, 1629, 1490, 1285, 1286, 1489, 1334, 1335, 1337, 1338, + /* 940 */ 1339, 1340, 1341, 1342, 577, 573, 1350, 1351, 1353, 1354, + /* 950 */ 1355, 1356, 1358, 1361, 39, 37, 1379, 530, 1783, 530, + /* 960 */ 1616, 1783, 340, 1783, 1281, 125, 1336, 1296, 526, 1291, + /* 970 */ 528, 1911, 1413, 1783, 196, 1357, 1783, 1279, 1488, 1487, + /* 980 */ 162, 1899, 1900, 1606, 1904, 1629, 232, 1629, 152, 1813, + /* 990 */ 1299, 1301, 1906, 470, 466, 462, 458, 195, 1352, 1681, + /* 1000 */ 44, 4, 573, 1350, 1351, 1353, 1354, 1355, 1356, 1604, + /* 1010 */ 1287, 1380, 530, 530, 386, 1680, 1902, 1831, 614, 1783, + /* 1020 */ 1783, 1672, 324, 529, 261, 582, 1906, 387, 560, 147, + /* 1030 */ 1783, 565, 581, 75, 278, 2, 193, 571, 33, 32, + /* 1040 */ 1629, 1629, 40, 38, 36, 35, 34, 41, 276, 60, + /* 1050 */ 1901, 272, 59, 203, 1659, 1845, 202, 671, 216, 294, + /* 1060 */ 1814, 584, 1816, 1817, 580, 221, 575, 131, 180, 432, + /* 1070 */ 430, 1359, 1360, 29, 338, 1374, 1375, 1376, 1377, 1378, + /* 1080 */ 1382, 1383, 1384, 1385, 205, 1290, 207, 204, 615, 206, + /* 1090 */ 1536, 307, 53, 513, 11, 10, 1289, 77, 1232, 192, + /* 1100 */ 186, 1531, 191, 1529, 63, 1589, 449, 137, 136, 612, + /* 1110 */ 611, 610, 487, 134, 615, 135, 225, 1282, 523, 1280, + /* 1120 */ 51, 236, 184, 489, 1319, 492, 1479, 1480, 1804, 1612, + /* 1130 */ 51, 41, 1379, 137, 136, 612, 611, 610, 41, 588, + /* 1140 */ 1285, 1286, 93, 1334, 1335, 1337, 1338, 1339, 1340, 1341, + /* 1150 */ 1342, 577, 573, 1350, 1351, 1353, 1354, 1355, 1356, 1358, + /* 1160 */ 1361, 134, 135, 119, 229, 134, 1129, 1062, 213, 623, + /* 1170 */ 624, 1438, 239, 576, 1371, 1806, 608, 1497, 68, 67, + /* 1180 */ 382, 1387, 1343, 171, 255, 540, 471, 1380, 244, 271, + /* 1190 */ 1157, 1082, 1080, 1832, 1813, 351, 1520, 1669, 1921, 301, + /* 1200 */ 1063, 554, 370, 254, 368, 364, 360, 357, 354, 249, + /* 1210 */ 3, 257, 1161, 1168, 1166, 259, 138, 356, 5, 361, + /* 1220 */ 1248, 313, 1831, 174, 1304, 268, 388, 1293, 409, 1724, + /* 1230 */ 557, 416, 424, 423, 425, 1783, 429, 581, 1292, 431, + /* 1240 */ 433, 566, 1310, 167, 1312, 434, 442, 1813, 183, 29, + /* 1250 */ 338, 1374, 1375, 1376, 1377, 1378, 1382, 1383, 1384, 1385, + /* 1260 */ 1845, 445, 446, 185, 95, 1814, 584, 1816, 1817, 580, + /* 1270 */ 1311, 575, 447, 1313, 1891, 1831, 188, 450, 331, 1887, + /* 1280 */ 159, 448, 473, 582, 190, 72, 73, 1813, 1783, 454, + /* 1290 */ 581, 194, 269, 475, 1619, 198, 304, 1615, 118, 200, + /* 1300 */ 1918, 1763, 211, 140, 141, 1617, 1613, 504, 214, 505, + /* 1310 */ 142, 1813, 143, 1845, 511, 1831, 515, 95, 1814, 584, + /* 1320 */ 1816, 1817, 580, 582, 575, 218, 508, 1891, 1783, 524, + /* 1330 */ 581, 331, 1887, 1970, 518, 81, 538, 132, 1762, 1831, + /* 1340 */ 323, 83, 1925, 1734, 521, 1309, 270, 582, 325, 1630, + /* 1350 */ 541, 1813, 1783, 1845, 581, 6, 133, 279, 1814, 584, + /* 1360 */ 1816, 1817, 580, 1932, 575, 534, 525, 536, 537, 234, + /* 1370 */ 238, 330, 550, 544, 535, 533, 532, 1845, 1413, 1831, + /* 1380 */ 126, 95, 1814, 584, 1816, 1817, 580, 582, 575, 1922, + /* 1390 */ 245, 1891, 1783, 1931, 581, 331, 1887, 1970, 567, 1308, + /* 1400 */ 564, 248, 48, 332, 1813, 85, 1948, 1907, 586, 1673, + /* 1410 */ 1602, 273, 667, 52, 668, 1913, 243, 1845, 670, 299, + /* 1420 */ 264, 95, 1814, 584, 1816, 1817, 580, 277, 575, 246, + /* 1430 */ 153, 1891, 1831, 1872, 247, 331, 1887, 1970, 275, 286, + /* 1440 */ 582, 296, 295, 1777, 253, 1783, 1910, 581, 1776, 1775, + /* 1450 */ 1953, 65, 1774, 561, 1813, 568, 256, 1973, 258, 66, + /* 1460 */ 558, 1771, 358, 359, 1273, 1274, 170, 363, 1769, 365, + /* 1470 */ 1845, 1813, 366, 367, 285, 1814, 584, 1816, 1817, 580, + /* 1480 */ 1768, 575, 1831, 369, 1767, 371, 1766, 373, 1765, 375, + /* 1490 */ 582, 1251, 1250, 1745, 1744, 1783, 381, 581, 380, 1831, + /* 1500 */ 1957, 1743, 1742, 1220, 1717, 1716, 1715, 582, 129, 1714, + /* 1510 */ 1713, 1712, 1783, 164, 581, 69, 1711, 1954, 1710, 1709, + /* 1520 */ 1845, 1813, 398, 1706, 96, 1814, 584, 1816, 1817, 580, + /* 1530 */ 1708, 575, 1707, 399, 1891, 401, 1705, 1845, 1890, 1887, + /* 1540 */ 1704, 96, 1814, 584, 1816, 1817, 580, 1703, 575, 1831, + /* 1550 */ 1702, 1891, 1701, 1700, 1699, 569, 1887, 582, 1698, 1697, + /* 1560 */ 1696, 1695, 1783, 1694, 581, 1693, 1692, 130, 1691, 1813, + /* 1570 */ 1690, 1689, 1688, 1687, 1686, 1685, 1684, 1222, 1683, 1813, + /* 1580 */ 178, 1550, 179, 1548, 1516, 181, 182, 1845, 1682, 1551, + /* 1590 */ 120, 96, 1814, 584, 1816, 1817, 580, 1831, 575, 157, + /* 1600 */ 1025, 1891, 531, 438, 1024, 582, 1888, 1831, 440, 1515, + /* 1610 */ 1783, 1758, 581, 1752, 1741, 582, 121, 189, 187, 1740, + /* 1620 */ 1783, 1726, 581, 1608, 1547, 1545, 1055, 1543, 455, 456, + /* 1630 */ 1813, 459, 460, 457, 461, 1845, 1541, 463, 464, 294, + /* 1640 */ 1814, 584, 1816, 1817, 580, 1845, 575, 465, 1539, 289, + /* 1650 */ 1814, 584, 1816, 1817, 580, 467, 575, 468, 1831, 1171, + /* 1660 */ 1609, 469, 1528, 1527, 1512, 1610, 582, 1172, 1097, 50, + /* 1670 */ 1096, 1783, 639, 581, 641, 1093, 1092, 1091, 1537, 319, + /* 1680 */ 1532, 1530, 493, 320, 1511, 495, 1510, 549, 490, 321, + /* 1690 */ 1813, 497, 1509, 499, 97, 1757, 1845, 197, 1751, 56, + /* 1700 */ 149, 1814, 584, 1816, 1817, 580, 144, 575, 1813, 506, + /* 1710 */ 1257, 507, 1739, 1737, 1738, 1736, 1735, 1265, 1831, 15, + /* 1720 */ 519, 1733, 220, 226, 1725, 231, 579, 79, 215, 80, + /* 1730 */ 45, 1783, 41, 581, 47, 82, 1831, 23, 233, 16, + /* 1740 */ 1453, 339, 241, 235, 582, 1972, 87, 522, 1435, 1783, + /* 1750 */ 242, 581, 1437, 1804, 25, 1465, 1845, 228, 1813, 237, + /* 1760 */ 293, 1814, 584, 1816, 1817, 580, 322, 575, 251, 1864, + /* 1770 */ 512, 1430, 46, 1803, 1845, 17, 151, 154, 294, 1814, + /* 1780 */ 584, 1816, 1817, 580, 1813, 575, 1831, 240, 18, 24, + /* 1790 */ 86, 341, 1464, 333, 582, 1410, 1409, 1470, 1469, 1783, + /* 1800 */ 1459, 581, 1468, 334, 10, 1297, 155, 19, 1813, 1327, + /* 1810 */ 1372, 1848, 1831, 587, 585, 168, 583, 1347, 574, 31, + /* 1820 */ 582, 342, 1345, 12, 1845, 1783, 1344, 581, 294, 1814, + /* 1830 */ 584, 1816, 1817, 580, 1813, 575, 1831, 13, 1158, 20, + /* 1840 */ 21, 589, 591, 1155, 582, 592, 594, 1152, 595, 1783, + /* 1850 */ 1845, 581, 597, 600, 280, 1814, 584, 1816, 1817, 580, + /* 1860 */ 1146, 575, 1831, 598, 1135, 1144, 601, 1150, 1149, 88, + /* 1870 */ 582, 1148, 1167, 1147, 1845, 1783, 262, 581, 281, 1814, + /* 1880 */ 584, 1816, 1817, 580, 607, 575, 89, 62, 1163, 1053, + /* 1890 */ 616, 1088, 1813, 1087, 1086, 1085, 1084, 1083, 1081, 1079, + /* 1900 */ 1845, 628, 1078, 1077, 288, 1814, 584, 1816, 1817, 580, + /* 1910 */ 1813, 575, 553, 1104, 263, 1075, 1074, 1073, 1072, 1071, + /* 1920 */ 1831, 1070, 1069, 1068, 1099, 1101, 1065, 1064, 582, 1061, + /* 1930 */ 1059, 1060, 1544, 1783, 1058, 581, 649, 650, 1831, 651, + /* 1940 */ 127, 1542, 653, 655, 1540, 654, 582, 657, 659, 658, + /* 1950 */ 1538, 1783, 661, 581, 663, 1526, 1508, 662, 1845, 665, + /* 1960 */ 558, 1015, 290, 1814, 584, 1816, 1817, 580, 266, 575, + /* 1970 */ 669, 1813, 1483, 1283, 274, 672, 1845, 673, 125, 1483, + /* 1980 */ 282, 1814, 584, 1816, 1817, 580, 1483, 575, 1483, 1483, + /* 1990 */ 1483, 1483, 1483, 250, 1899, 552, 1483, 551, 1483, 1831, + /* 2000 */ 1957, 1483, 1483, 1483, 1483, 1483, 1483, 582, 1483, 1483, + /* 2010 */ 1483, 1483, 1783, 164, 581, 1483, 1483, 1954, 1483, 1483, + /* 2020 */ 1483, 1813, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2030 */ 1483, 1813, 1483, 1483, 1483, 1483, 1483, 1845, 1483, 1483, + /* 2040 */ 1483, 291, 1814, 584, 1816, 1817, 580, 1483, 575, 1831, + /* 2050 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 582, 1483, 1831, + /* 2060 */ 1483, 1483, 1783, 1483, 581, 1483, 1483, 582, 1483, 1483, + /* 2070 */ 1483, 1813, 1783, 1483, 581, 1483, 1483, 1483, 1483, 1483, + /* 2080 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1845, 1483, 1483, + /* 2090 */ 1483, 283, 1814, 584, 1816, 1817, 580, 1845, 575, 1831, + /* 2100 */ 1483, 292, 1814, 584, 1816, 1817, 580, 582, 575, 1483, + /* 2110 */ 1483, 1483, 1783, 1483, 581, 1483, 1483, 1483, 1483, 1483, + /* 2120 */ 1483, 1813, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2130 */ 1483, 1483, 1483, 1813, 1483, 1483, 1483, 1845, 1483, 1483, + /* 2140 */ 1483, 284, 1814, 584, 1816, 1817, 580, 1813, 575, 1831, + /* 2150 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 582, 1483, 1483, + /* 2160 */ 1483, 1831, 1783, 1483, 581, 1483, 1483, 1483, 1483, 582, + /* 2170 */ 1483, 1483, 1483, 1483, 1783, 1831, 581, 1483, 1483, 1483, + /* 2180 */ 1483, 1483, 1483, 582, 1483, 1483, 1483, 1845, 1783, 1483, + /* 2190 */ 581, 297, 1814, 584, 1816, 1817, 580, 1483, 575, 1845, + /* 2200 */ 1483, 1483, 1483, 298, 1814, 584, 1816, 1817, 580, 1483, + /* 2210 */ 575, 1483, 1483, 1845, 1483, 1483, 1483, 1825, 1814, 584, + /* 2220 */ 1816, 1817, 580, 1813, 575, 1483, 1483, 1483, 1483, 1483, + /* 2230 */ 1483, 1483, 1483, 1813, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2240 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2250 */ 1483, 1831, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 582, + /* 2260 */ 1483, 1831, 1483, 1483, 1783, 1483, 581, 1483, 1483, 582, + /* 2270 */ 1483, 1483, 1483, 1483, 1783, 1483, 581, 1483, 1483, 1483, + /* 2280 */ 1483, 1483, 1483, 1483, 1813, 1483, 1483, 1483, 1483, 1845, + /* 2290 */ 1483, 1483, 1483, 1824, 1814, 584, 1816, 1817, 580, 1845, + /* 2300 */ 575, 1483, 1483, 1823, 1814, 584, 1816, 1817, 580, 1483, + /* 2310 */ 575, 1483, 1831, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2320 */ 582, 1483, 1483, 1483, 1483, 1783, 1483, 581, 1483, 1483, + /* 2330 */ 1483, 1483, 1483, 1483, 1813, 1483, 1483, 1483, 1483, 1483, + /* 2340 */ 1483, 1483, 1483, 1483, 1813, 1483, 1483, 1483, 1483, 1483, + /* 2350 */ 1845, 1483, 1483, 1483, 310, 1814, 584, 1816, 1817, 580, + /* 2360 */ 1483, 575, 1831, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2370 */ 582, 1483, 1831, 1483, 1483, 1783, 1483, 581, 1483, 1483, + /* 2380 */ 582, 1483, 1483, 1483, 1813, 1783, 1483, 581, 1483, 1483, + /* 2390 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2400 */ 1845, 1483, 1483, 1483, 309, 1814, 584, 1816, 1817, 580, + /* 2410 */ 1845, 575, 1831, 1483, 311, 1814, 584, 1816, 1817, 580, + /* 2420 */ 582, 575, 1483, 1483, 1483, 1783, 1483, 581, 1483, 1483, + /* 2430 */ 1483, 1483, 1483, 1483, 1813, 1483, 1483, 1483, 1483, 1483, + /* 2440 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2450 */ 1845, 1483, 1483, 1483, 308, 1814, 584, 1816, 1817, 580, + /* 2460 */ 1483, 575, 1831, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2470 */ 582, 1483, 1483, 1483, 1483, 1783, 1483, 581, 1483, 1483, + /* 2480 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2490 */ 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, 1483, + /* 2500 */ 1845, 1483, 1483, 1483, 287, 1814, 584, 1816, 1817, 580, + /* 2510 */ 1483, 575, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 312, 404, 314, 315, 337, 312, 316, 314, 315, 351, - /* 10 */ 389, 390, 12, 13, 417, 328, 358, 327, 421, 361, - /* 20 */ 20, 0, 22, 336, 334, 12, 13, 14, 15, 16, - /* 30 */ 308, 20, 345, 33, 344, 35, 347, 20, 20, 350, - /* 40 */ 351, 364, 21, 338, 316, 24, 25, 26, 27, 28, - /* 50 */ 29, 30, 31, 32, 349, 327, 56, 313, 336, 4, - /* 60 */ 316, 61, 334, 14, 15, 16, 344, 307, 68, 309, - /* 70 */ 60, 349, 344, 351, 12, 13, 14, 338, 373, 374, - /* 80 */ 375, 404, 20, 0, 22, 336, 56, 20, 349, 89, - /* 90 */ 385, 4, 336, 344, 417, 33, 374, 35, 421, 343, - /* 100 */ 378, 379, 380, 381, 382, 383, 89, 385, 352, 335, - /* 110 */ 388, 111, 373, 374, 392, 393, 394, 351, 56, 89, - /* 120 */ 404, 91, 348, 61, 385, 125, 126, 361, 406, 380, - /* 130 */ 68, 44, 45, 417, 8, 9, 414, 421, 12, 13, - /* 140 */ 14, 15, 16, 20, 89, 89, 63, 64, 65, 66, - /* 150 */ 67, 89, 69, 70, 71, 72, 73, 74, 75, 76, - /* 160 */ 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, - /* 170 */ 170, 21, 172, 111, 24, 25, 26, 27, 28, 29, - /* 180 */ 30, 31, 32, 313, 338, 20, 316, 125, 126, 120, - /* 190 */ 121, 1, 2, 193, 194, 349, 196, 197, 198, 199, - /* 200 */ 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, - /* 210 */ 210, 211, 212, 213, 0, 21, 90, 8, 9, 373, - /* 220 */ 374, 12, 13, 14, 15, 16, 316, 227, 34, 321, - /* 230 */ 36, 385, 170, 325, 172, 22, 121, 327, 24, 25, - /* 240 */ 26, 27, 28, 29, 30, 31, 32, 178, 35, 89, - /* 250 */ 181, 360, 43, 362, 344, 193, 194, 227, 196, 197, - /* 260 */ 198, 199, 200, 201, 202, 203, 204, 205, 206, 207, - /* 270 */ 208, 209, 210, 211, 212, 213, 12, 13, 350, 351, - /* 280 */ 90, 68, 227, 227, 20, 20, 22, 60, 20, 89, - /* 290 */ 125, 126, 177, 178, 308, 308, 181, 33, 193, 35, - /* 300 */ 102, 103, 104, 105, 106, 107, 108, 109, 110, 111, - /* 310 */ 112, 328, 114, 115, 116, 117, 118, 119, 20, 336, - /* 320 */ 56, 0, 336, 56, 111, 61, 322, 323, 345, 169, - /* 330 */ 344, 171, 68, 336, 404, 349, 349, 351, 12, 13, - /* 340 */ 343, 236, 237, 238, 239, 240, 20, 417, 22, 352, - /* 350 */ 0, 421, 318, 89, 89, 88, 336, 89, 91, 33, - /* 360 */ 374, 35, 68, 343, 378, 379, 380, 381, 382, 383, - /* 370 */ 100, 385, 352, 339, 388, 111, 328, 337, 392, 393, - /* 380 */ 394, 20, 56, 170, 336, 172, 20, 227, 22, 125, - /* 390 */ 126, 8, 9, 345, 68, 12, 13, 14, 15, 16, - /* 400 */ 414, 404, 316, 12, 13, 14, 193, 194, 308, 308, - /* 410 */ 101, 20, 308, 22, 417, 89, 50, 14, 421, 322, - /* 420 */ 323, 35, 101, 20, 33, 68, 35, 227, 316, 120, - /* 430 */ 121, 122, 123, 124, 170, 344, 172, 111, 308, 353, - /* 440 */ 336, 120, 121, 122, 123, 124, 355, 56, 344, 349, - /* 450 */ 349, 125, 126, 349, 68, 351, 344, 193, 194, 68, - /* 460 */ 196, 197, 198, 199, 200, 201, 202, 203, 204, 205, - /* 470 */ 206, 207, 208, 209, 210, 211, 212, 213, 374, 349, - /* 480 */ 89, 337, 378, 379, 380, 381, 382, 383, 384, 385, - /* 490 */ 386, 387, 227, 320, 382, 227, 170, 120, 172, 8, - /* 500 */ 9, 0, 111, 12, 13, 14, 15, 16, 396, 397, - /* 510 */ 398, 399, 337, 401, 308, 342, 125, 126, 157, 193, - /* 520 */ 194, 0, 196, 197, 198, 199, 200, 201, 202, 203, - /* 530 */ 204, 205, 206, 207, 208, 209, 210, 211, 212, 213, - /* 540 */ 360, 316, 362, 8, 9, 162, 318, 12, 13, 14, - /* 550 */ 15, 16, 327, 227, 37, 349, 179, 180, 324, 331, - /* 560 */ 326, 170, 376, 172, 2, 64, 65, 339, 305, 344, - /* 570 */ 8, 9, 71, 0, 12, 13, 14, 15, 16, 337, - /* 580 */ 308, 90, 81, 82, 193, 194, 400, 196, 197, 198, - /* 590 */ 199, 200, 201, 202, 203, 204, 205, 206, 207, 208, - /* 600 */ 209, 210, 211, 212, 213, 12, 13, 101, 336, 92, - /* 610 */ 336, 94, 95, 20, 97, 22, 344, 152, 101, 345, - /* 620 */ 316, 349, 101, 351, 2, 90, 33, 364, 35, 123, - /* 630 */ 8, 9, 249, 60, 12, 13, 14, 15, 16, 155, - /* 640 */ 123, 120, 121, 122, 123, 124, 374, 344, 344, 56, - /* 650 */ 378, 379, 380, 381, 382, 383, 337, 385, 355, 320, - /* 660 */ 388, 68, 316, 346, 392, 393, 349, 404, 364, 364, - /* 670 */ 8, 9, 333, 327, 12, 13, 14, 15, 16, 214, - /* 680 */ 417, 342, 89, 308, 421, 20, 382, 22, 39, 224, - /* 690 */ 344, 8, 9, 308, 325, 12, 13, 14, 15, 16, - /* 700 */ 35, 397, 398, 399, 111, 401, 8, 9, 404, 404, - /* 710 */ 12, 13, 14, 15, 16, 50, 232, 233, 125, 126, - /* 720 */ 316, 417, 417, 61, 349, 421, 421, 43, 4, 14, - /* 730 */ 316, 327, 12, 13, 349, 20, 308, 18, 376, 20, - /* 740 */ 20, 327, 22, 19, 61, 316, 27, 337, 344, 30, - /* 750 */ 316, 316, 90, 33, 309, 35, 327, 33, 344, 3, - /* 760 */ 98, 327, 400, 170, 165, 172, 47, 308, 49, 308, - /* 770 */ 51, 47, 424, 344, 90, 51, 56, 349, 344, 344, - /* 780 */ 56, 98, 308, 184, 185, 77, 193, 194, 68, 196, - /* 790 */ 197, 198, 199, 200, 201, 202, 203, 204, 205, 206, - /* 800 */ 207, 208, 209, 210, 211, 212, 213, 88, 349, 89, - /* 810 */ 349, 4, 88, 316, 152, 91, 376, 382, 368, 100, - /* 820 */ 20, 8, 9, 349, 327, 12, 13, 14, 15, 16, - /* 830 */ 308, 111, 397, 398, 399, 152, 401, 129, 130, 0, - /* 840 */ 400, 344, 14, 44, 45, 125, 126, 128, 20, 0, - /* 850 */ 131, 132, 133, 134, 135, 136, 137, 138, 139, 140, - /* 860 */ 141, 142, 143, 144, 145, 146, 147, 148, 316, 150, - /* 870 */ 151, 349, 336, 13, 308, 226, 214, 215, 216, 217, - /* 880 */ 218, 219, 220, 221, 222, 223, 224, 48, 352, 317, - /* 890 */ 170, 346, 172, 20, 349, 35, 344, 214, 215, 216, - /* 900 */ 217, 218, 219, 220, 221, 222, 223, 224, 329, 60, - /* 910 */ 415, 332, 197, 193, 194, 349, 196, 197, 198, 199, - /* 920 */ 200, 201, 202, 203, 204, 205, 206, 207, 208, 209, - /* 930 */ 210, 211, 212, 213, 382, 18, 308, 308, 43, 364, - /* 940 */ 23, 364, 225, 226, 0, 64, 65, 0, 338, 397, - /* 950 */ 398, 399, 71, 401, 37, 38, 336, 157, 41, 349, - /* 960 */ 316, 316, 81, 82, 308, 336, 42, 43, 411, 22, - /* 970 */ 157, 327, 327, 344, 57, 58, 59, 349, 349, 404, - /* 980 */ 351, 404, 315, 373, 374, 375, 308, 308, 344, 344, - /* 990 */ 316, 47, 417, 364, 417, 385, 421, 56, 421, 364, - /* 1000 */ 244, 327, 308, 374, 308, 349, 89, 378, 379, 380, - /* 1010 */ 381, 382, 383, 316, 385, 336, 43, 388, 344, 12, - /* 1020 */ 13, 392, 393, 344, 327, 197, 316, 349, 349, 22, - /* 1030 */ 351, 93, 91, 404, 96, 228, 348, 327, 316, 404, - /* 1040 */ 33, 344, 35, 349, 127, 349, 417, 0, 308, 327, - /* 1050 */ 421, 317, 417, 374, 344, 308, 421, 378, 379, 380, - /* 1060 */ 381, 382, 383, 56, 385, 316, 344, 388, 43, 22, - /* 1070 */ 197, 392, 393, 394, 316, 68, 327, 157, 158, 162, - /* 1080 */ 163, 164, 403, 336, 167, 327, 43, 93, 316, 349, - /* 1090 */ 96, 344, 93, 344, 93, 96, 349, 96, 351, 327, - /* 1100 */ 183, 0, 344, 186, 377, 188, 189, 190, 191, 192, - /* 1110 */ 35, 364, 43, 316, 308, 90, 344, 35, 111, 61, - /* 1120 */ 316, 374, 43, 22, 327, 378, 379, 380, 381, 382, - /* 1130 */ 383, 327, 385, 90, 43, 388, 43, 402, 19, 392, - /* 1140 */ 393, 344, 336, 248, 227, 43, 43, 341, 344, 43, - /* 1150 */ 344, 404, 33, 89, 308, 349, 13, 351, 43, 90, - /* 1160 */ 46, 1, 2, 99, 417, 395, 47, 405, 421, 90, - /* 1170 */ 418, 52, 53, 54, 55, 56, 43, 170, 35, 172, - /* 1180 */ 374, 90, 336, 90, 378, 379, 380, 381, 382, 383, - /* 1190 */ 344, 385, 90, 90, 308, 349, 90, 351, 125, 126, - /* 1200 */ 193, 194, 418, 89, 418, 90, 43, 88, 43, 229, - /* 1210 */ 91, 35, 205, 206, 207, 208, 209, 210, 211, 246, - /* 1220 */ 374, 47, 336, 90, 378, 379, 380, 381, 382, 383, - /* 1230 */ 344, 385, 366, 372, 388, 349, 43, 351, 392, 393, - /* 1240 */ 394, 43, 316, 124, 68, 371, 168, 172, 42, 403, - /* 1250 */ 356, 193, 20, 90, 172, 90, 316, 356, 316, 152, - /* 1260 */ 374, 354, 354, 308, 378, 379, 380, 381, 382, 383, - /* 1270 */ 344, 385, 316, 316, 388, 156, 316, 20, 392, 393, - /* 1280 */ 394, 310, 310, 90, 20, 370, 320, 351, 90, 403, - /* 1290 */ 364, 336, 20, 174, 320, 176, 363, 20, 320, 344, - /* 1300 */ 365, 320, 363, 320, 349, 320, 351, 316, 382, 320, - /* 1310 */ 310, 336, 336, 336, 316, 336, 310, 308, 336, 364, - /* 1320 */ 336, 370, 336, 397, 398, 399, 336, 401, 336, 374, - /* 1330 */ 404, 336, 336, 378, 379, 380, 381, 382, 383, 318, - /* 1340 */ 385, 175, 349, 417, 318, 336, 316, 421, 351, 316, - /* 1350 */ 318, 363, 234, 344, 349, 154, 369, 349, 349, 404, - /* 1360 */ 351, 359, 349, 349, 349, 20, 318, 308, 357, 359, - /* 1370 */ 332, 318, 417, 364, 235, 344, 421, 349, 410, 359, - /* 1380 */ 377, 359, 241, 374, 308, 349, 349, 378, 379, 380, - /* 1390 */ 381, 382, 383, 349, 385, 336, 161, 243, 349, 242, - /* 1400 */ 230, 372, 20, 344, 226, 344, 247, 245, 349, 89, - /* 1410 */ 351, 376, 336, 404, 410, 89, 349, 250, 326, 36, - /* 1420 */ 344, 340, 318, 311, 412, 349, 417, 351, 410, 413, - /* 1430 */ 421, 407, 409, 374, 308, 408, 419, 378, 379, 380, - /* 1440 */ 381, 382, 383, 391, 385, 316, 310, 388, 362, 319, - /* 1450 */ 374, 392, 393, 419, 378, 379, 380, 381, 382, 383, - /* 1460 */ 308, 385, 336, 367, 425, 420, 420, 330, 330, 419, - /* 1470 */ 344, 420, 330, 0, 306, 349, 0, 351, 177, 0, - /* 1480 */ 0, 42, 0, 35, 187, 35, 35, 35, 336, 187, - /* 1490 */ 0, 35, 35, 341, 187, 0, 344, 187, 422, 423, - /* 1500 */ 374, 349, 0, 351, 378, 379, 380, 381, 382, 383, - /* 1510 */ 308, 385, 35, 0, 388, 22, 0, 170, 35, 393, - /* 1520 */ 172, 0, 308, 166, 165, 0, 374, 0, 0, 0, - /* 1530 */ 378, 379, 380, 381, 382, 383, 308, 385, 336, 0, - /* 1540 */ 46, 42, 0, 0, 149, 0, 344, 144, 0, 35, - /* 1550 */ 336, 349, 0, 351, 0, 0, 0, 144, 344, 0, - /* 1560 */ 0, 0, 0, 349, 336, 351, 0, 0, 0, 0, - /* 1570 */ 0, 0, 344, 0, 0, 0, 374, 349, 0, 351, - /* 1580 */ 378, 379, 380, 381, 382, 383, 42, 385, 374, 308, - /* 1590 */ 0, 0, 378, 379, 380, 381, 382, 383, 0, 385, - /* 1600 */ 0, 0, 374, 22, 0, 0, 378, 379, 380, 381, - /* 1610 */ 382, 383, 0, 385, 0, 387, 0, 336, 416, 0, - /* 1620 */ 56, 0, 341, 39, 42, 344, 33, 0, 56, 0, - /* 1630 */ 349, 43, 351, 46, 14, 46, 14, 423, 0, 40, - /* 1640 */ 47, 39, 308, 35, 47, 52, 53, 54, 55, 56, - /* 1650 */ 0, 0, 0, 161, 308, 374, 39, 0, 0, 378, - /* 1660 */ 379, 380, 381, 382, 383, 0, 385, 0, 308, 62, - /* 1670 */ 336, 0, 39, 0, 35, 341, 39, 47, 344, 0, - /* 1680 */ 35, 88, 336, 349, 91, 351, 39, 47, 0, 35, - /* 1690 */ 344, 47, 39, 0, 0, 349, 336, 351, 0, 0, - /* 1700 */ 22, 35, 96, 0, 344, 35, 22, 98, 374, 349, - /* 1710 */ 0, 351, 378, 379, 380, 381, 382, 383, 43, 385, - /* 1720 */ 374, 35, 308, 43, 378, 379, 380, 381, 382, 383, - /* 1730 */ 35, 385, 0, 22, 374, 22, 308, 0, 378, 379, - /* 1740 */ 380, 381, 382, 383, 22, 385, 153, 154, 308, 156, - /* 1750 */ 336, 49, 35, 160, 0, 35, 0, 0, 344, 35, - /* 1760 */ 22, 20, 0, 349, 336, 351, 35, 157, 0, 176, - /* 1770 */ 22, 0, 344, 0, 159, 157, 336, 349, 154, 351, - /* 1780 */ 0, 0, 157, 0, 344, 89, 173, 90, 374, 349, - /* 1790 */ 35, 351, 378, 379, 380, 381, 382, 383, 0, 385, - /* 1800 */ 0, 89, 374, 155, 89, 308, 378, 379, 380, 381, - /* 1810 */ 382, 383, 153, 385, 374, 308, 182, 39, 378, 379, - /* 1820 */ 380, 381, 382, 383, 89, 385, 99, 43, 46, 89, - /* 1830 */ 43, 308, 90, 336, 89, 43, 90, 90, 89, 46, - /* 1840 */ 90, 344, 231, 336, 43, 46, 349, 89, 351, 89, - /* 1850 */ 89, 344, 43, 90, 89, 308, 349, 231, 351, 336, - /* 1860 */ 90, 46, 90, 46, 46, 90, 225, 344, 43, 90, - /* 1870 */ 35, 374, 349, 231, 351, 378, 379, 380, 381, 382, - /* 1880 */ 383, 374, 385, 336, 35, 378, 379, 380, 381, 382, - /* 1890 */ 383, 344, 385, 35, 35, 35, 349, 374, 351, 35, - /* 1900 */ 2, 378, 379, 380, 381, 382, 383, 22, 385, 193, - /* 1910 */ 43, 308, 89, 22, 90, 89, 46, 90, 89, 100, - /* 1920 */ 90, 374, 89, 89, 46, 378, 379, 380, 381, 382, - /* 1930 */ 383, 308, 385, 89, 35, 90, 35, 89, 195, 336, - /* 1940 */ 90, 35, 89, 35, 90, 89, 35, 344, 35, 113, - /* 1950 */ 89, 308, 349, 90, 351, 90, 113, 89, 22, 336, - /* 1960 */ 113, 89, 35, 101, 113, 89, 89, 344, 43, 22, - /* 1970 */ 61, 308, 349, 35, 351, 35, 62, 374, 35, 336, - /* 1980 */ 35, 378, 379, 380, 381, 382, 383, 344, 385, 35, - /* 1990 */ 35, 308, 349, 35, 351, 43, 68, 374, 35, 336, - /* 2000 */ 35, 378, 379, 380, 381, 382, 383, 344, 385, 87, - /* 2010 */ 22, 35, 349, 22, 351, 35, 35, 374, 68, 336, - /* 2020 */ 35, 378, 379, 380, 381, 382, 383, 344, 385, 35, - /* 2030 */ 35, 308, 349, 35, 351, 35, 22, 374, 35, 0, - /* 2040 */ 35, 378, 379, 380, 381, 382, 383, 308, 385, 47, - /* 2050 */ 39, 0, 35, 47, 39, 0, 35, 374, 39, 336, - /* 2060 */ 47, 378, 379, 380, 381, 382, 383, 344, 385, 0, - /* 2070 */ 35, 47, 349, 39, 351, 336, 0, 35, 35, 0, - /* 2080 */ 22, 21, 21, 344, 22, 22, 20, 426, 349, 426, - /* 2090 */ 351, 426, 426, 426, 426, 426, 426, 374, 308, 426, - /* 2100 */ 426, 378, 379, 380, 381, 382, 383, 426, 385, 426, - /* 2110 */ 426, 426, 426, 374, 308, 426, 426, 378, 379, 380, - /* 2120 */ 381, 382, 383, 426, 385, 426, 336, 426, 426, 426, - /* 2130 */ 426, 426, 426, 426, 344, 426, 426, 426, 426, 349, - /* 2140 */ 426, 351, 336, 426, 426, 426, 426, 426, 426, 426, - /* 2150 */ 344, 426, 426, 426, 426, 349, 426, 351, 426, 426, - /* 2160 */ 426, 426, 426, 426, 374, 308, 426, 426, 378, 379, - /* 2170 */ 380, 381, 382, 383, 426, 385, 426, 426, 426, 426, - /* 2180 */ 374, 308, 426, 426, 378, 379, 380, 381, 382, 383, - /* 2190 */ 426, 385, 426, 336, 426, 426, 426, 426, 426, 426, - /* 2200 */ 426, 344, 426, 426, 426, 426, 349, 426, 351, 336, - /* 2210 */ 426, 426, 426, 426, 426, 426, 426, 344, 426, 426, - /* 2220 */ 426, 426, 349, 426, 351, 426, 426, 426, 426, 426, - /* 2230 */ 426, 374, 426, 426, 426, 378, 379, 380, 381, 382, - /* 2240 */ 383, 426, 385, 426, 426, 426, 426, 374, 426, 426, - /* 2250 */ 426, 378, 379, 380, 381, 382, 383, 426, 385, 426, - /* 2260 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2270 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2280 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2290 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2300 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2310 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2320 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2330 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2340 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2350 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2360 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2370 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2380 */ 426, 426, 426, 426, 426, 426, 426, 426, 426, 426, - /* 2390 */ 426, 426, + /* 0 */ 316, 312, 318, 319, 316, 351, 318, 319, 354, 355, + /* 10 */ 8, 9, 12, 13, 12, 13, 14, 15, 16, 342, + /* 20 */ 20, 320, 22, 342, 12, 13, 14, 15, 16, 340, + /* 30 */ 353, 340, 331, 33, 353, 35, 0, 348, 347, 338, + /* 40 */ 393, 394, 353, 0, 355, 320, 311, 356, 313, 348, + /* 50 */ 20, 312, 22, 342, 377, 378, 56, 368, 377, 378, + /* 60 */ 379, 61, 354, 355, 353, 35, 389, 378, 68, 0, + /* 70 */ 389, 382, 383, 384, 385, 386, 387, 0, 389, 340, + /* 80 */ 50, 392, 357, 47, 342, 396, 397, 348, 377, 378, + /* 90 */ 379, 48, 353, 93, 355, 353, 94, 408, 21, 408, + /* 100 */ 389, 24, 25, 26, 27, 28, 29, 30, 31, 32, + /* 110 */ 421, 340, 421, 20, 425, 115, 425, 378, 347, 377, + /* 120 */ 378, 382, 383, 384, 385, 386, 387, 356, 389, 129, + /* 130 */ 130, 389, 63, 64, 65, 66, 67, 0, 69, 70, + /* 140 */ 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, + /* 150 */ 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, + /* 160 */ 14, 15, 16, 8, 9, 426, 427, 12, 13, 14, + /* 170 */ 15, 16, 20, 20, 20, 175, 20, 177, 106, 107, + /* 180 */ 108, 109, 110, 111, 112, 113, 114, 115, 116, 93, + /* 190 */ 118, 119, 120, 121, 122, 123, 60, 60, 198, 199, + /* 200 */ 320, 201, 202, 203, 204, 205, 206, 207, 208, 209, + /* 210 */ 210, 211, 212, 213, 214, 215, 216, 217, 218, 12, + /* 220 */ 13, 14, 129, 130, 20, 340, 21, 20, 348, 22, + /* 230 */ 8, 9, 232, 348, 12, 13, 14, 15, 16, 34, + /* 240 */ 33, 36, 35, 64, 65, 93, 93, 93, 368, 21, + /* 250 */ 71, 368, 24, 25, 26, 27, 28, 29, 30, 31, + /* 260 */ 32, 82, 83, 56, 4, 43, 386, 88, 61, 384, + /* 270 */ 174, 322, 176, 4, 312, 68, 12, 13, 320, 19, + /* 280 */ 4, 401, 402, 403, 20, 405, 22, 105, 408, 331, + /* 290 */ 312, 408, 343, 33, 39, 332, 338, 33, 312, 35, + /* 300 */ 93, 421, 340, 340, 421, 425, 348, 47, 425, 127, + /* 310 */ 348, 51, 349, 44, 45, 353, 56, 355, 340, 325, + /* 320 */ 56, 408, 115, 329, 317, 61, 348, 320, 232, 198, + /* 330 */ 368, 353, 68, 355, 421, 78, 129, 130, 425, 353, + /* 340 */ 378, 43, 56, 355, 382, 383, 384, 385, 386, 387, + /* 350 */ 362, 389, 92, 365, 392, 95, 378, 93, 396, 397, + /* 360 */ 382, 383, 384, 385, 386, 387, 388, 389, 390, 391, + /* 370 */ 408, 0, 241, 242, 243, 244, 245, 124, 92, 115, + /* 380 */ 340, 95, 175, 421, 177, 232, 232, 425, 312, 349, + /* 390 */ 133, 134, 94, 129, 130, 24, 25, 26, 27, 28, + /* 400 */ 29, 30, 31, 32, 93, 198, 199, 35, 201, 202, + /* 410 */ 203, 204, 205, 206, 207, 208, 209, 210, 211, 212, + /* 420 */ 213, 214, 215, 216, 217, 218, 8, 9, 340, 353, + /* 430 */ 12, 13, 14, 15, 16, 347, 2, 184, 185, 175, + /* 440 */ 68, 177, 8, 9, 356, 2, 12, 13, 14, 15, + /* 450 */ 16, 8, 9, 56, 43, 12, 13, 14, 15, 16, + /* 460 */ 312, 37, 198, 199, 160, 201, 202, 203, 204, 205, + /* 470 */ 206, 207, 208, 209, 210, 211, 212, 213, 214, 215, + /* 480 */ 216, 217, 218, 12, 13, 18, 231, 20, 340, 320, + /* 490 */ 93, 20, 95, 22, 27, 125, 348, 30, 124, 125, + /* 500 */ 331, 353, 408, 355, 33, 20, 35, 14, 368, 233, + /* 510 */ 368, 408, 94, 20, 47, 421, 49, 348, 51, 425, + /* 520 */ 96, 93, 98, 99, 421, 101, 378, 56, 425, 105, + /* 530 */ 382, 383, 384, 385, 386, 387, 348, 389, 20, 68, + /* 540 */ 392, 237, 238, 232, 396, 397, 398, 359, 408, 22, + /* 550 */ 408, 127, 182, 183, 19, 4, 186, 183, 410, 92, + /* 560 */ 186, 421, 35, 421, 93, 425, 418, 425, 33, 320, + /* 570 */ 20, 104, 339, 8, 9, 332, 332, 12, 13, 14, + /* 580 */ 15, 16, 47, 340, 340, 352, 115, 52, 53, 54, + /* 590 */ 55, 56, 349, 349, 309, 68, 20, 348, 22, 132, + /* 600 */ 129, 130, 135, 136, 137, 138, 139, 140, 141, 142, + /* 610 */ 143, 144, 145, 146, 147, 148, 149, 150, 151, 152, + /* 620 */ 312, 154, 155, 156, 60, 14, 50, 92, 14, 232, + /* 630 */ 95, 20, 8, 9, 20, 386, 12, 13, 14, 15, + /* 640 */ 16, 20, 115, 157, 93, 105, 175, 162, 177, 400, + /* 650 */ 401, 402, 403, 368, 405, 368, 1, 2, 326, 327, + /* 660 */ 232, 353, 251, 128, 124, 125, 126, 127, 128, 198, + /* 670 */ 199, 0, 201, 202, 203, 204, 205, 206, 207, 208, + /* 680 */ 209, 210, 211, 212, 213, 214, 215, 216, 217, 218, + /* 690 */ 12, 13, 14, 408, 68, 408, 161, 368, 20, 170, + /* 700 */ 22, 324, 175, 232, 177, 219, 421, 324, 421, 322, + /* 710 */ 425, 33, 425, 35, 179, 229, 181, 312, 189, 190, + /* 720 */ 337, 0, 335, 346, 312, 198, 199, 348, 312, 346, + /* 730 */ 343, 312, 167, 320, 56, 8, 9, 408, 359, 12, + /* 740 */ 13, 14, 15, 16, 331, 320, 68, 12, 13, 94, + /* 750 */ 421, 320, 340, 0, 425, 20, 331, 22, 353, 312, + /* 760 */ 348, 348, 331, 317, 320, 353, 320, 355, 33, 353, + /* 770 */ 35, 93, 353, 348, 320, 326, 327, 355, 320, 348, + /* 780 */ 368, 60, 3, 232, 20, 331, 162, 365, 312, 331, + /* 790 */ 378, 56, 348, 115, 382, 383, 384, 385, 386, 387, + /* 800 */ 353, 389, 348, 68, 12, 13, 348, 129, 130, 104, + /* 810 */ 312, 320, 320, 202, 22, 320, 202, 64, 65, 254, + /* 820 */ 408, 94, 331, 331, 71, 33, 331, 35, 93, 353, + /* 830 */ 386, 320, 312, 421, 312, 82, 83, 425, 68, 348, + /* 840 */ 348, 88, 331, 348, 20, 401, 402, 403, 56, 405, + /* 850 */ 115, 353, 364, 175, 366, 177, 328, 341, 330, 348, + /* 860 */ 68, 93, 8, 9, 129, 130, 12, 13, 14, 15, + /* 870 */ 16, 103, 364, 353, 366, 353, 198, 199, 312, 201, + /* 880 */ 202, 203, 204, 205, 206, 207, 208, 209, 210, 211, + /* 890 */ 212, 213, 214, 215, 216, 217, 218, 320, 320, 320, + /* 900 */ 320, 97, 320, 320, 100, 380, 372, 115, 331, 331, + /* 910 */ 175, 331, 177, 331, 331, 61, 341, 312, 350, 353, + /* 920 */ 312, 353, 312, 44, 45, 348, 348, 348, 348, 404, + /* 930 */ 348, 348, 312, 198, 199, 312, 201, 202, 203, 204, + /* 940 */ 205, 206, 207, 208, 209, 210, 211, 212, 213, 214, + /* 950 */ 215, 216, 217, 218, 12, 13, 102, 320, 353, 320, + /* 960 */ 341, 353, 20, 353, 22, 386, 202, 175, 331, 177, + /* 970 */ 331, 230, 231, 353, 33, 33, 353, 35, 312, 312, + /* 980 */ 401, 402, 403, 0, 405, 348, 162, 348, 47, 312, + /* 990 */ 198, 199, 380, 52, 53, 54, 55, 56, 56, 340, + /* 1000 */ 42, 43, 210, 211, 212, 213, 214, 215, 216, 0, + /* 1010 */ 68, 157, 320, 320, 22, 356, 404, 340, 350, 353, + /* 1020 */ 353, 353, 345, 331, 331, 348, 380, 35, 249, 18, + /* 1030 */ 353, 43, 355, 92, 23, 93, 95, 61, 8, 9, + /* 1040 */ 348, 348, 12, 13, 14, 15, 16, 43, 37, 38, + /* 1050 */ 404, 333, 41, 97, 336, 378, 100, 115, 56, 382, + /* 1060 */ 383, 384, 385, 386, 387, 43, 389, 43, 57, 58, + /* 1070 */ 59, 129, 130, 219, 220, 221, 222, 223, 224, 225, + /* 1080 */ 226, 227, 228, 229, 97, 35, 97, 100, 105, 100, + /* 1090 */ 0, 61, 162, 163, 1, 2, 35, 95, 94, 158, + /* 1100 */ 159, 0, 161, 0, 93, 329, 165, 124, 125, 126, + /* 1110 */ 127, 128, 22, 43, 105, 43, 94, 175, 94, 177, + /* 1120 */ 43, 43, 181, 22, 94, 22, 129, 130, 46, 341, + /* 1130 */ 43, 43, 102, 124, 125, 126, 127, 128, 43, 43, + /* 1140 */ 198, 199, 131, 201, 202, 203, 204, 205, 206, 207, + /* 1150 */ 208, 209, 210, 211, 212, 213, 214, 215, 216, 217, + /* 1160 */ 218, 43, 43, 43, 94, 43, 94, 35, 341, 13, + /* 1170 */ 13, 94, 94, 341, 198, 93, 341, 313, 167, 168, + /* 1180 */ 169, 94, 94, 172, 428, 419, 321, 157, 415, 94, + /* 1190 */ 94, 35, 35, 340, 312, 321, 319, 352, 381, 188, + /* 1200 */ 68, 406, 191, 422, 193, 194, 195, 196, 197, 399, + /* 1210 */ 409, 422, 94, 94, 94, 422, 94, 376, 234, 47, + /* 1220 */ 173, 375, 340, 42, 20, 370, 360, 177, 320, 320, + /* 1230 */ 348, 360, 157, 358, 358, 353, 320, 355, 177, 320, + /* 1240 */ 320, 253, 20, 232, 20, 314, 314, 312, 324, 219, + /* 1250 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, + /* 1260 */ 378, 374, 355, 324, 382, 383, 384, 385, 386, 387, + /* 1270 */ 20, 389, 367, 20, 392, 340, 324, 367, 396, 397, + /* 1280 */ 398, 369, 314, 348, 324, 324, 324, 312, 353, 320, + /* 1290 */ 355, 324, 374, 340, 340, 340, 314, 340, 320, 340, + /* 1300 */ 418, 353, 322, 340, 340, 340, 340, 180, 322, 373, + /* 1310 */ 340, 312, 340, 378, 320, 340, 320, 382, 383, 384, + /* 1320 */ 385, 386, 387, 348, 389, 322, 355, 392, 353, 159, + /* 1330 */ 355, 396, 397, 398, 353, 322, 239, 363, 353, 340, + /* 1340 */ 367, 322, 407, 353, 353, 20, 336, 348, 353, 348, + /* 1350 */ 240, 312, 353, 378, 355, 246, 363, 382, 383, 384, + /* 1360 */ 385, 386, 387, 414, 389, 353, 361, 353, 353, 363, + /* 1370 */ 363, 353, 166, 353, 248, 247, 235, 378, 231, 340, + /* 1380 */ 348, 382, 383, 384, 385, 386, 387, 348, 389, 381, + /* 1390 */ 413, 392, 353, 414, 355, 396, 397, 398, 252, 20, + /* 1400 */ 250, 376, 93, 255, 312, 93, 407, 380, 344, 353, + /* 1410 */ 330, 320, 36, 371, 315, 417, 416, 378, 314, 366, + /* 1420 */ 322, 382, 383, 384, 385, 386, 387, 310, 389, 412, + /* 1430 */ 414, 392, 340, 395, 411, 396, 397, 398, 323, 334, + /* 1440 */ 348, 334, 334, 0, 423, 353, 407, 355, 0, 0, + /* 1450 */ 424, 182, 0, 424, 312, 424, 423, 429, 423, 42, + /* 1460 */ 368, 0, 35, 192, 35, 35, 35, 192, 0, 35, + /* 1470 */ 378, 312, 35, 192, 382, 383, 384, 385, 386, 387, + /* 1480 */ 0, 389, 340, 192, 0, 35, 0, 22, 0, 35, + /* 1490 */ 348, 177, 175, 0, 0, 353, 170, 355, 171, 340, + /* 1500 */ 408, 0, 0, 46, 0, 0, 0, 348, 42, 0, + /* 1510 */ 0, 0, 353, 421, 355, 153, 0, 425, 0, 0, + /* 1520 */ 378, 312, 148, 0, 382, 383, 384, 385, 386, 387, + /* 1530 */ 0, 389, 0, 35, 392, 148, 0, 378, 396, 397, + /* 1540 */ 0, 382, 383, 384, 385, 386, 387, 0, 389, 340, + /* 1550 */ 0, 392, 0, 0, 0, 396, 397, 348, 0, 0, + /* 1560 */ 0, 0, 353, 0, 355, 0, 0, 42, 0, 312, + /* 1570 */ 0, 0, 0, 0, 0, 0, 0, 22, 0, 312, + /* 1580 */ 56, 0, 56, 0, 0, 42, 40, 378, 0, 0, + /* 1590 */ 39, 382, 383, 384, 385, 386, 387, 340, 389, 43, + /* 1600 */ 14, 392, 345, 46, 14, 348, 397, 340, 46, 0, + /* 1610 */ 353, 0, 355, 0, 0, 348, 39, 166, 39, 0, + /* 1620 */ 353, 0, 355, 0, 0, 0, 62, 0, 35, 47, + /* 1630 */ 312, 35, 47, 39, 39, 378, 0, 35, 47, 382, + /* 1640 */ 383, 384, 385, 386, 387, 378, 389, 39, 0, 382, + /* 1650 */ 383, 384, 385, 386, 387, 35, 389, 47, 340, 22, + /* 1660 */ 0, 39, 0, 0, 0, 0, 348, 35, 35, 102, + /* 1670 */ 35, 353, 43, 355, 43, 35, 35, 22, 0, 22, + /* 1680 */ 0, 0, 35, 22, 0, 35, 0, 420, 49, 22, + /* 1690 */ 312, 35, 0, 22, 20, 0, 378, 100, 0, 162, + /* 1700 */ 382, 383, 384, 385, 386, 387, 178, 389, 312, 22, + /* 1710 */ 35, 162, 0, 0, 0, 0, 0, 35, 340, 93, + /* 1720 */ 187, 0, 94, 93, 0, 46, 348, 93, 159, 39, + /* 1730 */ 230, 353, 43, 355, 43, 93, 340, 93, 93, 236, + /* 1740 */ 94, 345, 43, 94, 348, 427, 103, 160, 94, 353, + /* 1750 */ 46, 355, 94, 46, 43, 35, 378, 158, 312, 93, + /* 1760 */ 382, 383, 384, 385, 386, 387, 162, 389, 46, 391, + /* 1770 */ 164, 94, 43, 46, 378, 236, 93, 46, 382, 383, + /* 1780 */ 384, 385, 386, 387, 312, 389, 340, 93, 43, 93, + /* 1790 */ 93, 345, 35, 35, 348, 94, 94, 94, 35, 353, + /* 1800 */ 94, 355, 35, 35, 2, 22, 46, 43, 312, 22, + /* 1810 */ 198, 93, 340, 35, 104, 46, 200, 94, 93, 93, + /* 1820 */ 348, 35, 94, 93, 378, 353, 94, 355, 382, 383, + /* 1830 */ 384, 385, 386, 387, 312, 389, 340, 236, 94, 93, + /* 1840 */ 93, 93, 35, 94, 348, 93, 35, 94, 93, 353, + /* 1850 */ 378, 355, 35, 35, 382, 383, 384, 385, 386, 387, + /* 1860 */ 94, 389, 340, 93, 22, 94, 93, 117, 117, 93, + /* 1870 */ 348, 117, 35, 117, 378, 353, 43, 355, 382, 383, + /* 1880 */ 384, 385, 386, 387, 105, 389, 93, 93, 22, 62, + /* 1890 */ 61, 35, 312, 35, 35, 35, 35, 35, 35, 35, + /* 1900 */ 378, 91, 35, 35, 382, 383, 384, 385, 386, 387, + /* 1910 */ 312, 389, 320, 68, 43, 35, 35, 22, 35, 22, + /* 1920 */ 340, 35, 35, 35, 35, 68, 35, 35, 348, 35, + /* 1930 */ 22, 35, 0, 353, 35, 355, 35, 47, 340, 39, + /* 1940 */ 348, 0, 35, 39, 0, 47, 348, 35, 39, 47, + /* 1950 */ 0, 353, 35, 355, 39, 0, 0, 47, 378, 35, + /* 1960 */ 368, 35, 382, 383, 384, 385, 386, 387, 22, 389, + /* 1970 */ 21, 312, 430, 22, 22, 21, 378, 20, 386, 430, + /* 1980 */ 382, 383, 384, 385, 386, 387, 430, 389, 430, 430, + /* 1990 */ 430, 430, 430, 401, 402, 403, 430, 405, 430, 340, + /* 2000 */ 408, 430, 430, 430, 430, 430, 430, 348, 430, 430, + /* 2010 */ 430, 430, 353, 421, 355, 430, 430, 425, 430, 430, + /* 2020 */ 430, 312, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2030 */ 430, 312, 430, 430, 430, 430, 430, 378, 430, 430, + /* 2040 */ 430, 382, 383, 384, 385, 386, 387, 430, 389, 340, + /* 2050 */ 430, 430, 430, 430, 430, 430, 430, 348, 430, 340, + /* 2060 */ 430, 430, 353, 430, 355, 430, 430, 348, 430, 430, + /* 2070 */ 430, 312, 353, 430, 355, 430, 430, 430, 430, 430, + /* 2080 */ 430, 430, 430, 430, 430, 430, 430, 378, 430, 430, + /* 2090 */ 430, 382, 383, 384, 385, 386, 387, 378, 389, 340, + /* 2100 */ 430, 382, 383, 384, 385, 386, 387, 348, 389, 430, + /* 2110 */ 430, 430, 353, 430, 355, 430, 430, 430, 430, 430, + /* 2120 */ 430, 312, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2130 */ 430, 430, 430, 312, 430, 430, 430, 378, 430, 430, + /* 2140 */ 430, 382, 383, 384, 385, 386, 387, 312, 389, 340, + /* 2150 */ 430, 430, 430, 430, 430, 430, 430, 348, 430, 430, + /* 2160 */ 430, 340, 353, 430, 355, 430, 430, 430, 430, 348, + /* 2170 */ 430, 430, 430, 430, 353, 340, 355, 430, 430, 430, + /* 2180 */ 430, 430, 430, 348, 430, 430, 430, 378, 353, 430, + /* 2190 */ 355, 382, 383, 384, 385, 386, 387, 430, 389, 378, + /* 2200 */ 430, 430, 430, 382, 383, 384, 385, 386, 387, 430, + /* 2210 */ 389, 430, 430, 378, 430, 430, 430, 382, 383, 384, + /* 2220 */ 385, 386, 387, 312, 389, 430, 430, 430, 430, 430, + /* 2230 */ 430, 430, 430, 312, 430, 430, 430, 430, 430, 430, + /* 2240 */ 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2250 */ 430, 340, 430, 430, 430, 430, 430, 430, 430, 348, + /* 2260 */ 430, 340, 430, 430, 353, 430, 355, 430, 430, 348, + /* 2270 */ 430, 430, 430, 430, 353, 430, 355, 430, 430, 430, + /* 2280 */ 430, 430, 430, 430, 312, 430, 430, 430, 430, 378, + /* 2290 */ 430, 430, 430, 382, 383, 384, 385, 386, 387, 378, + /* 2300 */ 389, 430, 430, 382, 383, 384, 385, 386, 387, 430, + /* 2310 */ 389, 430, 340, 430, 430, 430, 430, 430, 430, 430, + /* 2320 */ 348, 430, 430, 430, 430, 353, 430, 355, 430, 430, + /* 2330 */ 430, 430, 430, 430, 312, 430, 430, 430, 430, 430, + /* 2340 */ 430, 430, 430, 430, 312, 430, 430, 430, 430, 430, + /* 2350 */ 378, 430, 430, 430, 382, 383, 384, 385, 386, 387, + /* 2360 */ 430, 389, 340, 430, 430, 430, 430, 430, 430, 430, + /* 2370 */ 348, 430, 340, 430, 430, 353, 430, 355, 430, 430, + /* 2380 */ 348, 430, 430, 430, 312, 353, 430, 355, 430, 430, + /* 2390 */ 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2400 */ 378, 430, 430, 430, 382, 383, 384, 385, 386, 387, + /* 2410 */ 378, 389, 340, 430, 382, 383, 384, 385, 386, 387, + /* 2420 */ 348, 389, 430, 430, 430, 353, 430, 355, 430, 430, + /* 2430 */ 430, 430, 430, 430, 312, 430, 430, 430, 430, 430, + /* 2440 */ 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2450 */ 378, 430, 430, 430, 382, 383, 384, 385, 386, 387, + /* 2460 */ 430, 389, 340, 430, 430, 430, 430, 430, 430, 430, + /* 2470 */ 348, 430, 430, 430, 430, 353, 430, 355, 430, 430, + /* 2480 */ 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2490 */ 430, 430, 430, 430, 430, 430, 430, 430, 430, 430, + /* 2500 */ 378, 430, 430, 430, 382, 383, 384, 385, 386, 387, + /* 2510 */ 430, 389, }; -#define YY_SHIFT_COUNT (666) +#define YY_SHIFT_COUNT (674) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2079) +#define YY_SHIFT_MAX (1957) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 917, 0, 0, 62, 62, 264, 264, 264, 326, 326, - /* 10 */ 264, 264, 391, 593, 720, 593, 593, 593, 593, 593, - /* 20 */ 593, 593, 593, 593, 593, 593, 593, 593, 593, 593, - /* 30 */ 593, 593, 593, 593, 593, 593, 593, 593, 593, 593, - /* 40 */ 593, 593, 265, 265, 17, 17, 17, 1007, 1007, 268, - /* 50 */ 1007, 1007, 160, 30, 56, 200, 56, 11, 11, 87, - /* 60 */ 87, 55, 165, 56, 56, 11, 11, 11, 11, 11, - /* 70 */ 11, 11, 11, 11, 11, 10, 11, 11, 11, 18, - /* 80 */ 11, 11, 67, 11, 11, 67, 123, 11, 67, 67, - /* 90 */ 67, 11, 227, 719, 662, 683, 683, 150, 213, 213, - /* 100 */ 213, 213, 213, 213, 213, 213, 213, 213, 213, 213, - /* 110 */ 213, 213, 213, 213, 213, 213, 213, 517, 881, 165, - /* 120 */ 403, 403, 573, 386, 849, 361, 361, 361, 386, 298, - /* 130 */ 298, 18, 350, 350, 67, 67, 294, 294, 270, 357, - /* 140 */ 198, 198, 198, 198, 198, 198, 198, 1119, 21, 383, - /* 150 */ 501, 105, 665, 484, 715, 828, 366, 799, 506, 800, - /* 160 */ 717, 649, 717, 924, 756, 756, 756, 807, 873, 980, - /* 170 */ 1174, 1078, 1206, 1232, 1232, 1206, 1107, 1107, 1232, 1232, - /* 180 */ 1232, 1257, 1257, 1264, 10, 18, 10, 1272, 1277, 10, - /* 190 */ 1272, 10, 10, 10, 1232, 10, 1257, 67, 67, 67, - /* 200 */ 67, 67, 67, 67, 67, 67, 67, 67, 1232, 1257, - /* 210 */ 294, 1264, 227, 1166, 18, 227, 1232, 1232, 1272, 227, - /* 220 */ 1118, 294, 294, 294, 294, 1118, 294, 1201, 227, 270, - /* 230 */ 227, 298, 1345, 294, 1139, 1118, 294, 294, 1139, 1118, - /* 240 */ 294, 294, 67, 1141, 1235, 1139, 1154, 1157, 1170, 980, - /* 250 */ 1178, 298, 1382, 1159, 1162, 1167, 1159, 1162, 1159, 1162, - /* 260 */ 1320, 1326, 294, 357, 1232, 227, 1383, 1257, 2259, 2259, - /* 270 */ 2259, 2259, 2259, 2259, 2259, 83, 1593, 214, 724, 126, - /* 280 */ 209, 491, 562, 622, 813, 535, 321, 698, 698, 698, - /* 290 */ 698, 698, 698, 698, 698, 521, 309, 13, 13, 115, - /* 300 */ 69, 599, 267, 708, 194, 377, 190, 465, 49, 49, - /* 310 */ 49, 49, 684, 944, 938, 994, 999, 1001, 947, 1047, - /* 320 */ 1101, 941, 920, 1025, 1043, 1069, 1079, 1091, 1093, 1102, - /* 330 */ 1160, 1073, 973, 895, 1103, 1075, 1082, 1058, 1106, 1114, - /* 340 */ 1115, 1133, 1163, 1165, 1193, 1198, 1064, 860, 1143, 1176, - /* 350 */ 839, 1473, 1476, 1301, 1479, 1480, 1439, 1482, 1448, 1297, - /* 360 */ 1450, 1451, 1452, 1302, 1490, 1456, 1457, 1307, 1495, 1310, - /* 370 */ 1502, 1477, 1513, 1493, 1516, 1483, 1348, 1347, 1521, 1527, - /* 380 */ 1357, 1359, 1525, 1528, 1494, 1529, 1499, 1539, 1542, 1543, - /* 390 */ 1395, 1545, 1552, 1554, 1555, 1556, 1403, 1514, 1548, 1413, - /* 400 */ 1559, 1560, 1561, 1562, 1566, 1567, 1568, 1569, 1570, 1571, - /* 410 */ 1573, 1574, 1575, 1578, 1544, 1590, 1591, 1598, 1600, 1601, - /* 420 */ 1612, 1581, 1604, 1605, 1614, 1616, 1619, 1564, 1621, 1572, - /* 430 */ 1627, 1629, 1582, 1584, 1588, 1620, 1587, 1622, 1589, 1638, - /* 440 */ 1599, 1602, 1650, 1651, 1652, 1617, 1492, 1657, 1658, 1665, - /* 450 */ 1607, 1667, 1671, 1608, 1597, 1633, 1673, 1639, 1630, 1637, - /* 460 */ 1679, 1645, 1640, 1647, 1688, 1654, 1644, 1653, 1693, 1694, - /* 470 */ 1698, 1699, 1609, 1606, 1666, 1678, 1703, 1670, 1675, 1680, - /* 480 */ 1686, 1695, 1684, 1710, 1711, 1732, 1713, 1702, 1737, 1722, - /* 490 */ 1717, 1754, 1720, 1756, 1724, 1757, 1738, 1741, 1762, 1610, - /* 500 */ 1731, 1768, 1613, 1748, 1618, 1624, 1771, 1773, 1625, 1615, - /* 510 */ 1780, 1781, 1783, 1696, 1697, 1755, 1634, 1798, 1712, 1648, - /* 520 */ 1715, 1800, 1778, 1659, 1735, 1727, 1782, 1784, 1611, 1740, - /* 530 */ 1742, 1745, 1746, 1747, 1749, 1787, 1750, 1758, 1760, 1761, - /* 540 */ 1763, 1792, 1793, 1799, 1765, 1801, 1626, 1770, 1772, 1815, - /* 550 */ 1641, 1809, 1817, 1818, 1775, 1825, 1642, 1779, 1835, 1849, - /* 560 */ 1858, 1859, 1860, 1864, 1779, 1898, 1885, 1716, 1867, 1823, - /* 570 */ 1824, 1826, 1827, 1829, 1830, 1870, 1833, 1834, 1878, 1891, - /* 580 */ 1743, 1844, 1819, 1845, 1899, 1901, 1848, 1850, 1906, 1853, - /* 590 */ 1854, 1908, 1856, 1863, 1911, 1861, 1865, 1913, 1868, 1836, - /* 600 */ 1843, 1847, 1851, 1936, 1862, 1872, 1876, 1927, 1877, 1925, - /* 610 */ 1925, 1947, 1914, 1909, 1938, 1940, 1943, 1945, 1954, 1955, - /* 620 */ 1958, 1928, 1922, 1952, 1963, 1965, 1988, 1976, 1991, 1980, - /* 630 */ 1981, 1950, 1675, 1985, 1680, 1994, 1995, 1998, 2000, 2014, - /* 640 */ 2003, 2039, 2005, 2002, 2011, 2051, 2017, 2006, 2015, 2055, - /* 650 */ 2021, 2013, 2019, 2069, 2035, 2024, 2034, 2076, 2042, 2043, - /* 660 */ 2079, 2058, 2060, 2062, 2063, 2061, 2066, + /* 0 */ 1011, 0, 0, 207, 207, 264, 264, 264, 471, 471, + /* 10 */ 264, 264, 678, 735, 942, 735, 735, 735, 735, 735, + /* 20 */ 735, 735, 735, 735, 735, 735, 735, 735, 735, 735, + /* 30 */ 735, 735, 735, 735, 735, 735, 735, 735, 735, 735, + /* 40 */ 735, 735, 153, 153, 152, 152, 152, 792, 792, 154, + /* 50 */ 792, 792, 96, 397, 311, 428, 311, 156, 156, 269, + /* 60 */ 269, 551, 93, 311, 311, 156, 156, 156, 156, 156, + /* 70 */ 156, 156, 156, 156, 156, 136, 156, 156, 156, 204, + /* 80 */ 156, 156, 518, 156, 156, 518, 550, 156, 518, 518, + /* 90 */ 518, 156, 564, 467, 1030, 854, 854, 228, 527, 527, + /* 100 */ 527, 527, 527, 527, 527, 527, 527, 527, 527, 527, + /* 110 */ 527, 527, 527, 527, 527, 527, 527, 424, 179, 93, + /* 120 */ 493, 493, 137, 372, 721, 485, 485, 485, 372, 621, + /* 130 */ 621, 204, 671, 671, 518, 518, 626, 626, 705, 770, + /* 140 */ 72, 72, 72, 72, 72, 72, 72, 535, 77, 565, + /* 150 */ 753, 131, 30, 304, 611, 614, 576, 879, 182, 824, + /* 160 */ 741, 255, 741, 958, 779, 779, 779, 276, 764, 984, + /* 170 */ 1172, 1047, 1181, 1204, 1204, 1181, 1075, 1075, 1204, 1204, + /* 180 */ 1204, 1222, 1222, 1224, 136, 204, 136, 1250, 1253, 136, + /* 190 */ 1250, 136, 136, 136, 1204, 136, 1222, 518, 518, 518, + /* 200 */ 518, 518, 518, 518, 518, 518, 518, 518, 1204, 1222, + /* 210 */ 626, 1224, 564, 1127, 204, 564, 1204, 1204, 1250, 564, + /* 220 */ 1097, 626, 626, 626, 626, 1097, 626, 1170, 564, 705, + /* 230 */ 564, 621, 1325, 626, 1110, 1097, 626, 626, 1110, 1097, + /* 240 */ 626, 626, 518, 1109, 1206, 1110, 1126, 1128, 1141, 984, + /* 250 */ 1147, 621, 1379, 1146, 1150, 1148, 1146, 1150, 1146, 1150, + /* 260 */ 1309, 1312, 626, 770, 1204, 564, 1376, 1222, 2512, 2512, + /* 270 */ 2512, 2512, 2512, 2512, 2512, 69, 941, 371, 260, 2, + /* 280 */ 222, 418, 434, 443, 624, 727, 983, 155, 155, 155, + /* 290 */ 155, 155, 155, 155, 155, 1009, 540, 12, 12, 370, + /* 300 */ 374, 529, 286, 257, 205, 253, 655, 486, 146, 146, + /* 310 */ 146, 146, 298, 36, 992, 804, 956, 987, 989, 1090, + /* 320 */ 1101, 1103, 1002, 930, 1004, 1022, 1024, 1070, 1072, 1077, + /* 330 */ 1078, 1093, 997, 411, 988, 1087, 1050, 1061, 976, 1088, + /* 340 */ 1082, 1095, 1096, 1118, 1119, 1120, 1122, 768, 1156, 1157, + /* 350 */ 1132, 43, 1443, 1448, 1269, 1449, 1452, 1417, 1461, 1427, + /* 360 */ 1271, 1429, 1430, 1431, 1275, 1468, 1434, 1437, 1281, 1480, + /* 370 */ 1291, 1484, 1450, 1486, 1465, 1488, 1454, 1314, 1317, 1493, + /* 380 */ 1494, 1327, 1326, 1501, 1502, 1457, 1504, 1505, 1506, 1466, + /* 390 */ 1509, 1510, 1511, 1362, 1516, 1518, 1519, 1530, 1532, 1374, + /* 400 */ 1498, 1523, 1387, 1536, 1540, 1547, 1550, 1552, 1553, 1554, + /* 410 */ 1558, 1559, 1560, 1561, 1563, 1565, 1566, 1525, 1568, 1570, + /* 420 */ 1571, 1572, 1573, 1574, 1555, 1575, 1576, 1578, 1588, 1589, + /* 430 */ 1524, 1581, 1526, 1583, 1584, 1543, 1551, 1556, 1586, 1557, + /* 440 */ 1590, 1562, 1609, 1546, 1577, 1611, 1613, 1614, 1579, 1451, + /* 450 */ 1619, 1621, 1623, 1564, 1624, 1625, 1593, 1582, 1594, 1627, + /* 460 */ 1596, 1585, 1595, 1636, 1602, 1591, 1608, 1648, 1620, 1610, + /* 470 */ 1622, 1662, 1663, 1664, 1665, 1567, 1597, 1632, 1637, 1660, + /* 480 */ 1633, 1635, 1629, 1631, 1640, 1641, 1655, 1678, 1657, 1680, + /* 490 */ 1661, 1639, 1681, 1667, 1647, 1684, 1650, 1686, 1656, 1692, + /* 500 */ 1671, 1674, 1695, 1537, 1675, 1698, 1528, 1687, 1549, 1569, + /* 510 */ 1712, 1713, 1604, 1606, 1714, 1715, 1716, 1626, 1628, 1682, + /* 520 */ 1533, 1721, 1630, 1587, 1634, 1724, 1690, 1599, 1642, 1643, + /* 530 */ 1679, 1689, 1503, 1644, 1646, 1645, 1649, 1654, 1666, 1691, + /* 540 */ 1658, 1683, 1694, 1696, 1677, 1699, 1704, 1707, 1697, 1711, + /* 550 */ 1539, 1701, 1702, 1722, 1500, 1729, 1727, 1731, 1703, 1745, + /* 560 */ 1601, 1706, 1720, 1757, 1758, 1763, 1767, 1768, 1706, 1802, + /* 570 */ 1783, 1612, 1764, 1718, 1723, 1725, 1728, 1726, 1732, 1760, + /* 580 */ 1730, 1746, 1769, 1787, 1616, 1747, 1710, 1744, 1778, 1786, + /* 590 */ 1748, 1749, 1807, 1752, 1753, 1811, 1755, 1766, 1817, 1770, + /* 600 */ 1771, 1818, 1773, 1750, 1751, 1754, 1756, 1842, 1779, 1776, + /* 610 */ 1793, 1837, 1794, 1833, 1833, 1866, 1827, 1829, 1856, 1858, + /* 620 */ 1859, 1860, 1861, 1862, 1863, 1864, 1867, 1868, 1845, 1810, + /* 630 */ 1871, 1880, 1881, 1895, 1883, 1897, 1886, 1887, 1888, 1857, + /* 640 */ 1629, 1889, 1631, 1891, 1892, 1894, 1896, 1908, 1899, 1932, + /* 650 */ 1901, 1890, 1900, 1941, 1907, 1898, 1904, 1944, 1912, 1902, + /* 660 */ 1909, 1950, 1917, 1910, 1915, 1955, 1924, 1926, 1956, 1946, + /* 670 */ 1949, 1951, 1952, 1954, 1957, }; #define YY_REDUCE_COUNT (274) -#define YY_REDUCE_MIN (-403) -#define YY_REDUCE_MAX (1873) +#define YY_REDUCE_MIN (-353) +#define YY_REDUCE_MAX (2122) static const short yy_reduce_ofst[] = { - /* 0 */ 263, 629, 747, -278, -14, 679, 846, 886, 955, 1009, - /* 10 */ 272, 1059, 104, 1076, 1126, 806, 1152, 1202, 1214, 1228, - /* 20 */ 1281, 1334, 1346, 1360, 1414, 1428, 1440, 1497, 1507, 1523, - /* 30 */ 1547, 1603, 1623, 1643, 1663, 1683, 1723, 1739, 1790, 1806, - /* 40 */ 1857, 1873, 304, 926, 112, 435, 552, -295, 610, -3, - /* 50 */ -261, -154, -323, 305, 575, 577, 635, -310, -272, -312, - /* 60 */ -307, -403, -311, -284, -70, -90, 225, 346, 404, 414, - /* 70 */ 429, 434, 497, 644, 645, 339, 674, 697, 710, -342, - /* 80 */ 722, 749, -313, 758, 772, -244, -251, 797, -17, 20, - /* 90 */ 48, 804, 228, 86, -379, -379, -379, -240, -13, 100, - /* 100 */ 101, 130, 206, 375, 385, 428, 459, 461, 474, 522, - /* 110 */ 566, 628, 656, 678, 694, 696, 740, -226, -92, -72, - /* 120 */ -256, -130, 173, 4, 34, 186, 362, 440, 97, 91, - /* 130 */ 303, -234, -109, 180, 274, 536, 317, 545, 579, 234, - /* 140 */ -333, 40, 144, 175, 242, 319, 410, 450, 445, 348, - /* 150 */ 369, 495, 572, 557, 620, 620, 734, 667, 688, 727, - /* 160 */ 735, 735, 735, 770, 752, 784, 786, 762, 620, 861, - /* 170 */ 874, 866, 894, 940, 942, 901, 907, 908, 956, 957, - /* 180 */ 960, 971, 972, 915, 966, 936, 974, 933, 935, 978, - /* 190 */ 939, 981, 983, 985, 991, 989, 1000, 975, 976, 977, - /* 200 */ 979, 982, 984, 986, 990, 992, 995, 996, 998, 1006, - /* 210 */ 993, 951, 1021, 987, 997, 1026, 1030, 1033, 988, 1032, - /* 220 */ 1002, 1005, 1008, 1013, 1014, 1010, 1015, 1011, 1048, 1038, - /* 230 */ 1053, 1031, 1003, 1028, 968, 1020, 1036, 1037, 1004, 1022, - /* 240 */ 1044, 1049, 620, 1016, 1012, 1018, 1023, 1027, 1024, 1029, - /* 250 */ 735, 1061, 1035, 1045, 1017, 1039, 1046, 1034, 1051, 1050, - /* 260 */ 1052, 1081, 1067, 1092, 1129, 1104, 1112, 1136, 1096, 1086, - /* 270 */ 1137, 1138, 1142, 1130, 1168, + /* 0 */ 285, -311, -38, 148, 882, 935, 999, 1039, 412, 1092, + /* 10 */ 1142, 1159, -22, -261, 1209, 677, 1257, 1267, 1318, 1378, + /* 20 */ 1396, 1446, 975, 1472, 1496, 1522, 1580, 1598, 1659, 1709, + /* 30 */ 1719, 1759, 1809, 1821, 1835, 1911, 1921, 1972, 2022, 2032, + /* 40 */ 2072, 2122, -120, 1592, 249, 444, 579, -319, -289, -309, + /* 50 */ -323, -258, -117, 140, 142, 287, 329, -299, -42, -316, + /* 60 */ -312, -87, -346, 94, 103, 169, 413, 425, 431, 454, + /* 70 */ 458, 491, 492, 495, 511, 383, 578, 580, 582, -12, + /* 80 */ 583, 637, -37, 639, 692, -229, -115, 693, 243, 88, + /* 90 */ 244, 577, 387, -275, -353, -353, -353, -265, -14, 76, + /* 100 */ 308, 405, 416, 419, 447, 476, 498, 520, 522, 566, + /* 110 */ 605, 608, 610, 620, 623, 666, 667, 233, -6, -292, + /* 120 */ 7, 446, 377, 332, -51, 525, 612, 646, 449, 188, + /* 130 */ 379, 422, 488, 508, 40, 659, 568, 668, 718, 528, + /* 140 */ 516, 575, 619, 788, 827, 832, 835, 534, 864, 756, + /* 150 */ 776, 766, 865, 773, 853, 853, 874, 877, 845, 817, + /* 160 */ 795, 795, 795, 810, 781, 789, 793, 801, 853, 841, + /* 170 */ 846, 855, 866, 908, 909, 871, 875, 876, 916, 919, + /* 180 */ 920, 931, 932, 887, 924, 907, 939, 905, 912, 952, + /* 190 */ 910, 960, 961, 962, 969, 967, 968, 953, 954, 955, + /* 200 */ 957, 959, 963, 964, 965, 966, 970, 972, 978, 982, + /* 210 */ 948, 918, 980, 936, 971, 986, 994, 996, 973, 1003, + /* 220 */ 974, 981, 985, 990, 991, 993, 995, 1005, 1013, 1010, + /* 230 */ 1019, 1001, 1008, 1012, 949, 1006, 1014, 1015, 979, 1007, + /* 240 */ 1018, 1020, 853, 998, 1000, 1016, 977, 1017, 1023, 1025, + /* 250 */ 795, 1032, 1027, 1026, 1021, 1028, 1029, 1033, 1031, 1035, + /* 260 */ 1038, 1064, 1056, 1080, 1091, 1098, 1099, 1104, 1042, 1053, + /* 270 */ 1105, 1107, 1108, 1115, 1117, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 10 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 20 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 30 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 40 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 50 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 60 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 70 */ 1461, 1461, 1461, 1461, 1461, 1535, 1461, 1461, 1461, 1461, - /* 80 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 90 */ 1461, 1461, 1533, 1691, 1461, 1866, 1461, 1461, 1461, 1461, - /* 100 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 110 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 120 */ 1461, 1461, 1535, 1461, 1533, 1878, 1878, 1878, 1461, 1461, - /* 130 */ 1461, 1461, 1732, 1732, 1461, 1461, 1461, 1461, 1633, 1461, - /* 140 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1726, 1461, 1947, - /* 150 */ 1461, 1461, 1461, 1901, 1461, 1461, 1461, 1461, 1586, 1893, - /* 160 */ 1870, 1884, 1871, 1868, 1932, 1932, 1932, 1887, 1461, 1897, - /* 170 */ 1461, 1719, 1696, 1461, 1461, 1696, 1693, 1693, 1461, 1461, - /* 180 */ 1461, 1461, 1461, 1461, 1535, 1461, 1535, 1461, 1461, 1535, - /* 190 */ 1461, 1535, 1535, 1535, 1461, 1535, 1461, 1461, 1461, 1461, - /* 200 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 210 */ 1461, 1461, 1533, 1728, 1461, 1533, 1461, 1461, 1461, 1533, - /* 220 */ 1906, 1461, 1461, 1461, 1461, 1906, 1461, 1461, 1533, 1461, - /* 230 */ 1533, 1461, 1461, 1461, 1908, 1906, 1461, 1461, 1908, 1906, - /* 240 */ 1461, 1461, 1461, 1920, 1916, 1908, 1924, 1922, 1899, 1897, - /* 250 */ 1884, 1461, 1461, 1938, 1934, 1950, 1938, 1934, 1938, 1934, - /* 260 */ 1461, 1602, 1461, 1461, 1461, 1533, 1493, 1461, 1721, 1732, - /* 270 */ 1636, 1636, 1636, 1536, 1466, 1461, 1461, 1461, 1461, 1461, - /* 280 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1803, 1919, 1918, - /* 290 */ 1842, 1841, 1840, 1838, 1802, 1461, 1598, 1801, 1800, 1461, - /* 300 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1794, 1795, - /* 310 */ 1793, 1792, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 320 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 330 */ 1867, 1461, 1935, 1939, 1461, 1461, 1461, 1461, 1461, 1778, - /* 340 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 350 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 360 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 370 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 380 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 390 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 400 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 410 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 420 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 430 */ 1461, 1461, 1461, 1461, 1498, 1461, 1461, 1461, 1461, 1461, - /* 440 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 450 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 460 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 470 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1570, 1569, - /* 480 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 490 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 500 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 510 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1736, 1461, 1461, - /* 520 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1900, 1461, 1461, - /* 530 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 540 */ 1461, 1461, 1461, 1778, 1461, 1917, 1461, 1877, 1873, 1461, - /* 550 */ 1461, 1869, 1777, 1461, 1461, 1933, 1461, 1461, 1461, 1461, - /* 560 */ 1461, 1461, 1461, 1461, 1461, 1862, 1461, 1461, 1835, 1820, - /* 570 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 580 */ 1788, 1461, 1461, 1461, 1461, 1461, 1630, 1461, 1461, 1461, - /* 590 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1615, - /* 600 */ 1613, 1612, 1611, 1461, 1608, 1461, 1461, 1461, 1461, 1639, - /* 610 */ 1638, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 620 */ 1461, 1461, 1461, 1554, 1461, 1461, 1461, 1461, 1461, 1461, - /* 630 */ 1461, 1461, 1546, 1461, 1545, 1461, 1461, 1461, 1461, 1461, - /* 640 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 650 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, 1461, - /* 660 */ 1461, 1461, 1461, 1461, 1461, 1461, 1461, + /* 0 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 10 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 20 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 30 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 40 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 50 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 60 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 70 */ 1481, 1481, 1481, 1481, 1481, 1555, 1481, 1481, 1481, 1481, + /* 80 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 90 */ 1481, 1481, 1553, 1718, 1481, 1893, 1481, 1481, 1481, 1481, + /* 100 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 110 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 120 */ 1481, 1481, 1555, 1481, 1553, 1905, 1905, 1905, 1481, 1481, + /* 130 */ 1481, 1481, 1759, 1759, 1481, 1481, 1481, 1481, 1658, 1481, + /* 140 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1753, 1481, 1974, + /* 150 */ 1481, 1481, 1481, 1928, 1481, 1481, 1481, 1481, 1611, 1920, + /* 160 */ 1897, 1911, 1898, 1895, 1959, 1959, 1959, 1914, 1481, 1924, + /* 170 */ 1481, 1746, 1723, 1481, 1481, 1723, 1720, 1720, 1481, 1481, + /* 180 */ 1481, 1481, 1481, 1481, 1555, 1481, 1555, 1481, 1481, 1555, + /* 190 */ 1481, 1555, 1555, 1555, 1481, 1555, 1481, 1481, 1481, 1481, + /* 200 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 210 */ 1481, 1481, 1553, 1755, 1481, 1553, 1481, 1481, 1481, 1553, + /* 220 */ 1933, 1481, 1481, 1481, 1481, 1933, 1481, 1481, 1553, 1481, + /* 230 */ 1553, 1481, 1481, 1481, 1935, 1933, 1481, 1481, 1935, 1933, + /* 240 */ 1481, 1481, 1481, 1947, 1943, 1935, 1951, 1949, 1926, 1924, + /* 250 */ 1911, 1481, 1481, 1965, 1961, 1977, 1965, 1961, 1965, 1961, + /* 260 */ 1481, 1627, 1481, 1481, 1481, 1553, 1513, 1481, 1748, 1759, + /* 270 */ 1661, 1661, 1661, 1556, 1486, 1481, 1481, 1481, 1481, 1481, + /* 280 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1830, 1946, 1945, + /* 290 */ 1869, 1868, 1867, 1865, 1829, 1481, 1623, 1828, 1827, 1481, + /* 300 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1821, 1822, + /* 310 */ 1820, 1819, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 320 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 330 */ 1481, 1894, 1481, 1962, 1966, 1481, 1481, 1481, 1481, 1481, + /* 340 */ 1805, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 350 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 360 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 370 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 380 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 390 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 400 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 410 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 420 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 430 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1518, 1481, 1481, + /* 440 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 450 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 460 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 470 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 480 */ 1481, 1481, 1594, 1593, 1481, 1481, 1481, 1481, 1481, 1481, + /* 490 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 500 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 510 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 520 */ 1481, 1763, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 530 */ 1481, 1927, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 540 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1805, 1481, 1944, + /* 550 */ 1481, 1904, 1900, 1481, 1481, 1896, 1804, 1481, 1481, 1960, + /* 560 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1889, + /* 570 */ 1481, 1481, 1862, 1847, 1481, 1481, 1481, 1481, 1481, 1481, + /* 580 */ 1481, 1481, 1481, 1481, 1815, 1481, 1481, 1481, 1481, 1481, + /* 590 */ 1655, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 600 */ 1481, 1481, 1481, 1640, 1638, 1637, 1636, 1481, 1633, 1481, + /* 610 */ 1481, 1481, 1481, 1664, 1663, 1481, 1481, 1481, 1481, 1481, + /* 620 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 630 */ 1575, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 640 */ 1566, 1481, 1565, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 650 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 660 */ 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, 1481, + /* 670 */ 1481, 1481, 1481, 1481, 1481, }; /********** End of lemon-generated parsing tables *****************************/ @@ -953,6 +993,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* KEEP => nothing */ 0, /* PAGES => nothing */ 0, /* PAGESIZE => nothing */ + 0, /* TSDB_PAGESIZE => nothing */ 0, /* PRECISION => nothing */ 0, /* REPLICA => nothing */ 0, /* STRICT => nothing */ @@ -966,6 +1007,9 @@ static const YYCODETYPE yyFallback[] = { 0, /* WAL_RETENTION_SIZE => nothing */ 0, /* WAL_ROLL_PERIOD => nothing */ 0, /* WAL_SEGMENT_SIZE => nothing */ + 0, /* STT_TRIGGER => nothing */ + 0, /* TABLE_PREFIX => nothing */ + 0, /* TABLE_SUFFIX => nothing */ 0, /* NK_COLON => nothing */ 0, /* TABLE => nothing */ 0, /* NK_LP => nothing */ @@ -1031,6 +1075,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* DISTRIBUTED => nothing */ 0, /* CONSUMERS => nothing */ 0, /* SUBSCRIPTIONS => nothing */ + 0, /* VNODES => nothing */ 0, /* LIKE => nothing */ 0, /* INDEX => nothing */ 0, /* FUNCTION => nothing */ @@ -1131,59 +1176,58 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ABORT => nothing */ - 251, /* AFTER => ABORT */ - 251, /* ATTACH => ABORT */ - 251, /* BEFORE => ABORT */ - 251, /* BEGIN => ABORT */ - 251, /* BITAND => ABORT */ - 251, /* BITNOT => ABORT */ - 251, /* BITOR => ABORT */ - 251, /* BLOCKS => ABORT */ - 251, /* CHANGE => ABORT */ - 251, /* COMMA => ABORT */ - 251, /* COMPACT => ABORT */ - 251, /* CONCAT => ABORT */ - 251, /* CONFLICT => ABORT */ - 251, /* COPY => ABORT */ - 251, /* DEFERRED => ABORT */ - 251, /* DELIMITERS => ABORT */ - 251, /* DETACH => ABORT */ - 251, /* DIVIDE => ABORT */ - 251, /* DOT => ABORT */ - 251, /* EACH => ABORT */ - 251, /* END => ABORT */ - 251, /* FAIL => ABORT */ - 251, /* FILE => ABORT */ - 251, /* FOR => ABORT */ - 251, /* GLOB => ABORT */ - 251, /* ID => ABORT */ - 251, /* IMMEDIATE => ABORT */ - 251, /* IMPORT => ABORT */ - 251, /* INITIALLY => ABORT */ - 251, /* INSTEAD => ABORT */ - 251, /* ISNULL => ABORT */ - 251, /* KEY => ABORT */ - 251, /* NK_BITNOT => ABORT */ - 251, /* NK_SEMI => ABORT */ - 251, /* NOTNULL => ABORT */ - 251, /* OF => ABORT */ - 251, /* PLUS => ABORT */ - 251, /* PRIVILEGE => ABORT */ - 251, /* RAISE => ABORT */ - 251, /* REPLACE => ABORT */ - 251, /* RESTRICT => ABORT */ - 251, /* ROW => ABORT */ - 251, /* SEMI => ABORT */ - 251, /* STAR => ABORT */ - 251, /* STATEMENT => ABORT */ - 251, /* STRING => ABORT */ - 251, /* TIMES => ABORT */ - 251, /* UPDATE => ABORT */ - 251, /* VALUES => ABORT */ - 251, /* VARIABLE => ABORT */ - 251, /* VIEW => ABORT */ - 251, /* VNODES => ABORT */ - 251, /* WAL => ABORT */ + 256, /* AFTER => ABORT */ + 256, /* ATTACH => ABORT */ + 256, /* BEFORE => ABORT */ + 256, /* BEGIN => ABORT */ + 256, /* BITAND => ABORT */ + 256, /* BITNOT => ABORT */ + 256, /* BITOR => ABORT */ + 256, /* BLOCKS => ABORT */ + 256, /* CHANGE => ABORT */ + 256, /* COMMA => ABORT */ + 256, /* COMPACT => ABORT */ + 256, /* CONCAT => ABORT */ + 256, /* CONFLICT => ABORT */ + 256, /* COPY => ABORT */ + 256, /* DEFERRED => ABORT */ + 256, /* DELIMITERS => ABORT */ + 256, /* DETACH => ABORT */ + 256, /* DIVIDE => ABORT */ + 256, /* DOT => ABORT */ + 256, /* EACH => ABORT */ + 256, /* END => ABORT */ + 256, /* FAIL => ABORT */ + 256, /* FILE => ABORT */ + 256, /* FOR => ABORT */ + 256, /* GLOB => ABORT */ + 256, /* ID => ABORT */ + 256, /* IMMEDIATE => ABORT */ + 256, /* IMPORT => ABORT */ + 256, /* INITIALLY => ABORT */ + 256, /* INSTEAD => ABORT */ + 256, /* ISNULL => ABORT */ + 256, /* KEY => ABORT */ + 256, /* NK_BITNOT => ABORT */ + 256, /* NK_SEMI => ABORT */ + 256, /* NOTNULL => ABORT */ + 256, /* OF => ABORT */ + 256, /* PLUS => ABORT */ + 256, /* PRIVILEGE => ABORT */ + 256, /* RAISE => ABORT */ + 256, /* REPLACE => ABORT */ + 256, /* RESTRICT => ABORT */ + 256, /* ROW => ABORT */ + 256, /* SEMI => ABORT */ + 256, /* STAR => ABORT */ + 256, /* STATEMENT => ABORT */ + 256, /* STRING => ABORT */ + 256, /* TIMES => ABORT */ + 256, /* UPDATE => ABORT */ + 256, /* VALUES => ABORT */ + 256, /* VARIABLE => ABORT */ + 256, /* VIEW => ABORT */ + 256, /* WAL => ABORT */ }; #endif /* YYFALLBACK */ @@ -1345,358 +1389,362 @@ static const char *const yyTokenName[] = { /* 71 */ "KEEP", /* 72 */ "PAGES", /* 73 */ "PAGESIZE", - /* 74 */ "PRECISION", - /* 75 */ "REPLICA", - /* 76 */ "STRICT", - /* 77 */ "VGROUPS", - /* 78 */ "SINGLE_STABLE", - /* 79 */ "RETENTIONS", - /* 80 */ "SCHEMALESS", - /* 81 */ "WAL_LEVEL", - /* 82 */ "WAL_FSYNC_PERIOD", - /* 83 */ "WAL_RETENTION_PERIOD", - /* 84 */ "WAL_RETENTION_SIZE", - /* 85 */ "WAL_ROLL_PERIOD", - /* 86 */ "WAL_SEGMENT_SIZE", - /* 87 */ "NK_COLON", - /* 88 */ "TABLE", - /* 89 */ "NK_LP", - /* 90 */ "NK_RP", - /* 91 */ "STABLE", - /* 92 */ "ADD", - /* 93 */ "COLUMN", - /* 94 */ "MODIFY", - /* 95 */ "RENAME", - /* 96 */ "TAG", - /* 97 */ "SET", - /* 98 */ "NK_EQ", - /* 99 */ "USING", - /* 100 */ "TAGS", - /* 101 */ "COMMENT", - /* 102 */ "BOOL", - /* 103 */ "TINYINT", - /* 104 */ "SMALLINT", - /* 105 */ "INT", - /* 106 */ "INTEGER", - /* 107 */ "BIGINT", - /* 108 */ "FLOAT", - /* 109 */ "DOUBLE", - /* 110 */ "BINARY", - /* 111 */ "TIMESTAMP", - /* 112 */ "NCHAR", - /* 113 */ "UNSIGNED", - /* 114 */ "JSON", - /* 115 */ "VARCHAR", - /* 116 */ "MEDIUMBLOB", - /* 117 */ "BLOB", - /* 118 */ "VARBINARY", - /* 119 */ "DECIMAL", - /* 120 */ "MAX_DELAY", - /* 121 */ "WATERMARK", - /* 122 */ "ROLLUP", - /* 123 */ "TTL", - /* 124 */ "SMA", - /* 125 */ "FIRST", - /* 126 */ "LAST", - /* 127 */ "SHOW", - /* 128 */ "DATABASES", - /* 129 */ "TABLES", - /* 130 */ "STABLES", - /* 131 */ "MNODES", - /* 132 */ "MODULES", - /* 133 */ "QNODES", - /* 134 */ "FUNCTIONS", - /* 135 */ "INDEXES", - /* 136 */ "ACCOUNTS", - /* 137 */ "APPS", - /* 138 */ "CONNECTIONS", - /* 139 */ "LICENCES", - /* 140 */ "GRANTS", - /* 141 */ "QUERIES", - /* 142 */ "SCORES", - /* 143 */ "TOPICS", - /* 144 */ "VARIABLES", - /* 145 */ "BNODES", - /* 146 */ "SNODES", - /* 147 */ "CLUSTER", - /* 148 */ "TRANSACTIONS", - /* 149 */ "DISTRIBUTED", - /* 150 */ "CONSUMERS", - /* 151 */ "SUBSCRIPTIONS", - /* 152 */ "LIKE", - /* 153 */ "INDEX", - /* 154 */ "FUNCTION", - /* 155 */ "INTERVAL", - /* 156 */ "TOPIC", - /* 157 */ "AS", - /* 158 */ "WITH", - /* 159 */ "META", - /* 160 */ "CONSUMER", - /* 161 */ "GROUP", - /* 162 */ "DESC", - /* 163 */ "DESCRIBE", - /* 164 */ "RESET", - /* 165 */ "QUERY", - /* 166 */ "CACHE", - /* 167 */ "EXPLAIN", - /* 168 */ "ANALYZE", - /* 169 */ "VERBOSE", - /* 170 */ "NK_BOOL", - /* 171 */ "RATIO", - /* 172 */ "NK_FLOAT", - /* 173 */ "OUTPUTTYPE", - /* 174 */ "AGGREGATE", - /* 175 */ "BUFSIZE", - /* 176 */ "STREAM", - /* 177 */ "INTO", - /* 178 */ "TRIGGER", - /* 179 */ "AT_ONCE", - /* 180 */ "WINDOW_CLOSE", - /* 181 */ "IGNORE", - /* 182 */ "EXPIRED", - /* 183 */ "KILL", - /* 184 */ "CONNECTION", - /* 185 */ "TRANSACTION", - /* 186 */ "BALANCE", - /* 187 */ "VGROUP", - /* 188 */ "MERGE", - /* 189 */ "REDISTRIBUTE", - /* 190 */ "SPLIT", - /* 191 */ "DELETE", - /* 192 */ "INSERT", - /* 193 */ "NULL", - /* 194 */ "NK_QUESTION", - /* 195 */ "NK_ARROW", - /* 196 */ "ROWTS", - /* 197 */ "TBNAME", - /* 198 */ "QSTART", - /* 199 */ "QEND", - /* 200 */ "QDURATION", - /* 201 */ "WSTART", - /* 202 */ "WEND", - /* 203 */ "WDURATION", - /* 204 */ "CAST", - /* 205 */ "NOW", - /* 206 */ "TODAY", - /* 207 */ "TIMEZONE", - /* 208 */ "CLIENT_VERSION", - /* 209 */ "SERVER_VERSION", - /* 210 */ "SERVER_STATUS", - /* 211 */ "CURRENT_USER", - /* 212 */ "COUNT", - /* 213 */ "LAST_ROW", - /* 214 */ "BETWEEN", - /* 215 */ "IS", - /* 216 */ "NK_LT", - /* 217 */ "NK_GT", - /* 218 */ "NK_LE", - /* 219 */ "NK_GE", - /* 220 */ "NK_NE", - /* 221 */ "MATCH", - /* 222 */ "NMATCH", - /* 223 */ "CONTAINS", - /* 224 */ "IN", - /* 225 */ "JOIN", - /* 226 */ "INNER", - /* 227 */ "SELECT", - /* 228 */ "DISTINCT", - /* 229 */ "WHERE", - /* 230 */ "PARTITION", - /* 231 */ "BY", - /* 232 */ "SESSION", - /* 233 */ "STATE_WINDOW", - /* 234 */ "SLIDING", - /* 235 */ "FILL", - /* 236 */ "VALUE", - /* 237 */ "NONE", - /* 238 */ "PREV", - /* 239 */ "LINEAR", - /* 240 */ "NEXT", - /* 241 */ "HAVING", - /* 242 */ "RANGE", - /* 243 */ "EVERY", - /* 244 */ "ORDER", - /* 245 */ "SLIMIT", - /* 246 */ "SOFFSET", - /* 247 */ "LIMIT", - /* 248 */ "OFFSET", - /* 249 */ "ASC", - /* 250 */ "NULLS", - /* 251 */ "ABORT", - /* 252 */ "AFTER", - /* 253 */ "ATTACH", - /* 254 */ "BEFORE", - /* 255 */ "BEGIN", - /* 256 */ "BITAND", - /* 257 */ "BITNOT", - /* 258 */ "BITOR", - /* 259 */ "BLOCKS", - /* 260 */ "CHANGE", - /* 261 */ "COMMA", - /* 262 */ "COMPACT", - /* 263 */ "CONCAT", - /* 264 */ "CONFLICT", - /* 265 */ "COPY", - /* 266 */ "DEFERRED", - /* 267 */ "DELIMITERS", - /* 268 */ "DETACH", - /* 269 */ "DIVIDE", - /* 270 */ "DOT", - /* 271 */ "EACH", - /* 272 */ "END", - /* 273 */ "FAIL", - /* 274 */ "FILE", - /* 275 */ "FOR", - /* 276 */ "GLOB", - /* 277 */ "ID", - /* 278 */ "IMMEDIATE", - /* 279 */ "IMPORT", - /* 280 */ "INITIALLY", - /* 281 */ "INSTEAD", - /* 282 */ "ISNULL", - /* 283 */ "KEY", - /* 284 */ "NK_BITNOT", - /* 285 */ "NK_SEMI", - /* 286 */ "NOTNULL", - /* 287 */ "OF", - /* 288 */ "PLUS", - /* 289 */ "PRIVILEGE", - /* 290 */ "RAISE", - /* 291 */ "REPLACE", - /* 292 */ "RESTRICT", - /* 293 */ "ROW", - /* 294 */ "SEMI", - /* 295 */ "STAR", - /* 296 */ "STATEMENT", - /* 297 */ "STRING", - /* 298 */ "TIMES", - /* 299 */ "UPDATE", - /* 300 */ "VALUES", - /* 301 */ "VARIABLE", - /* 302 */ "VIEW", - /* 303 */ "VNODES", - /* 304 */ "WAL", - /* 305 */ "cmd", - /* 306 */ "account_options", - /* 307 */ "alter_account_options", - /* 308 */ "literal", - /* 309 */ "alter_account_option", - /* 310 */ "user_name", - /* 311 */ "sysinfo_opt", - /* 312 */ "privileges", - /* 313 */ "priv_level", - /* 314 */ "priv_type_list", - /* 315 */ "priv_type", - /* 316 */ "db_name", - /* 317 */ "dnode_endpoint", - /* 318 */ "not_exists_opt", - /* 319 */ "db_options", - /* 320 */ "exists_opt", - /* 321 */ "alter_db_options", - /* 322 */ "integer_list", - /* 323 */ "variable_list", - /* 324 */ "retention_list", - /* 325 */ "alter_db_option", - /* 326 */ "retention", - /* 327 */ "full_table_name", - /* 328 */ "column_def_list", - /* 329 */ "tags_def_opt", - /* 330 */ "table_options", - /* 331 */ "multi_create_clause", - /* 332 */ "tags_def", - /* 333 */ "multi_drop_clause", - /* 334 */ "alter_table_clause", - /* 335 */ "alter_table_options", - /* 336 */ "column_name", - /* 337 */ "type_name", - /* 338 */ "signed_literal", - /* 339 */ "create_subtable_clause", - /* 340 */ "specific_cols_opt", - /* 341 */ "expression_list", - /* 342 */ "drop_table_clause", - /* 343 */ "col_name_list", - /* 344 */ "table_name", - /* 345 */ "column_def", - /* 346 */ "duration_list", - /* 347 */ "rollup_func_list", - /* 348 */ "alter_table_option", - /* 349 */ "duration_literal", - /* 350 */ "rollup_func_name", - /* 351 */ "function_name", - /* 352 */ "col_name", - /* 353 */ "db_name_cond_opt", - /* 354 */ "like_pattern_opt", - /* 355 */ "table_name_cond", - /* 356 */ "from_db_opt", - /* 357 */ "index_options", - /* 358 */ "func_list", - /* 359 */ "sliding_opt", - /* 360 */ "sma_stream_opt", - /* 361 */ "func", - /* 362 */ "stream_options", - /* 363 */ "topic_name", - /* 364 */ "query_expression", - /* 365 */ "cgroup_name", - /* 366 */ "analyze_opt", - /* 367 */ "explain_options", - /* 368 */ "agg_func_opt", - /* 369 */ "bufsize_opt", - /* 370 */ "stream_name", - /* 371 */ "dnode_list", - /* 372 */ "where_clause_opt", - /* 373 */ "signed", - /* 374 */ "literal_func", - /* 375 */ "literal_list", - /* 376 */ "table_alias", - /* 377 */ "column_alias", - /* 378 */ "expression", - /* 379 */ "pseudo_column", - /* 380 */ "column_reference", - /* 381 */ "function_expression", - /* 382 */ "subquery", - /* 383 */ "star_func", - /* 384 */ "star_func_para_list", - /* 385 */ "noarg_func", - /* 386 */ "other_para_list", - /* 387 */ "star_func_para", - /* 388 */ "predicate", - /* 389 */ "compare_op", - /* 390 */ "in_op", - /* 391 */ "in_predicate_value", - /* 392 */ "boolean_value_expression", - /* 393 */ "boolean_primary", - /* 394 */ "common_expression", - /* 395 */ "from_clause_opt", - /* 396 */ "table_reference_list", - /* 397 */ "table_reference", - /* 398 */ "table_primary", - /* 399 */ "joined_table", - /* 400 */ "alias_opt", - /* 401 */ "parenthesized_joined_table", - /* 402 */ "join_type", - /* 403 */ "search_condition", - /* 404 */ "query_specification", - /* 405 */ "set_quantifier_opt", - /* 406 */ "select_list", - /* 407 */ "partition_by_clause_opt", - /* 408 */ "range_opt", - /* 409 */ "every_opt", - /* 410 */ "fill_opt", - /* 411 */ "twindow_clause_opt", - /* 412 */ "group_by_clause_opt", - /* 413 */ "having_clause_opt", - /* 414 */ "select_item", - /* 415 */ "fill_mode", - /* 416 */ "group_by_list", - /* 417 */ "query_expression_body", - /* 418 */ "order_by_clause_opt", - /* 419 */ "slimit_clause_opt", - /* 420 */ "limit_clause_opt", - /* 421 */ "query_primary", - /* 422 */ "sort_specification_list", - /* 423 */ "sort_specification", - /* 424 */ "ordering_specification_opt", - /* 425 */ "null_ordering_opt", + /* 74 */ "TSDB_PAGESIZE", + /* 75 */ "PRECISION", + /* 76 */ "REPLICA", + /* 77 */ "STRICT", + /* 78 */ "VGROUPS", + /* 79 */ "SINGLE_STABLE", + /* 80 */ "RETENTIONS", + /* 81 */ "SCHEMALESS", + /* 82 */ "WAL_LEVEL", + /* 83 */ "WAL_FSYNC_PERIOD", + /* 84 */ "WAL_RETENTION_PERIOD", + /* 85 */ "WAL_RETENTION_SIZE", + /* 86 */ "WAL_ROLL_PERIOD", + /* 87 */ "WAL_SEGMENT_SIZE", + /* 88 */ "STT_TRIGGER", + /* 89 */ "TABLE_PREFIX", + /* 90 */ "TABLE_SUFFIX", + /* 91 */ "NK_COLON", + /* 92 */ "TABLE", + /* 93 */ "NK_LP", + /* 94 */ "NK_RP", + /* 95 */ "STABLE", + /* 96 */ "ADD", + /* 97 */ "COLUMN", + /* 98 */ "MODIFY", + /* 99 */ "RENAME", + /* 100 */ "TAG", + /* 101 */ "SET", + /* 102 */ "NK_EQ", + /* 103 */ "USING", + /* 104 */ "TAGS", + /* 105 */ "COMMENT", + /* 106 */ "BOOL", + /* 107 */ "TINYINT", + /* 108 */ "SMALLINT", + /* 109 */ "INT", + /* 110 */ "INTEGER", + /* 111 */ "BIGINT", + /* 112 */ "FLOAT", + /* 113 */ "DOUBLE", + /* 114 */ "BINARY", + /* 115 */ "TIMESTAMP", + /* 116 */ "NCHAR", + /* 117 */ "UNSIGNED", + /* 118 */ "JSON", + /* 119 */ "VARCHAR", + /* 120 */ "MEDIUMBLOB", + /* 121 */ "BLOB", + /* 122 */ "VARBINARY", + /* 123 */ "DECIMAL", + /* 124 */ "MAX_DELAY", + /* 125 */ "WATERMARK", + /* 126 */ "ROLLUP", + /* 127 */ "TTL", + /* 128 */ "SMA", + /* 129 */ "FIRST", + /* 130 */ "LAST", + /* 131 */ "SHOW", + /* 132 */ "DATABASES", + /* 133 */ "TABLES", + /* 134 */ "STABLES", + /* 135 */ "MNODES", + /* 136 */ "MODULES", + /* 137 */ "QNODES", + /* 138 */ "FUNCTIONS", + /* 139 */ "INDEXES", + /* 140 */ "ACCOUNTS", + /* 141 */ "APPS", + /* 142 */ "CONNECTIONS", + /* 143 */ "LICENCES", + /* 144 */ "GRANTS", + /* 145 */ "QUERIES", + /* 146 */ "SCORES", + /* 147 */ "TOPICS", + /* 148 */ "VARIABLES", + /* 149 */ "BNODES", + /* 150 */ "SNODES", + /* 151 */ "CLUSTER", + /* 152 */ "TRANSACTIONS", + /* 153 */ "DISTRIBUTED", + /* 154 */ "CONSUMERS", + /* 155 */ "SUBSCRIPTIONS", + /* 156 */ "VNODES", + /* 157 */ "LIKE", + /* 158 */ "INDEX", + /* 159 */ "FUNCTION", + /* 160 */ "INTERVAL", + /* 161 */ "TOPIC", + /* 162 */ "AS", + /* 163 */ "WITH", + /* 164 */ "META", + /* 165 */ "CONSUMER", + /* 166 */ "GROUP", + /* 167 */ "DESC", + /* 168 */ "DESCRIBE", + /* 169 */ "RESET", + /* 170 */ "QUERY", + /* 171 */ "CACHE", + /* 172 */ "EXPLAIN", + /* 173 */ "ANALYZE", + /* 174 */ "VERBOSE", + /* 175 */ "NK_BOOL", + /* 176 */ "RATIO", + /* 177 */ "NK_FLOAT", + /* 178 */ "OUTPUTTYPE", + /* 179 */ "AGGREGATE", + /* 180 */ "BUFSIZE", + /* 181 */ "STREAM", + /* 182 */ "INTO", + /* 183 */ "TRIGGER", + /* 184 */ "AT_ONCE", + /* 185 */ "WINDOW_CLOSE", + /* 186 */ "IGNORE", + /* 187 */ "EXPIRED", + /* 188 */ "KILL", + /* 189 */ "CONNECTION", + /* 190 */ "TRANSACTION", + /* 191 */ "BALANCE", + /* 192 */ "VGROUP", + /* 193 */ "MERGE", + /* 194 */ "REDISTRIBUTE", + /* 195 */ "SPLIT", + /* 196 */ "DELETE", + /* 197 */ "INSERT", + /* 198 */ "NULL", + /* 199 */ "NK_QUESTION", + /* 200 */ "NK_ARROW", + /* 201 */ "ROWTS", + /* 202 */ "TBNAME", + /* 203 */ "QSTART", + /* 204 */ "QEND", + /* 205 */ "QDURATION", + /* 206 */ "WSTART", + /* 207 */ "WEND", + /* 208 */ "WDURATION", + /* 209 */ "CAST", + /* 210 */ "NOW", + /* 211 */ "TODAY", + /* 212 */ "TIMEZONE", + /* 213 */ "CLIENT_VERSION", + /* 214 */ "SERVER_VERSION", + /* 215 */ "SERVER_STATUS", + /* 216 */ "CURRENT_USER", + /* 217 */ "COUNT", + /* 218 */ "LAST_ROW", + /* 219 */ "BETWEEN", + /* 220 */ "IS", + /* 221 */ "NK_LT", + /* 222 */ "NK_GT", + /* 223 */ "NK_LE", + /* 224 */ "NK_GE", + /* 225 */ "NK_NE", + /* 226 */ "MATCH", + /* 227 */ "NMATCH", + /* 228 */ "CONTAINS", + /* 229 */ "IN", + /* 230 */ "JOIN", + /* 231 */ "INNER", + /* 232 */ "SELECT", + /* 233 */ "DISTINCT", + /* 234 */ "WHERE", + /* 235 */ "PARTITION", + /* 236 */ "BY", + /* 237 */ "SESSION", + /* 238 */ "STATE_WINDOW", + /* 239 */ "SLIDING", + /* 240 */ "FILL", + /* 241 */ "VALUE", + /* 242 */ "NONE", + /* 243 */ "PREV", + /* 244 */ "LINEAR", + /* 245 */ "NEXT", + /* 246 */ "HAVING", + /* 247 */ "RANGE", + /* 248 */ "EVERY", + /* 249 */ "ORDER", + /* 250 */ "SLIMIT", + /* 251 */ "SOFFSET", + /* 252 */ "LIMIT", + /* 253 */ "OFFSET", + /* 254 */ "ASC", + /* 255 */ "NULLS", + /* 256 */ "ABORT", + /* 257 */ "AFTER", + /* 258 */ "ATTACH", + /* 259 */ "BEFORE", + /* 260 */ "BEGIN", + /* 261 */ "BITAND", + /* 262 */ "BITNOT", + /* 263 */ "BITOR", + /* 264 */ "BLOCKS", + /* 265 */ "CHANGE", + /* 266 */ "COMMA", + /* 267 */ "COMPACT", + /* 268 */ "CONCAT", + /* 269 */ "CONFLICT", + /* 270 */ "COPY", + /* 271 */ "DEFERRED", + /* 272 */ "DELIMITERS", + /* 273 */ "DETACH", + /* 274 */ "DIVIDE", + /* 275 */ "DOT", + /* 276 */ "EACH", + /* 277 */ "END", + /* 278 */ "FAIL", + /* 279 */ "FILE", + /* 280 */ "FOR", + /* 281 */ "GLOB", + /* 282 */ "ID", + /* 283 */ "IMMEDIATE", + /* 284 */ "IMPORT", + /* 285 */ "INITIALLY", + /* 286 */ "INSTEAD", + /* 287 */ "ISNULL", + /* 288 */ "KEY", + /* 289 */ "NK_BITNOT", + /* 290 */ "NK_SEMI", + /* 291 */ "NOTNULL", + /* 292 */ "OF", + /* 293 */ "PLUS", + /* 294 */ "PRIVILEGE", + /* 295 */ "RAISE", + /* 296 */ "REPLACE", + /* 297 */ "RESTRICT", + /* 298 */ "ROW", + /* 299 */ "SEMI", + /* 300 */ "STAR", + /* 301 */ "STATEMENT", + /* 302 */ "STRING", + /* 303 */ "TIMES", + /* 304 */ "UPDATE", + /* 305 */ "VALUES", + /* 306 */ "VARIABLE", + /* 307 */ "VIEW", + /* 308 */ "WAL", + /* 309 */ "cmd", + /* 310 */ "account_options", + /* 311 */ "alter_account_options", + /* 312 */ "literal", + /* 313 */ "alter_account_option", + /* 314 */ "user_name", + /* 315 */ "sysinfo_opt", + /* 316 */ "privileges", + /* 317 */ "priv_level", + /* 318 */ "priv_type_list", + /* 319 */ "priv_type", + /* 320 */ "db_name", + /* 321 */ "dnode_endpoint", + /* 322 */ "not_exists_opt", + /* 323 */ "db_options", + /* 324 */ "exists_opt", + /* 325 */ "alter_db_options", + /* 326 */ "integer_list", + /* 327 */ "variable_list", + /* 328 */ "retention_list", + /* 329 */ "alter_db_option", + /* 330 */ "retention", + /* 331 */ "full_table_name", + /* 332 */ "column_def_list", + /* 333 */ "tags_def_opt", + /* 334 */ "table_options", + /* 335 */ "multi_create_clause", + /* 336 */ "tags_def", + /* 337 */ "multi_drop_clause", + /* 338 */ "alter_table_clause", + /* 339 */ "alter_table_options", + /* 340 */ "column_name", + /* 341 */ "type_name", + /* 342 */ "signed_literal", + /* 343 */ "create_subtable_clause", + /* 344 */ "specific_cols_opt", + /* 345 */ "expression_list", + /* 346 */ "drop_table_clause", + /* 347 */ "col_name_list", + /* 348 */ "table_name", + /* 349 */ "column_def", + /* 350 */ "duration_list", + /* 351 */ "rollup_func_list", + /* 352 */ "alter_table_option", + /* 353 */ "duration_literal", + /* 354 */ "rollup_func_name", + /* 355 */ "function_name", + /* 356 */ "col_name", + /* 357 */ "db_name_cond_opt", + /* 358 */ "like_pattern_opt", + /* 359 */ "table_name_cond", + /* 360 */ "from_db_opt", + /* 361 */ "index_options", + /* 362 */ "func_list", + /* 363 */ "sliding_opt", + /* 364 */ "sma_stream_opt", + /* 365 */ "func", + /* 366 */ "stream_options", + /* 367 */ "topic_name", + /* 368 */ "query_expression", + /* 369 */ "cgroup_name", + /* 370 */ "analyze_opt", + /* 371 */ "explain_options", + /* 372 */ "agg_func_opt", + /* 373 */ "bufsize_opt", + /* 374 */ "stream_name", + /* 375 */ "dnode_list", + /* 376 */ "where_clause_opt", + /* 377 */ "signed", + /* 378 */ "literal_func", + /* 379 */ "literal_list", + /* 380 */ "table_alias", + /* 381 */ "column_alias", + /* 382 */ "expression", + /* 383 */ "pseudo_column", + /* 384 */ "column_reference", + /* 385 */ "function_expression", + /* 386 */ "subquery", + /* 387 */ "star_func", + /* 388 */ "star_func_para_list", + /* 389 */ "noarg_func", + /* 390 */ "other_para_list", + /* 391 */ "star_func_para", + /* 392 */ "predicate", + /* 393 */ "compare_op", + /* 394 */ "in_op", + /* 395 */ "in_predicate_value", + /* 396 */ "boolean_value_expression", + /* 397 */ "boolean_primary", + /* 398 */ "common_expression", + /* 399 */ "from_clause_opt", + /* 400 */ "table_reference_list", + /* 401 */ "table_reference", + /* 402 */ "table_primary", + /* 403 */ "joined_table", + /* 404 */ "alias_opt", + /* 405 */ "parenthesized_joined_table", + /* 406 */ "join_type", + /* 407 */ "search_condition", + /* 408 */ "query_specification", + /* 409 */ "set_quantifier_opt", + /* 410 */ "select_list", + /* 411 */ "partition_by_clause_opt", + /* 412 */ "range_opt", + /* 413 */ "every_opt", + /* 414 */ "fill_opt", + /* 415 */ "twindow_clause_opt", + /* 416 */ "group_by_clause_opt", + /* 417 */ "having_clause_opt", + /* 418 */ "select_item", + /* 419 */ "fill_mode", + /* 420 */ "group_by_list", + /* 421 */ "query_expression_body", + /* 422 */ "order_by_clause_opt", + /* 423 */ "slimit_clause_opt", + /* 424 */ "limit_clause_opt", + /* 425 */ "query_primary", + /* 426 */ "sort_specification_list", + /* 427 */ "sort_specification", + /* 428 */ "ordering_specification_opt", + /* 429 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1789,410 +1837,417 @@ static const char *const yyRuleName[] = { /* 82 */ "db_options ::= db_options KEEP variable_list", /* 83 */ "db_options ::= db_options PAGES NK_INTEGER", /* 84 */ "db_options ::= db_options PAGESIZE NK_INTEGER", - /* 85 */ "db_options ::= db_options PRECISION NK_STRING", - /* 86 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 87 */ "db_options ::= db_options STRICT NK_STRING", - /* 88 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 89 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 90 */ "db_options ::= db_options RETENTIONS retention_list", - /* 91 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 92 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", - /* 93 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", - /* 94 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", - /* 95 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 96 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", - /* 97 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 98 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", - /* 99 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", - /* 100 */ "alter_db_options ::= alter_db_option", - /* 101 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 102 */ "alter_db_option ::= CACHEMODEL NK_STRING", - /* 103 */ "alter_db_option ::= CACHESIZE NK_INTEGER", - /* 104 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", - /* 105 */ "alter_db_option ::= KEEP integer_list", - /* 106 */ "alter_db_option ::= KEEP variable_list", - /* 107 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", - /* 108 */ "integer_list ::= NK_INTEGER", - /* 109 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 110 */ "variable_list ::= NK_VARIABLE", - /* 111 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 112 */ "retention_list ::= retention", - /* 113 */ "retention_list ::= retention_list NK_COMMA retention", - /* 114 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 115 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 116 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 117 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 118 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 119 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 120 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 121 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 122 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 123 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 124 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 125 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 126 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 127 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 128 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 129 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 130 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 131 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 132 */ "multi_create_clause ::= create_subtable_clause", - /* 133 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 134 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", - /* 135 */ "multi_drop_clause ::= drop_table_clause", - /* 136 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 137 */ "drop_table_clause ::= exists_opt full_table_name", - /* 138 */ "specific_cols_opt ::=", - /* 139 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", - /* 140 */ "full_table_name ::= table_name", - /* 141 */ "full_table_name ::= db_name NK_DOT table_name", - /* 142 */ "column_def_list ::= column_def", - /* 143 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 144 */ "column_def ::= column_name type_name", - /* 145 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 146 */ "type_name ::= BOOL", - /* 147 */ "type_name ::= TINYINT", - /* 148 */ "type_name ::= SMALLINT", - /* 149 */ "type_name ::= INT", - /* 150 */ "type_name ::= INTEGER", - /* 151 */ "type_name ::= BIGINT", - /* 152 */ "type_name ::= FLOAT", - /* 153 */ "type_name ::= DOUBLE", - /* 154 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 155 */ "type_name ::= TIMESTAMP", - /* 156 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 157 */ "type_name ::= TINYINT UNSIGNED", - /* 158 */ "type_name ::= SMALLINT UNSIGNED", - /* 159 */ "type_name ::= INT UNSIGNED", - /* 160 */ "type_name ::= BIGINT UNSIGNED", - /* 161 */ "type_name ::= JSON", - /* 162 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 163 */ "type_name ::= MEDIUMBLOB", - /* 164 */ "type_name ::= BLOB", - /* 165 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 166 */ "type_name ::= DECIMAL", - /* 167 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 168 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 169 */ "tags_def_opt ::=", - /* 170 */ "tags_def_opt ::= tags_def", - /* 171 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 172 */ "table_options ::=", - /* 173 */ "table_options ::= table_options COMMENT NK_STRING", - /* 174 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 175 */ "table_options ::= table_options WATERMARK duration_list", - /* 176 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 177 */ "table_options ::= table_options TTL NK_INTEGER", - /* 178 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 179 */ "alter_table_options ::= alter_table_option", - /* 180 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 181 */ "alter_table_option ::= COMMENT NK_STRING", - /* 182 */ "alter_table_option ::= TTL NK_INTEGER", - /* 183 */ "duration_list ::= duration_literal", - /* 184 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 185 */ "rollup_func_list ::= rollup_func_name", - /* 186 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 187 */ "rollup_func_name ::= function_name", - /* 188 */ "rollup_func_name ::= FIRST", - /* 189 */ "rollup_func_name ::= LAST", - /* 190 */ "col_name_list ::= col_name", - /* 191 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 192 */ "col_name ::= column_name", - /* 193 */ "cmd ::= SHOW DNODES", - /* 194 */ "cmd ::= SHOW USERS", - /* 195 */ "cmd ::= SHOW DATABASES", - /* 196 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 197 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 198 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 199 */ "cmd ::= SHOW MNODES", - /* 200 */ "cmd ::= SHOW MODULES", - /* 201 */ "cmd ::= SHOW QNODES", - /* 202 */ "cmd ::= SHOW FUNCTIONS", - /* 203 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 204 */ "cmd ::= SHOW STREAMS", - /* 205 */ "cmd ::= SHOW ACCOUNTS", - /* 206 */ "cmd ::= SHOW APPS", - /* 207 */ "cmd ::= SHOW CONNECTIONS", - /* 208 */ "cmd ::= SHOW LICENCES", - /* 209 */ "cmd ::= SHOW GRANTS", - /* 210 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 211 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 212 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 213 */ "cmd ::= SHOW QUERIES", - /* 214 */ "cmd ::= SHOW SCORES", - /* 215 */ "cmd ::= SHOW TOPICS", - /* 216 */ "cmd ::= SHOW VARIABLES", - /* 217 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 218 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES", - /* 219 */ "cmd ::= SHOW BNODES", - /* 220 */ "cmd ::= SHOW SNODES", - /* 221 */ "cmd ::= SHOW CLUSTER", - /* 222 */ "cmd ::= SHOW TRANSACTIONS", - /* 223 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 224 */ "cmd ::= SHOW CONSUMERS", - /* 225 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 226 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 227 */ "db_name_cond_opt ::=", - /* 228 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 229 */ "like_pattern_opt ::=", - /* 230 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 231 */ "table_name_cond ::= table_name", - /* 232 */ "from_db_opt ::=", - /* 233 */ "from_db_opt ::= FROM db_name", - /* 234 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options", - /* 235 */ "cmd ::= DROP INDEX exists_opt full_table_name", - /* 236 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 237 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 238 */ "func_list ::= func", - /* 239 */ "func_list ::= func_list NK_COMMA func", - /* 240 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 241 */ "sma_stream_opt ::=", - /* 242 */ "sma_stream_opt ::= stream_options WATERMARK duration_literal", - /* 243 */ "sma_stream_opt ::= stream_options MAX_DELAY duration_literal", - /* 244 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", - /* 245 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 246 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 247 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 248 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 249 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 250 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 251 */ "cmd ::= DESC full_table_name", - /* 252 */ "cmd ::= DESCRIBE full_table_name", - /* 253 */ "cmd ::= RESET QUERY CACHE", - /* 254 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", - /* 255 */ "analyze_opt ::=", - /* 256 */ "analyze_opt ::= ANALYZE", - /* 257 */ "explain_options ::=", - /* 258 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 259 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 260 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 261 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 262 */ "agg_func_opt ::=", - /* 263 */ "agg_func_opt ::= AGGREGATE", - /* 264 */ "bufsize_opt ::=", - /* 265 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 266 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name AS query_expression", - /* 267 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 268 */ "stream_options ::=", - /* 269 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 270 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 271 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 272 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 273 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 274 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 275 */ "cmd ::= KILL QUERY NK_STRING", - /* 276 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 277 */ "cmd ::= BALANCE VGROUP", - /* 278 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 279 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 280 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 281 */ "dnode_list ::= DNODE NK_INTEGER", - /* 282 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 283 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 284 */ "cmd ::= query_expression", - /* 285 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_expression", - /* 286 */ "cmd ::= INSERT INTO full_table_name query_expression", - /* 287 */ "literal ::= NK_INTEGER", - /* 288 */ "literal ::= NK_FLOAT", - /* 289 */ "literal ::= NK_STRING", - /* 290 */ "literal ::= NK_BOOL", - /* 291 */ "literal ::= TIMESTAMP NK_STRING", - /* 292 */ "literal ::= duration_literal", - /* 293 */ "literal ::= NULL", - /* 294 */ "literal ::= NK_QUESTION", - /* 295 */ "duration_literal ::= NK_VARIABLE", - /* 296 */ "signed ::= NK_INTEGER", - /* 297 */ "signed ::= NK_PLUS NK_INTEGER", - /* 298 */ "signed ::= NK_MINUS NK_INTEGER", - /* 299 */ "signed ::= NK_FLOAT", - /* 300 */ "signed ::= NK_PLUS NK_FLOAT", - /* 301 */ "signed ::= NK_MINUS NK_FLOAT", - /* 302 */ "signed_literal ::= signed", - /* 303 */ "signed_literal ::= NK_STRING", - /* 304 */ "signed_literal ::= NK_BOOL", - /* 305 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 306 */ "signed_literal ::= duration_literal", - /* 307 */ "signed_literal ::= NULL", - /* 308 */ "signed_literal ::= literal_func", - /* 309 */ "signed_literal ::= NK_QUESTION", - /* 310 */ "literal_list ::= signed_literal", - /* 311 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 312 */ "db_name ::= NK_ID", - /* 313 */ "table_name ::= NK_ID", - /* 314 */ "column_name ::= NK_ID", - /* 315 */ "function_name ::= NK_ID", - /* 316 */ "table_alias ::= NK_ID", - /* 317 */ "column_alias ::= NK_ID", - /* 318 */ "user_name ::= NK_ID", - /* 319 */ "topic_name ::= NK_ID", - /* 320 */ "stream_name ::= NK_ID", - /* 321 */ "cgroup_name ::= NK_ID", - /* 322 */ "expression ::= literal", - /* 323 */ "expression ::= pseudo_column", - /* 324 */ "expression ::= column_reference", - /* 325 */ "expression ::= function_expression", - /* 326 */ "expression ::= subquery", - /* 327 */ "expression ::= NK_LP expression NK_RP", - /* 328 */ "expression ::= NK_PLUS expression", - /* 329 */ "expression ::= NK_MINUS expression", - /* 330 */ "expression ::= expression NK_PLUS expression", - /* 331 */ "expression ::= expression NK_MINUS expression", - /* 332 */ "expression ::= expression NK_STAR expression", - /* 333 */ "expression ::= expression NK_SLASH expression", - /* 334 */ "expression ::= expression NK_REM expression", - /* 335 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 336 */ "expression ::= expression NK_BITAND expression", - /* 337 */ "expression ::= expression NK_BITOR expression", - /* 338 */ "expression_list ::= expression", - /* 339 */ "expression_list ::= expression_list NK_COMMA expression", - /* 340 */ "column_reference ::= column_name", - /* 341 */ "column_reference ::= table_name NK_DOT column_name", - /* 342 */ "pseudo_column ::= ROWTS", - /* 343 */ "pseudo_column ::= TBNAME", - /* 344 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 345 */ "pseudo_column ::= QSTART", - /* 346 */ "pseudo_column ::= QEND", - /* 347 */ "pseudo_column ::= QDURATION", - /* 348 */ "pseudo_column ::= WSTART", - /* 349 */ "pseudo_column ::= WEND", - /* 350 */ "pseudo_column ::= WDURATION", - /* 351 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 352 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 353 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 354 */ "function_expression ::= literal_func", - /* 355 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 356 */ "literal_func ::= NOW", - /* 357 */ "noarg_func ::= NOW", - /* 358 */ "noarg_func ::= TODAY", - /* 359 */ "noarg_func ::= TIMEZONE", - /* 360 */ "noarg_func ::= DATABASE", - /* 361 */ "noarg_func ::= CLIENT_VERSION", - /* 362 */ "noarg_func ::= SERVER_VERSION", - /* 363 */ "noarg_func ::= SERVER_STATUS", - /* 364 */ "noarg_func ::= CURRENT_USER", - /* 365 */ "noarg_func ::= USER", - /* 366 */ "star_func ::= COUNT", - /* 367 */ "star_func ::= FIRST", - /* 368 */ "star_func ::= LAST", - /* 369 */ "star_func ::= LAST_ROW", - /* 370 */ "star_func_para_list ::= NK_STAR", - /* 371 */ "star_func_para_list ::= other_para_list", - /* 372 */ "other_para_list ::= star_func_para", - /* 373 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 374 */ "star_func_para ::= expression", - /* 375 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 376 */ "predicate ::= expression compare_op expression", - /* 377 */ "predicate ::= expression BETWEEN expression AND expression", - /* 378 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 379 */ "predicate ::= expression IS NULL", - /* 380 */ "predicate ::= expression IS NOT NULL", - /* 381 */ "predicate ::= expression in_op in_predicate_value", - /* 382 */ "compare_op ::= NK_LT", - /* 383 */ "compare_op ::= NK_GT", - /* 384 */ "compare_op ::= NK_LE", - /* 385 */ "compare_op ::= NK_GE", - /* 386 */ "compare_op ::= NK_NE", - /* 387 */ "compare_op ::= NK_EQ", - /* 388 */ "compare_op ::= LIKE", - /* 389 */ "compare_op ::= NOT LIKE", - /* 390 */ "compare_op ::= MATCH", - /* 391 */ "compare_op ::= NMATCH", - /* 392 */ "compare_op ::= CONTAINS", - /* 393 */ "in_op ::= IN", - /* 394 */ "in_op ::= NOT IN", - /* 395 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 396 */ "boolean_value_expression ::= boolean_primary", - /* 397 */ "boolean_value_expression ::= NOT boolean_primary", - /* 398 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 399 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 400 */ "boolean_primary ::= predicate", - /* 401 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 402 */ "common_expression ::= expression", - /* 403 */ "common_expression ::= boolean_value_expression", - /* 404 */ "from_clause_opt ::=", - /* 405 */ "from_clause_opt ::= FROM table_reference_list", - /* 406 */ "table_reference_list ::= table_reference", - /* 407 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 408 */ "table_reference ::= table_primary", - /* 409 */ "table_reference ::= joined_table", - /* 410 */ "table_primary ::= table_name alias_opt", - /* 411 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 412 */ "table_primary ::= subquery alias_opt", - /* 413 */ "table_primary ::= parenthesized_joined_table", - /* 414 */ "alias_opt ::=", - /* 415 */ "alias_opt ::= table_alias", - /* 416 */ "alias_opt ::= AS table_alias", - /* 417 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 418 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 419 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 420 */ "join_type ::=", - /* 421 */ "join_type ::= INNER", - /* 422 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 423 */ "set_quantifier_opt ::=", - /* 424 */ "set_quantifier_opt ::= DISTINCT", - /* 425 */ "set_quantifier_opt ::= ALL", - /* 426 */ "select_list ::= select_item", - /* 427 */ "select_list ::= select_list NK_COMMA select_item", - /* 428 */ "select_item ::= NK_STAR", - /* 429 */ "select_item ::= common_expression", - /* 430 */ "select_item ::= common_expression column_alias", - /* 431 */ "select_item ::= common_expression AS column_alias", - /* 432 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 433 */ "where_clause_opt ::=", - /* 434 */ "where_clause_opt ::= WHERE search_condition", - /* 435 */ "partition_by_clause_opt ::=", - /* 436 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 437 */ "twindow_clause_opt ::=", - /* 438 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 439 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 440 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 441 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 442 */ "sliding_opt ::=", - /* 443 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 444 */ "fill_opt ::=", - /* 445 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 446 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 447 */ "fill_mode ::= NONE", - /* 448 */ "fill_mode ::= PREV", - /* 449 */ "fill_mode ::= NULL", - /* 450 */ "fill_mode ::= LINEAR", - /* 451 */ "fill_mode ::= NEXT", - /* 452 */ "group_by_clause_opt ::=", - /* 453 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 454 */ "group_by_list ::= expression", - /* 455 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 456 */ "having_clause_opt ::=", - /* 457 */ "having_clause_opt ::= HAVING search_condition", - /* 458 */ "range_opt ::=", - /* 459 */ "range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP", - /* 460 */ "every_opt ::=", - /* 461 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 462 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 463 */ "query_expression_body ::= query_primary", - /* 464 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 465 */ "query_expression_body ::= query_expression_body UNION query_expression_body", - /* 466 */ "query_primary ::= query_specification", - /* 467 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", - /* 468 */ "order_by_clause_opt ::=", - /* 469 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 470 */ "slimit_clause_opt ::=", - /* 471 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 472 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 473 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 474 */ "limit_clause_opt ::=", - /* 475 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 476 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 477 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 478 */ "subquery ::= NK_LP query_expression NK_RP", - /* 479 */ "search_condition ::= common_expression", - /* 480 */ "sort_specification_list ::= sort_specification", - /* 481 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 482 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 483 */ "ordering_specification_opt ::=", - /* 484 */ "ordering_specification_opt ::= ASC", - /* 485 */ "ordering_specification_opt ::= DESC", - /* 486 */ "null_ordering_opt ::=", - /* 487 */ "null_ordering_opt ::= NULLS FIRST", - /* 488 */ "null_ordering_opt ::= NULLS LAST", + /* 85 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", + /* 86 */ "db_options ::= db_options PRECISION NK_STRING", + /* 87 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 88 */ "db_options ::= db_options STRICT NK_STRING", + /* 89 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 90 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 91 */ "db_options ::= db_options RETENTIONS retention_list", + /* 92 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 93 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", + /* 94 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", + /* 95 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", + /* 96 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 97 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", + /* 98 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 99 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", + /* 100 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", + /* 101 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", + /* 102 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", + /* 103 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 104 */ "alter_db_options ::= alter_db_option", + /* 105 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 106 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 107 */ "alter_db_option ::= CACHESIZE NK_INTEGER", + /* 108 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", + /* 109 */ "alter_db_option ::= KEEP integer_list", + /* 110 */ "alter_db_option ::= KEEP variable_list", + /* 111 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", + /* 112 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", + /* 113 */ "integer_list ::= NK_INTEGER", + /* 114 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 115 */ "variable_list ::= NK_VARIABLE", + /* 116 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 117 */ "retention_list ::= retention", + /* 118 */ "retention_list ::= retention_list NK_COMMA retention", + /* 119 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 120 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 121 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 122 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 123 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 124 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 125 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 126 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 127 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 128 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 129 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 130 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 131 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 132 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 133 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 134 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 135 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 136 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 137 */ "multi_create_clause ::= create_subtable_clause", + /* 138 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 139 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", + /* 140 */ "multi_drop_clause ::= drop_table_clause", + /* 141 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 142 */ "drop_table_clause ::= exists_opt full_table_name", + /* 143 */ "specific_cols_opt ::=", + /* 144 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", + /* 145 */ "full_table_name ::= table_name", + /* 146 */ "full_table_name ::= db_name NK_DOT table_name", + /* 147 */ "column_def_list ::= column_def", + /* 148 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 149 */ "column_def ::= column_name type_name", + /* 150 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 151 */ "type_name ::= BOOL", + /* 152 */ "type_name ::= TINYINT", + /* 153 */ "type_name ::= SMALLINT", + /* 154 */ "type_name ::= INT", + /* 155 */ "type_name ::= INTEGER", + /* 156 */ "type_name ::= BIGINT", + /* 157 */ "type_name ::= FLOAT", + /* 158 */ "type_name ::= DOUBLE", + /* 159 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 160 */ "type_name ::= TIMESTAMP", + /* 161 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 162 */ "type_name ::= TINYINT UNSIGNED", + /* 163 */ "type_name ::= SMALLINT UNSIGNED", + /* 164 */ "type_name ::= INT UNSIGNED", + /* 165 */ "type_name ::= BIGINT UNSIGNED", + /* 166 */ "type_name ::= JSON", + /* 167 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 168 */ "type_name ::= MEDIUMBLOB", + /* 169 */ "type_name ::= BLOB", + /* 170 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 171 */ "type_name ::= DECIMAL", + /* 172 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 173 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 174 */ "tags_def_opt ::=", + /* 175 */ "tags_def_opt ::= tags_def", + /* 176 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 177 */ "table_options ::=", + /* 178 */ "table_options ::= table_options COMMENT NK_STRING", + /* 179 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 180 */ "table_options ::= table_options WATERMARK duration_list", + /* 181 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 182 */ "table_options ::= table_options TTL NK_INTEGER", + /* 183 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 184 */ "alter_table_options ::= alter_table_option", + /* 185 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 186 */ "alter_table_option ::= COMMENT NK_STRING", + /* 187 */ "alter_table_option ::= TTL NK_INTEGER", + /* 188 */ "duration_list ::= duration_literal", + /* 189 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 190 */ "rollup_func_list ::= rollup_func_name", + /* 191 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 192 */ "rollup_func_name ::= function_name", + /* 193 */ "rollup_func_name ::= FIRST", + /* 194 */ "rollup_func_name ::= LAST", + /* 195 */ "col_name_list ::= col_name", + /* 196 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 197 */ "col_name ::= column_name", + /* 198 */ "cmd ::= SHOW DNODES", + /* 199 */ "cmd ::= SHOW USERS", + /* 200 */ "cmd ::= SHOW DATABASES", + /* 201 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 202 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 203 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 204 */ "cmd ::= SHOW MNODES", + /* 205 */ "cmd ::= SHOW MODULES", + /* 206 */ "cmd ::= SHOW QNODES", + /* 207 */ "cmd ::= SHOW FUNCTIONS", + /* 208 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 209 */ "cmd ::= SHOW STREAMS", + /* 210 */ "cmd ::= SHOW ACCOUNTS", + /* 211 */ "cmd ::= SHOW APPS", + /* 212 */ "cmd ::= SHOW CONNECTIONS", + /* 213 */ "cmd ::= SHOW LICENCES", + /* 214 */ "cmd ::= SHOW GRANTS", + /* 215 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 216 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 217 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 218 */ "cmd ::= SHOW QUERIES", + /* 219 */ "cmd ::= SHOW SCORES", + /* 220 */ "cmd ::= SHOW TOPICS", + /* 221 */ "cmd ::= SHOW VARIABLES", + /* 222 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 223 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES", + /* 224 */ "cmd ::= SHOW BNODES", + /* 225 */ "cmd ::= SHOW SNODES", + /* 226 */ "cmd ::= SHOW CLUSTER", + /* 227 */ "cmd ::= SHOW TRANSACTIONS", + /* 228 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 229 */ "cmd ::= SHOW CONSUMERS", + /* 230 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 231 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 232 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 233 */ "cmd ::= SHOW VNODES NK_STRING", + /* 234 */ "db_name_cond_opt ::=", + /* 235 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 236 */ "like_pattern_opt ::=", + /* 237 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 238 */ "table_name_cond ::= table_name", + /* 239 */ "from_db_opt ::=", + /* 240 */ "from_db_opt ::= FROM db_name", + /* 241 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options", + /* 242 */ "cmd ::= DROP INDEX exists_opt full_table_name", + /* 243 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 244 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 245 */ "func_list ::= func", + /* 246 */ "func_list ::= func_list NK_COMMA func", + /* 247 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 248 */ "sma_stream_opt ::=", + /* 249 */ "sma_stream_opt ::= stream_options WATERMARK duration_literal", + /* 250 */ "sma_stream_opt ::= stream_options MAX_DELAY duration_literal", + /* 251 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", + /* 252 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 253 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 254 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 255 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 256 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 257 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 258 */ "cmd ::= DESC full_table_name", + /* 259 */ "cmd ::= DESCRIBE full_table_name", + /* 260 */ "cmd ::= RESET QUERY CACHE", + /* 261 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", + /* 262 */ "analyze_opt ::=", + /* 263 */ "analyze_opt ::= ANALYZE", + /* 264 */ "explain_options ::=", + /* 265 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 266 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 267 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 268 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 269 */ "agg_func_opt ::=", + /* 270 */ "agg_func_opt ::= AGGREGATE", + /* 271 */ "bufsize_opt ::=", + /* 272 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 273 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name AS query_expression", + /* 274 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 275 */ "stream_options ::=", + /* 276 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 277 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 278 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 279 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 280 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 281 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 282 */ "cmd ::= KILL QUERY NK_STRING", + /* 283 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 284 */ "cmd ::= BALANCE VGROUP", + /* 285 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 286 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 287 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 288 */ "dnode_list ::= DNODE NK_INTEGER", + /* 289 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 290 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 291 */ "cmd ::= query_expression", + /* 292 */ "cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_expression", + /* 293 */ "cmd ::= INSERT INTO full_table_name query_expression", + /* 294 */ "literal ::= NK_INTEGER", + /* 295 */ "literal ::= NK_FLOAT", + /* 296 */ "literal ::= NK_STRING", + /* 297 */ "literal ::= NK_BOOL", + /* 298 */ "literal ::= TIMESTAMP NK_STRING", + /* 299 */ "literal ::= duration_literal", + /* 300 */ "literal ::= NULL", + /* 301 */ "literal ::= NK_QUESTION", + /* 302 */ "duration_literal ::= NK_VARIABLE", + /* 303 */ "signed ::= NK_INTEGER", + /* 304 */ "signed ::= NK_PLUS NK_INTEGER", + /* 305 */ "signed ::= NK_MINUS NK_INTEGER", + /* 306 */ "signed ::= NK_FLOAT", + /* 307 */ "signed ::= NK_PLUS NK_FLOAT", + /* 308 */ "signed ::= NK_MINUS NK_FLOAT", + /* 309 */ "signed_literal ::= signed", + /* 310 */ "signed_literal ::= NK_STRING", + /* 311 */ "signed_literal ::= NK_BOOL", + /* 312 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 313 */ "signed_literal ::= duration_literal", + /* 314 */ "signed_literal ::= NULL", + /* 315 */ "signed_literal ::= literal_func", + /* 316 */ "signed_literal ::= NK_QUESTION", + /* 317 */ "literal_list ::= signed_literal", + /* 318 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 319 */ "db_name ::= NK_ID", + /* 320 */ "table_name ::= NK_ID", + /* 321 */ "column_name ::= NK_ID", + /* 322 */ "function_name ::= NK_ID", + /* 323 */ "table_alias ::= NK_ID", + /* 324 */ "column_alias ::= NK_ID", + /* 325 */ "user_name ::= NK_ID", + /* 326 */ "topic_name ::= NK_ID", + /* 327 */ "stream_name ::= NK_ID", + /* 328 */ "cgroup_name ::= NK_ID", + /* 329 */ "expression ::= literal", + /* 330 */ "expression ::= pseudo_column", + /* 331 */ "expression ::= column_reference", + /* 332 */ "expression ::= function_expression", + /* 333 */ "expression ::= subquery", + /* 334 */ "expression ::= NK_LP expression NK_RP", + /* 335 */ "expression ::= NK_PLUS expression", + /* 336 */ "expression ::= NK_MINUS expression", + /* 337 */ "expression ::= expression NK_PLUS expression", + /* 338 */ "expression ::= expression NK_MINUS expression", + /* 339 */ "expression ::= expression NK_STAR expression", + /* 340 */ "expression ::= expression NK_SLASH expression", + /* 341 */ "expression ::= expression NK_REM expression", + /* 342 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 343 */ "expression ::= expression NK_BITAND expression", + /* 344 */ "expression ::= expression NK_BITOR expression", + /* 345 */ "expression_list ::= expression", + /* 346 */ "expression_list ::= expression_list NK_COMMA expression", + /* 347 */ "column_reference ::= column_name", + /* 348 */ "column_reference ::= table_name NK_DOT column_name", + /* 349 */ "pseudo_column ::= ROWTS", + /* 350 */ "pseudo_column ::= TBNAME", + /* 351 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 352 */ "pseudo_column ::= QSTART", + /* 353 */ "pseudo_column ::= QEND", + /* 354 */ "pseudo_column ::= QDURATION", + /* 355 */ "pseudo_column ::= WSTART", + /* 356 */ "pseudo_column ::= WEND", + /* 357 */ "pseudo_column ::= WDURATION", + /* 358 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 359 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 360 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 361 */ "function_expression ::= literal_func", + /* 362 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 363 */ "literal_func ::= NOW", + /* 364 */ "noarg_func ::= NOW", + /* 365 */ "noarg_func ::= TODAY", + /* 366 */ "noarg_func ::= TIMEZONE", + /* 367 */ "noarg_func ::= DATABASE", + /* 368 */ "noarg_func ::= CLIENT_VERSION", + /* 369 */ "noarg_func ::= SERVER_VERSION", + /* 370 */ "noarg_func ::= SERVER_STATUS", + /* 371 */ "noarg_func ::= CURRENT_USER", + /* 372 */ "noarg_func ::= USER", + /* 373 */ "star_func ::= COUNT", + /* 374 */ "star_func ::= FIRST", + /* 375 */ "star_func ::= LAST", + /* 376 */ "star_func ::= LAST_ROW", + /* 377 */ "star_func_para_list ::= NK_STAR", + /* 378 */ "star_func_para_list ::= other_para_list", + /* 379 */ "other_para_list ::= star_func_para", + /* 380 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 381 */ "star_func_para ::= expression", + /* 382 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 383 */ "predicate ::= expression compare_op expression", + /* 384 */ "predicate ::= expression BETWEEN expression AND expression", + /* 385 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 386 */ "predicate ::= expression IS NULL", + /* 387 */ "predicate ::= expression IS NOT NULL", + /* 388 */ "predicate ::= expression in_op in_predicate_value", + /* 389 */ "compare_op ::= NK_LT", + /* 390 */ "compare_op ::= NK_GT", + /* 391 */ "compare_op ::= NK_LE", + /* 392 */ "compare_op ::= NK_GE", + /* 393 */ "compare_op ::= NK_NE", + /* 394 */ "compare_op ::= NK_EQ", + /* 395 */ "compare_op ::= LIKE", + /* 396 */ "compare_op ::= NOT LIKE", + /* 397 */ "compare_op ::= MATCH", + /* 398 */ "compare_op ::= NMATCH", + /* 399 */ "compare_op ::= CONTAINS", + /* 400 */ "in_op ::= IN", + /* 401 */ "in_op ::= NOT IN", + /* 402 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 403 */ "boolean_value_expression ::= boolean_primary", + /* 404 */ "boolean_value_expression ::= NOT boolean_primary", + /* 405 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 406 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 407 */ "boolean_primary ::= predicate", + /* 408 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 409 */ "common_expression ::= expression", + /* 410 */ "common_expression ::= boolean_value_expression", + /* 411 */ "from_clause_opt ::=", + /* 412 */ "from_clause_opt ::= FROM table_reference_list", + /* 413 */ "table_reference_list ::= table_reference", + /* 414 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 415 */ "table_reference ::= table_primary", + /* 416 */ "table_reference ::= joined_table", + /* 417 */ "table_primary ::= table_name alias_opt", + /* 418 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 419 */ "table_primary ::= subquery alias_opt", + /* 420 */ "table_primary ::= parenthesized_joined_table", + /* 421 */ "alias_opt ::=", + /* 422 */ "alias_opt ::= table_alias", + /* 423 */ "alias_opt ::= AS table_alias", + /* 424 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 425 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 426 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 427 */ "join_type ::=", + /* 428 */ "join_type ::= INNER", + /* 429 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 430 */ "set_quantifier_opt ::=", + /* 431 */ "set_quantifier_opt ::= DISTINCT", + /* 432 */ "set_quantifier_opt ::= ALL", + /* 433 */ "select_list ::= select_item", + /* 434 */ "select_list ::= select_list NK_COMMA select_item", + /* 435 */ "select_item ::= NK_STAR", + /* 436 */ "select_item ::= common_expression", + /* 437 */ "select_item ::= common_expression column_alias", + /* 438 */ "select_item ::= common_expression AS column_alias", + /* 439 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 440 */ "where_clause_opt ::=", + /* 441 */ "where_clause_opt ::= WHERE search_condition", + /* 442 */ "partition_by_clause_opt ::=", + /* 443 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 444 */ "twindow_clause_opt ::=", + /* 445 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 446 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 447 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 448 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 449 */ "sliding_opt ::=", + /* 450 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 451 */ "fill_opt ::=", + /* 452 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 453 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 454 */ "fill_mode ::= NONE", + /* 455 */ "fill_mode ::= PREV", + /* 456 */ "fill_mode ::= NULL", + /* 457 */ "fill_mode ::= LINEAR", + /* 458 */ "fill_mode ::= NEXT", + /* 459 */ "group_by_clause_opt ::=", + /* 460 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 461 */ "group_by_list ::= expression", + /* 462 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 463 */ "having_clause_opt ::=", + /* 464 */ "having_clause_opt ::= HAVING search_condition", + /* 465 */ "range_opt ::=", + /* 466 */ "range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP", + /* 467 */ "every_opt ::=", + /* 468 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 469 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 470 */ "query_expression_body ::= query_primary", + /* 471 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 472 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 473 */ "query_primary ::= query_specification", + /* 474 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", + /* 475 */ "order_by_clause_opt ::=", + /* 476 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 477 */ "slimit_clause_opt ::=", + /* 478 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 479 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 480 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 481 */ "limit_clause_opt ::=", + /* 482 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 483 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 484 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 485 */ "subquery ::= NK_LP query_expression NK_RP", + /* 486 */ "search_condition ::= common_expression", + /* 487 */ "sort_specification_list ::= sort_specification", + /* 488 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 489 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 490 */ "ordering_specification_opt ::=", + /* 491 */ "ordering_specification_opt ::= ASC", + /* 492 */ "ordering_specification_opt ::= DESC", + /* 493 */ "null_ordering_opt ::=", + /* 494 */ "null_ordering_opt ::= NULLS FIRST", + /* 495 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2319,179 +2374,179 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 305: /* cmd */ - case 308: /* literal */ - case 319: /* db_options */ - case 321: /* alter_db_options */ - case 326: /* retention */ - case 327: /* full_table_name */ - case 330: /* table_options */ - case 334: /* alter_table_clause */ - case 335: /* alter_table_options */ - case 338: /* signed_literal */ - case 339: /* create_subtable_clause */ - case 342: /* drop_table_clause */ - case 345: /* column_def */ - case 349: /* duration_literal */ - case 350: /* rollup_func_name */ - case 352: /* col_name */ - case 353: /* db_name_cond_opt */ - case 354: /* like_pattern_opt */ - case 355: /* table_name_cond */ - case 356: /* from_db_opt */ - case 357: /* index_options */ - case 359: /* sliding_opt */ - case 360: /* sma_stream_opt */ - case 361: /* func */ - case 362: /* stream_options */ - case 364: /* query_expression */ - case 367: /* explain_options */ - case 372: /* where_clause_opt */ - case 373: /* signed */ - case 374: /* literal_func */ - case 378: /* expression */ - case 379: /* pseudo_column */ - case 380: /* column_reference */ - case 381: /* function_expression */ - case 382: /* subquery */ - case 387: /* star_func_para */ - case 388: /* predicate */ - case 391: /* in_predicate_value */ - case 392: /* boolean_value_expression */ - case 393: /* boolean_primary */ - case 394: /* common_expression */ - case 395: /* from_clause_opt */ - case 396: /* table_reference_list */ - case 397: /* table_reference */ - case 398: /* table_primary */ - case 399: /* joined_table */ - case 401: /* parenthesized_joined_table */ - case 403: /* search_condition */ - case 404: /* query_specification */ - case 408: /* range_opt */ - case 409: /* every_opt */ - case 410: /* fill_opt */ - case 411: /* twindow_clause_opt */ - case 413: /* having_clause_opt */ - case 414: /* select_item */ - case 417: /* query_expression_body */ - case 419: /* slimit_clause_opt */ - case 420: /* limit_clause_opt */ - case 421: /* query_primary */ - case 423: /* sort_specification */ + case 309: /* cmd */ + case 312: /* literal */ + case 323: /* db_options */ + case 325: /* alter_db_options */ + case 330: /* retention */ + case 331: /* full_table_name */ + case 334: /* table_options */ + case 338: /* alter_table_clause */ + case 339: /* alter_table_options */ + case 342: /* signed_literal */ + case 343: /* create_subtable_clause */ + case 346: /* drop_table_clause */ + case 349: /* column_def */ + case 353: /* duration_literal */ + case 354: /* rollup_func_name */ + case 356: /* col_name */ + case 357: /* db_name_cond_opt */ + case 358: /* like_pattern_opt */ + case 359: /* table_name_cond */ + case 360: /* from_db_opt */ + case 361: /* index_options */ + case 363: /* sliding_opt */ + case 364: /* sma_stream_opt */ + case 365: /* func */ + case 366: /* stream_options */ + case 368: /* query_expression */ + case 371: /* explain_options */ + case 376: /* where_clause_opt */ + case 377: /* signed */ + case 378: /* literal_func */ + case 382: /* expression */ + case 383: /* pseudo_column */ + case 384: /* column_reference */ + case 385: /* function_expression */ + case 386: /* subquery */ + case 391: /* star_func_para */ + case 392: /* predicate */ + case 395: /* in_predicate_value */ + case 396: /* boolean_value_expression */ + case 397: /* boolean_primary */ + case 398: /* common_expression */ + case 399: /* from_clause_opt */ + case 400: /* table_reference_list */ + case 401: /* table_reference */ + case 402: /* table_primary */ + case 403: /* joined_table */ + case 405: /* parenthesized_joined_table */ + case 407: /* search_condition */ + case 408: /* query_specification */ + case 412: /* range_opt */ + case 413: /* every_opt */ + case 414: /* fill_opt */ + case 415: /* twindow_clause_opt */ + case 417: /* having_clause_opt */ + case 418: /* select_item */ + case 421: /* query_expression_body */ + case 423: /* slimit_clause_opt */ + case 424: /* limit_clause_opt */ + case 425: /* query_primary */ + case 427: /* sort_specification */ { - nodesDestroyNode((yypminor->yy272)); + nodesDestroyNode((yypminor->yy312)); } break; - case 306: /* account_options */ - case 307: /* alter_account_options */ - case 309: /* alter_account_option */ - case 369: /* bufsize_opt */ + case 310: /* account_options */ + case 311: /* alter_account_options */ + case 313: /* alter_account_option */ + case 373: /* bufsize_opt */ { } break; - case 310: /* user_name */ - case 313: /* priv_level */ - case 316: /* db_name */ - case 317: /* dnode_endpoint */ - case 336: /* column_name */ - case 344: /* table_name */ - case 351: /* function_name */ - case 363: /* topic_name */ - case 365: /* cgroup_name */ - case 370: /* stream_name */ - case 376: /* table_alias */ - case 377: /* column_alias */ - case 383: /* star_func */ - case 385: /* noarg_func */ - case 400: /* alias_opt */ + case 314: /* user_name */ + case 317: /* priv_level */ + case 320: /* db_name */ + case 321: /* dnode_endpoint */ + case 340: /* column_name */ + case 348: /* table_name */ + case 355: /* function_name */ + case 367: /* topic_name */ + case 369: /* cgroup_name */ + case 374: /* stream_name */ + case 380: /* table_alias */ + case 381: /* column_alias */ + case 387: /* star_func */ + case 389: /* noarg_func */ + case 404: /* alias_opt */ { } break; - case 311: /* sysinfo_opt */ + case 315: /* sysinfo_opt */ { } break; - case 312: /* privileges */ - case 314: /* priv_type_list */ - case 315: /* priv_type */ + case 316: /* privileges */ + case 318: /* priv_type_list */ + case 319: /* priv_type */ { } break; - case 318: /* not_exists_opt */ - case 320: /* exists_opt */ - case 366: /* analyze_opt */ - case 368: /* agg_func_opt */ - case 405: /* set_quantifier_opt */ + case 322: /* not_exists_opt */ + case 324: /* exists_opt */ + case 370: /* analyze_opt */ + case 372: /* agg_func_opt */ + case 409: /* set_quantifier_opt */ { } break; - case 322: /* integer_list */ - case 323: /* variable_list */ - case 324: /* retention_list */ - case 328: /* column_def_list */ - case 329: /* tags_def_opt */ - case 331: /* multi_create_clause */ - case 332: /* tags_def */ - case 333: /* multi_drop_clause */ - case 340: /* specific_cols_opt */ - case 341: /* expression_list */ - case 343: /* col_name_list */ - case 346: /* duration_list */ - case 347: /* rollup_func_list */ - case 358: /* func_list */ - case 371: /* dnode_list */ - case 375: /* literal_list */ - case 384: /* star_func_para_list */ - case 386: /* other_para_list */ - case 406: /* select_list */ - case 407: /* partition_by_clause_opt */ - case 412: /* group_by_clause_opt */ - case 416: /* group_by_list */ - case 418: /* order_by_clause_opt */ - case 422: /* sort_specification_list */ + case 326: /* integer_list */ + case 327: /* variable_list */ + case 328: /* retention_list */ + case 332: /* column_def_list */ + case 333: /* tags_def_opt */ + case 335: /* multi_create_clause */ + case 336: /* tags_def */ + case 337: /* multi_drop_clause */ + case 344: /* specific_cols_opt */ + case 345: /* expression_list */ + case 347: /* col_name_list */ + case 350: /* duration_list */ + case 351: /* rollup_func_list */ + case 362: /* func_list */ + case 375: /* dnode_list */ + case 379: /* literal_list */ + case 388: /* star_func_para_list */ + case 390: /* other_para_list */ + case 410: /* select_list */ + case 411: /* partition_by_clause_opt */ + case 416: /* group_by_clause_opt */ + case 420: /* group_by_list */ + case 422: /* order_by_clause_opt */ + case 426: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy172)); + nodesDestroyList((yypminor->yy824)); } break; - case 325: /* alter_db_option */ - case 348: /* alter_table_option */ + case 329: /* alter_db_option */ + case 352: /* alter_table_option */ { } break; - case 337: /* type_name */ + case 341: /* type_name */ { } break; - case 389: /* compare_op */ - case 390: /* in_op */ + case 393: /* compare_op */ + case 394: /* in_op */ { } break; - case 402: /* join_type */ + case 406: /* join_type */ { } break; - case 415: /* fill_mode */ + case 419: /* fill_mode */ { } break; - case 424: /* ordering_specification_opt */ + case 428: /* ordering_specification_opt */ { } break; - case 425: /* null_ordering_opt */ + case 429: /* null_ordering_opt */ { } @@ -2790,495 +2845,502 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 305, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 305, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 306, 0 }, /* (2) account_options ::= */ - { 306, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 306, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 306, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 306, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 306, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 306, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 306, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 306, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 306, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 307, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 307, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 309, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 309, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 309, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 309, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 309, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 309, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 309, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 309, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 309, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 309, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 305, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ - { 305, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 305, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ - { 305, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ - { 305, -3 }, /* (28) cmd ::= DROP USER user_name */ - { 311, 0 }, /* (29) sysinfo_opt ::= */ - { 311, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - { 305, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 305, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ - { 312, -1 }, /* (33) privileges ::= ALL */ - { 312, -1 }, /* (34) privileges ::= priv_type_list */ - { 314, -1 }, /* (35) priv_type_list ::= priv_type */ - { 314, -3 }, /* (36) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 315, -1 }, /* (37) priv_type ::= READ */ - { 315, -1 }, /* (38) priv_type ::= WRITE */ - { 313, -3 }, /* (39) priv_level ::= NK_STAR NK_DOT NK_STAR */ - { 313, -3 }, /* (40) priv_level ::= db_name NK_DOT NK_STAR */ - { 305, -3 }, /* (41) cmd ::= CREATE DNODE dnode_endpoint */ - { 305, -5 }, /* (42) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 305, -3 }, /* (43) cmd ::= DROP DNODE NK_INTEGER */ - { 305, -3 }, /* (44) cmd ::= DROP DNODE dnode_endpoint */ - { 305, -4 }, /* (45) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 305, -5 }, /* (46) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 305, -4 }, /* (47) cmd ::= ALTER ALL DNODES NK_STRING */ - { 305, -5 }, /* (48) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 317, -1 }, /* (49) dnode_endpoint ::= NK_STRING */ - { 317, -1 }, /* (50) dnode_endpoint ::= NK_ID */ - { 317, -1 }, /* (51) dnode_endpoint ::= NK_IPTOKEN */ - { 305, -3 }, /* (52) cmd ::= ALTER LOCAL NK_STRING */ - { 305, -4 }, /* (53) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 305, -5 }, /* (54) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (55) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (56) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (57) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (58) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (59) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (60) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (61) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 305, -5 }, /* (62) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 305, -4 }, /* (63) cmd ::= DROP DATABASE exists_opt db_name */ - { 305, -2 }, /* (64) cmd ::= USE db_name */ - { 305, -4 }, /* (65) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 305, -3 }, /* (66) cmd ::= FLUSH DATABASE db_name */ - { 305, -3 }, /* (67) cmd ::= TRIM DATABASE db_name */ - { 318, -3 }, /* (68) not_exists_opt ::= IF NOT EXISTS */ - { 318, 0 }, /* (69) not_exists_opt ::= */ - { 320, -2 }, /* (70) exists_opt ::= IF EXISTS */ - { 320, 0 }, /* (71) exists_opt ::= */ - { 319, 0 }, /* (72) db_options ::= */ - { 319, -3 }, /* (73) db_options ::= db_options BUFFER NK_INTEGER */ - { 319, -3 }, /* (74) db_options ::= db_options CACHEMODEL NK_STRING */ - { 319, -3 }, /* (75) db_options ::= db_options CACHESIZE NK_INTEGER */ - { 319, -3 }, /* (76) db_options ::= db_options COMP NK_INTEGER */ - { 319, -3 }, /* (77) db_options ::= db_options DURATION NK_INTEGER */ - { 319, -3 }, /* (78) db_options ::= db_options DURATION NK_VARIABLE */ - { 319, -3 }, /* (79) db_options ::= db_options MAXROWS NK_INTEGER */ - { 319, -3 }, /* (80) db_options ::= db_options MINROWS NK_INTEGER */ - { 319, -3 }, /* (81) db_options ::= db_options KEEP integer_list */ - { 319, -3 }, /* (82) db_options ::= db_options KEEP variable_list */ - { 319, -3 }, /* (83) db_options ::= db_options PAGES NK_INTEGER */ - { 319, -3 }, /* (84) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 319, -3 }, /* (85) db_options ::= db_options PRECISION NK_STRING */ - { 319, -3 }, /* (86) db_options ::= db_options REPLICA NK_INTEGER */ - { 319, -3 }, /* (87) db_options ::= db_options STRICT NK_STRING */ - { 319, -3 }, /* (88) db_options ::= db_options VGROUPS NK_INTEGER */ - { 319, -3 }, /* (89) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 319, -3 }, /* (90) db_options ::= db_options RETENTIONS retention_list */ - { 319, -3 }, /* (91) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 319, -3 }, /* (92) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - { 319, -3 }, /* (93) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - { 319, -3 }, /* (94) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - { 319, -4 }, /* (95) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 319, -3 }, /* (96) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - { 319, -4 }, /* (97) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 319, -3 }, /* (98) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - { 319, -3 }, /* (99) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - { 321, -1 }, /* (100) alter_db_options ::= alter_db_option */ - { 321, -2 }, /* (101) alter_db_options ::= alter_db_options alter_db_option */ - { 325, -2 }, /* (102) alter_db_option ::= CACHEMODEL NK_STRING */ - { 325, -2 }, /* (103) alter_db_option ::= CACHESIZE NK_INTEGER */ - { 325, -2 }, /* (104) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - { 325, -2 }, /* (105) alter_db_option ::= KEEP integer_list */ - { 325, -2 }, /* (106) alter_db_option ::= KEEP variable_list */ - { 325, -2 }, /* (107) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - { 322, -1 }, /* (108) integer_list ::= NK_INTEGER */ - { 322, -3 }, /* (109) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 323, -1 }, /* (110) variable_list ::= NK_VARIABLE */ - { 323, -3 }, /* (111) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 324, -1 }, /* (112) retention_list ::= retention */ - { 324, -3 }, /* (113) retention_list ::= retention_list NK_COMMA retention */ - { 326, -3 }, /* (114) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 305, -9 }, /* (115) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 305, -3 }, /* (116) cmd ::= CREATE TABLE multi_create_clause */ - { 305, -9 }, /* (117) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 305, -3 }, /* (118) cmd ::= DROP TABLE multi_drop_clause */ - { 305, -4 }, /* (119) cmd ::= DROP STABLE exists_opt full_table_name */ - { 305, -3 }, /* (120) cmd ::= ALTER TABLE alter_table_clause */ - { 305, -3 }, /* (121) cmd ::= ALTER STABLE alter_table_clause */ - { 334, -2 }, /* (122) alter_table_clause ::= full_table_name alter_table_options */ - { 334, -5 }, /* (123) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 334, -4 }, /* (124) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 334, -5 }, /* (125) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 334, -5 }, /* (126) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 334, -5 }, /* (127) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 334, -4 }, /* (128) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 334, -5 }, /* (129) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 334, -5 }, /* (130) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 334, -6 }, /* (131) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 331, -1 }, /* (132) multi_create_clause ::= create_subtable_clause */ - { 331, -2 }, /* (133) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 339, -10 }, /* (134) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - { 333, -1 }, /* (135) multi_drop_clause ::= drop_table_clause */ - { 333, -2 }, /* (136) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 342, -2 }, /* (137) drop_table_clause ::= exists_opt full_table_name */ - { 340, 0 }, /* (138) specific_cols_opt ::= */ - { 340, -3 }, /* (139) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - { 327, -1 }, /* (140) full_table_name ::= table_name */ - { 327, -3 }, /* (141) full_table_name ::= db_name NK_DOT table_name */ - { 328, -1 }, /* (142) column_def_list ::= column_def */ - { 328, -3 }, /* (143) column_def_list ::= column_def_list NK_COMMA column_def */ - { 345, -2 }, /* (144) column_def ::= column_name type_name */ - { 345, -4 }, /* (145) column_def ::= column_name type_name COMMENT NK_STRING */ - { 337, -1 }, /* (146) type_name ::= BOOL */ - { 337, -1 }, /* (147) type_name ::= TINYINT */ - { 337, -1 }, /* (148) type_name ::= SMALLINT */ - { 337, -1 }, /* (149) type_name ::= INT */ - { 337, -1 }, /* (150) type_name ::= INTEGER */ - { 337, -1 }, /* (151) type_name ::= BIGINT */ - { 337, -1 }, /* (152) type_name ::= FLOAT */ - { 337, -1 }, /* (153) type_name ::= DOUBLE */ - { 337, -4 }, /* (154) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 337, -1 }, /* (155) type_name ::= TIMESTAMP */ - { 337, -4 }, /* (156) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 337, -2 }, /* (157) type_name ::= TINYINT UNSIGNED */ - { 337, -2 }, /* (158) type_name ::= SMALLINT UNSIGNED */ - { 337, -2 }, /* (159) type_name ::= INT UNSIGNED */ - { 337, -2 }, /* (160) type_name ::= BIGINT UNSIGNED */ - { 337, -1 }, /* (161) type_name ::= JSON */ - { 337, -4 }, /* (162) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 337, -1 }, /* (163) type_name ::= MEDIUMBLOB */ - { 337, -1 }, /* (164) type_name ::= BLOB */ - { 337, -4 }, /* (165) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 337, -1 }, /* (166) type_name ::= DECIMAL */ - { 337, -4 }, /* (167) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 337, -6 }, /* (168) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 329, 0 }, /* (169) tags_def_opt ::= */ - { 329, -1 }, /* (170) tags_def_opt ::= tags_def */ - { 332, -4 }, /* (171) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 330, 0 }, /* (172) table_options ::= */ - { 330, -3 }, /* (173) table_options ::= table_options COMMENT NK_STRING */ - { 330, -3 }, /* (174) table_options ::= table_options MAX_DELAY duration_list */ - { 330, -3 }, /* (175) table_options ::= table_options WATERMARK duration_list */ - { 330, -5 }, /* (176) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 330, -3 }, /* (177) table_options ::= table_options TTL NK_INTEGER */ - { 330, -5 }, /* (178) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 335, -1 }, /* (179) alter_table_options ::= alter_table_option */ - { 335, -2 }, /* (180) alter_table_options ::= alter_table_options alter_table_option */ - { 348, -2 }, /* (181) alter_table_option ::= COMMENT NK_STRING */ - { 348, -2 }, /* (182) alter_table_option ::= TTL NK_INTEGER */ - { 346, -1 }, /* (183) duration_list ::= duration_literal */ - { 346, -3 }, /* (184) duration_list ::= duration_list NK_COMMA duration_literal */ - { 347, -1 }, /* (185) rollup_func_list ::= rollup_func_name */ - { 347, -3 }, /* (186) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 350, -1 }, /* (187) rollup_func_name ::= function_name */ - { 350, -1 }, /* (188) rollup_func_name ::= FIRST */ - { 350, -1 }, /* (189) rollup_func_name ::= LAST */ - { 343, -1 }, /* (190) col_name_list ::= col_name */ - { 343, -3 }, /* (191) col_name_list ::= col_name_list NK_COMMA col_name */ - { 352, -1 }, /* (192) col_name ::= column_name */ - { 305, -2 }, /* (193) cmd ::= SHOW DNODES */ - { 305, -2 }, /* (194) cmd ::= SHOW USERS */ - { 305, -2 }, /* (195) cmd ::= SHOW DATABASES */ - { 305, -4 }, /* (196) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 305, -4 }, /* (197) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 305, -3 }, /* (198) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 305, -2 }, /* (199) cmd ::= SHOW MNODES */ - { 305, -2 }, /* (200) cmd ::= SHOW MODULES */ - { 305, -2 }, /* (201) cmd ::= SHOW QNODES */ - { 305, -2 }, /* (202) cmd ::= SHOW FUNCTIONS */ - { 305, -5 }, /* (203) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 305, -2 }, /* (204) cmd ::= SHOW STREAMS */ - { 305, -2 }, /* (205) cmd ::= SHOW ACCOUNTS */ - { 305, -2 }, /* (206) cmd ::= SHOW APPS */ - { 305, -2 }, /* (207) cmd ::= SHOW CONNECTIONS */ - { 305, -2 }, /* (208) cmd ::= SHOW LICENCES */ - { 305, -2 }, /* (209) cmd ::= SHOW GRANTS */ - { 305, -4 }, /* (210) cmd ::= SHOW CREATE DATABASE db_name */ - { 305, -4 }, /* (211) cmd ::= SHOW CREATE TABLE full_table_name */ - { 305, -4 }, /* (212) cmd ::= SHOW CREATE STABLE full_table_name */ - { 305, -2 }, /* (213) cmd ::= SHOW QUERIES */ - { 305, -2 }, /* (214) cmd ::= SHOW SCORES */ - { 305, -2 }, /* (215) cmd ::= SHOW TOPICS */ - { 305, -2 }, /* (216) cmd ::= SHOW VARIABLES */ - { 305, -3 }, /* (217) cmd ::= SHOW LOCAL VARIABLES */ - { 305, -4 }, /* (218) cmd ::= SHOW DNODE NK_INTEGER VARIABLES */ - { 305, -2 }, /* (219) cmd ::= SHOW BNODES */ - { 305, -2 }, /* (220) cmd ::= SHOW SNODES */ - { 305, -2 }, /* (221) cmd ::= SHOW CLUSTER */ - { 305, -2 }, /* (222) cmd ::= SHOW TRANSACTIONS */ - { 305, -4 }, /* (223) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 305, -2 }, /* (224) cmd ::= SHOW CONSUMERS */ - { 305, -2 }, /* (225) cmd ::= SHOW SUBSCRIPTIONS */ - { 305, -5 }, /* (226) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 353, 0 }, /* (227) db_name_cond_opt ::= */ - { 353, -2 }, /* (228) db_name_cond_opt ::= db_name NK_DOT */ - { 354, 0 }, /* (229) like_pattern_opt ::= */ - { 354, -2 }, /* (230) like_pattern_opt ::= LIKE NK_STRING */ - { 355, -1 }, /* (231) table_name_cond ::= table_name */ - { 356, 0 }, /* (232) from_db_opt ::= */ - { 356, -2 }, /* (233) from_db_opt ::= FROM db_name */ - { 305, -8 }, /* (234) cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ - { 305, -4 }, /* (235) cmd ::= DROP INDEX exists_opt full_table_name */ - { 357, -10 }, /* (236) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 357, -12 }, /* (237) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 358, -1 }, /* (238) func_list ::= func */ - { 358, -3 }, /* (239) func_list ::= func_list NK_COMMA func */ - { 361, -4 }, /* (240) func ::= function_name NK_LP expression_list NK_RP */ - { 360, 0 }, /* (241) sma_stream_opt ::= */ - { 360, -3 }, /* (242) sma_stream_opt ::= stream_options WATERMARK duration_literal */ - { 360, -3 }, /* (243) sma_stream_opt ::= stream_options MAX_DELAY duration_literal */ - { 305, -6 }, /* (244) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ - { 305, -7 }, /* (245) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 305, -9 }, /* (246) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 305, -7 }, /* (247) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 305, -9 }, /* (248) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 305, -4 }, /* (249) cmd ::= DROP TOPIC exists_opt topic_name */ - { 305, -7 }, /* (250) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 305, -2 }, /* (251) cmd ::= DESC full_table_name */ - { 305, -2 }, /* (252) cmd ::= DESCRIBE full_table_name */ - { 305, -3 }, /* (253) cmd ::= RESET QUERY CACHE */ - { 305, -4 }, /* (254) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 366, 0 }, /* (255) analyze_opt ::= */ - { 366, -1 }, /* (256) analyze_opt ::= ANALYZE */ - { 367, 0 }, /* (257) explain_options ::= */ - { 367, -3 }, /* (258) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 367, -3 }, /* (259) explain_options ::= explain_options RATIO NK_FLOAT */ - { 305, -10 }, /* (260) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 305, -4 }, /* (261) cmd ::= DROP FUNCTION exists_opt function_name */ - { 368, 0 }, /* (262) agg_func_opt ::= */ - { 368, -1 }, /* (263) agg_func_opt ::= AGGREGATE */ - { 369, 0 }, /* (264) bufsize_opt ::= */ - { 369, -2 }, /* (265) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 305, -9 }, /* (266) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name AS query_expression */ - { 305, -4 }, /* (267) cmd ::= DROP STREAM exists_opt stream_name */ - { 362, 0 }, /* (268) stream_options ::= */ - { 362, -3 }, /* (269) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 362, -3 }, /* (270) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 362, -4 }, /* (271) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 362, -3 }, /* (272) stream_options ::= stream_options WATERMARK duration_literal */ - { 362, -4 }, /* (273) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 305, -3 }, /* (274) cmd ::= KILL CONNECTION NK_INTEGER */ - { 305, -3 }, /* (275) cmd ::= KILL QUERY NK_STRING */ - { 305, -3 }, /* (276) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 305, -2 }, /* (277) cmd ::= BALANCE VGROUP */ - { 305, -4 }, /* (278) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 305, -4 }, /* (279) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 305, -3 }, /* (280) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 371, -2 }, /* (281) dnode_list ::= DNODE NK_INTEGER */ - { 371, -3 }, /* (282) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 305, -4 }, /* (283) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 305, -1 }, /* (284) cmd ::= query_expression */ - { 305, -7 }, /* (285) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_expression */ - { 305, -4 }, /* (286) cmd ::= INSERT INTO full_table_name query_expression */ - { 308, -1 }, /* (287) literal ::= NK_INTEGER */ - { 308, -1 }, /* (288) literal ::= NK_FLOAT */ - { 308, -1 }, /* (289) literal ::= NK_STRING */ - { 308, -1 }, /* (290) literal ::= NK_BOOL */ - { 308, -2 }, /* (291) literal ::= TIMESTAMP NK_STRING */ - { 308, -1 }, /* (292) literal ::= duration_literal */ - { 308, -1 }, /* (293) literal ::= NULL */ - { 308, -1 }, /* (294) literal ::= NK_QUESTION */ - { 349, -1 }, /* (295) duration_literal ::= NK_VARIABLE */ - { 373, -1 }, /* (296) signed ::= NK_INTEGER */ - { 373, -2 }, /* (297) signed ::= NK_PLUS NK_INTEGER */ - { 373, -2 }, /* (298) signed ::= NK_MINUS NK_INTEGER */ - { 373, -1 }, /* (299) signed ::= NK_FLOAT */ - { 373, -2 }, /* (300) signed ::= NK_PLUS NK_FLOAT */ - { 373, -2 }, /* (301) signed ::= NK_MINUS NK_FLOAT */ - { 338, -1 }, /* (302) signed_literal ::= signed */ - { 338, -1 }, /* (303) signed_literal ::= NK_STRING */ - { 338, -1 }, /* (304) signed_literal ::= NK_BOOL */ - { 338, -2 }, /* (305) signed_literal ::= TIMESTAMP NK_STRING */ - { 338, -1 }, /* (306) signed_literal ::= duration_literal */ - { 338, -1 }, /* (307) signed_literal ::= NULL */ - { 338, -1 }, /* (308) signed_literal ::= literal_func */ - { 338, -1 }, /* (309) signed_literal ::= NK_QUESTION */ - { 375, -1 }, /* (310) literal_list ::= signed_literal */ - { 375, -3 }, /* (311) literal_list ::= literal_list NK_COMMA signed_literal */ - { 316, -1 }, /* (312) db_name ::= NK_ID */ - { 344, -1 }, /* (313) table_name ::= NK_ID */ - { 336, -1 }, /* (314) column_name ::= NK_ID */ - { 351, -1 }, /* (315) function_name ::= NK_ID */ - { 376, -1 }, /* (316) table_alias ::= NK_ID */ - { 377, -1 }, /* (317) column_alias ::= NK_ID */ - { 310, -1 }, /* (318) user_name ::= NK_ID */ - { 363, -1 }, /* (319) topic_name ::= NK_ID */ - { 370, -1 }, /* (320) stream_name ::= NK_ID */ - { 365, -1 }, /* (321) cgroup_name ::= NK_ID */ - { 378, -1 }, /* (322) expression ::= literal */ - { 378, -1 }, /* (323) expression ::= pseudo_column */ - { 378, -1 }, /* (324) expression ::= column_reference */ - { 378, -1 }, /* (325) expression ::= function_expression */ - { 378, -1 }, /* (326) expression ::= subquery */ - { 378, -3 }, /* (327) expression ::= NK_LP expression NK_RP */ - { 378, -2 }, /* (328) expression ::= NK_PLUS expression */ - { 378, -2 }, /* (329) expression ::= NK_MINUS expression */ - { 378, -3 }, /* (330) expression ::= expression NK_PLUS expression */ - { 378, -3 }, /* (331) expression ::= expression NK_MINUS expression */ - { 378, -3 }, /* (332) expression ::= expression NK_STAR expression */ - { 378, -3 }, /* (333) expression ::= expression NK_SLASH expression */ - { 378, -3 }, /* (334) expression ::= expression NK_REM expression */ - { 378, -3 }, /* (335) expression ::= column_reference NK_ARROW NK_STRING */ - { 378, -3 }, /* (336) expression ::= expression NK_BITAND expression */ - { 378, -3 }, /* (337) expression ::= expression NK_BITOR expression */ - { 341, -1 }, /* (338) expression_list ::= expression */ - { 341, -3 }, /* (339) expression_list ::= expression_list NK_COMMA expression */ - { 380, -1 }, /* (340) column_reference ::= column_name */ - { 380, -3 }, /* (341) column_reference ::= table_name NK_DOT column_name */ - { 379, -1 }, /* (342) pseudo_column ::= ROWTS */ - { 379, -1 }, /* (343) pseudo_column ::= TBNAME */ - { 379, -3 }, /* (344) pseudo_column ::= table_name NK_DOT TBNAME */ - { 379, -1 }, /* (345) pseudo_column ::= QSTART */ - { 379, -1 }, /* (346) pseudo_column ::= QEND */ - { 379, -1 }, /* (347) pseudo_column ::= QDURATION */ - { 379, -1 }, /* (348) pseudo_column ::= WSTART */ - { 379, -1 }, /* (349) pseudo_column ::= WEND */ - { 379, -1 }, /* (350) pseudo_column ::= WDURATION */ - { 381, -4 }, /* (351) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 381, -4 }, /* (352) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 381, -6 }, /* (353) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ - { 381, -1 }, /* (354) function_expression ::= literal_func */ - { 374, -3 }, /* (355) literal_func ::= noarg_func NK_LP NK_RP */ - { 374, -1 }, /* (356) literal_func ::= NOW */ - { 385, -1 }, /* (357) noarg_func ::= NOW */ - { 385, -1 }, /* (358) noarg_func ::= TODAY */ - { 385, -1 }, /* (359) noarg_func ::= TIMEZONE */ - { 385, -1 }, /* (360) noarg_func ::= DATABASE */ - { 385, -1 }, /* (361) noarg_func ::= CLIENT_VERSION */ - { 385, -1 }, /* (362) noarg_func ::= SERVER_VERSION */ - { 385, -1 }, /* (363) noarg_func ::= SERVER_STATUS */ - { 385, -1 }, /* (364) noarg_func ::= CURRENT_USER */ - { 385, -1 }, /* (365) noarg_func ::= USER */ - { 383, -1 }, /* (366) star_func ::= COUNT */ - { 383, -1 }, /* (367) star_func ::= FIRST */ - { 383, -1 }, /* (368) star_func ::= LAST */ - { 383, -1 }, /* (369) star_func ::= LAST_ROW */ - { 384, -1 }, /* (370) star_func_para_list ::= NK_STAR */ - { 384, -1 }, /* (371) star_func_para_list ::= other_para_list */ - { 386, -1 }, /* (372) other_para_list ::= star_func_para */ - { 386, -3 }, /* (373) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 387, -1 }, /* (374) star_func_para ::= expression */ - { 387, -3 }, /* (375) star_func_para ::= table_name NK_DOT NK_STAR */ - { 388, -3 }, /* (376) predicate ::= expression compare_op expression */ - { 388, -5 }, /* (377) predicate ::= expression BETWEEN expression AND expression */ - { 388, -6 }, /* (378) predicate ::= expression NOT BETWEEN expression AND expression */ - { 388, -3 }, /* (379) predicate ::= expression IS NULL */ - { 388, -4 }, /* (380) predicate ::= expression IS NOT NULL */ - { 388, -3 }, /* (381) predicate ::= expression in_op in_predicate_value */ - { 389, -1 }, /* (382) compare_op ::= NK_LT */ - { 389, -1 }, /* (383) compare_op ::= NK_GT */ - { 389, -1 }, /* (384) compare_op ::= NK_LE */ - { 389, -1 }, /* (385) compare_op ::= NK_GE */ - { 389, -1 }, /* (386) compare_op ::= NK_NE */ - { 389, -1 }, /* (387) compare_op ::= NK_EQ */ - { 389, -1 }, /* (388) compare_op ::= LIKE */ - { 389, -2 }, /* (389) compare_op ::= NOT LIKE */ - { 389, -1 }, /* (390) compare_op ::= MATCH */ - { 389, -1 }, /* (391) compare_op ::= NMATCH */ - { 389, -1 }, /* (392) compare_op ::= CONTAINS */ - { 390, -1 }, /* (393) in_op ::= IN */ - { 390, -2 }, /* (394) in_op ::= NOT IN */ - { 391, -3 }, /* (395) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 392, -1 }, /* (396) boolean_value_expression ::= boolean_primary */ - { 392, -2 }, /* (397) boolean_value_expression ::= NOT boolean_primary */ - { 392, -3 }, /* (398) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 392, -3 }, /* (399) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 393, -1 }, /* (400) boolean_primary ::= predicate */ - { 393, -3 }, /* (401) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 394, -1 }, /* (402) common_expression ::= expression */ - { 394, -1 }, /* (403) common_expression ::= boolean_value_expression */ - { 395, 0 }, /* (404) from_clause_opt ::= */ - { 395, -2 }, /* (405) from_clause_opt ::= FROM table_reference_list */ - { 396, -1 }, /* (406) table_reference_list ::= table_reference */ - { 396, -3 }, /* (407) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 397, -1 }, /* (408) table_reference ::= table_primary */ - { 397, -1 }, /* (409) table_reference ::= joined_table */ - { 398, -2 }, /* (410) table_primary ::= table_name alias_opt */ - { 398, -4 }, /* (411) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 398, -2 }, /* (412) table_primary ::= subquery alias_opt */ - { 398, -1 }, /* (413) table_primary ::= parenthesized_joined_table */ - { 400, 0 }, /* (414) alias_opt ::= */ - { 400, -1 }, /* (415) alias_opt ::= table_alias */ - { 400, -2 }, /* (416) alias_opt ::= AS table_alias */ - { 401, -3 }, /* (417) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 401, -3 }, /* (418) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 399, -6 }, /* (419) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 402, 0 }, /* (420) join_type ::= */ - { 402, -1 }, /* (421) join_type ::= INNER */ - { 404, -12 }, /* (422) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 405, 0 }, /* (423) set_quantifier_opt ::= */ - { 405, -1 }, /* (424) set_quantifier_opt ::= DISTINCT */ - { 405, -1 }, /* (425) set_quantifier_opt ::= ALL */ - { 406, -1 }, /* (426) select_list ::= select_item */ - { 406, -3 }, /* (427) select_list ::= select_list NK_COMMA select_item */ - { 414, -1 }, /* (428) select_item ::= NK_STAR */ - { 414, -1 }, /* (429) select_item ::= common_expression */ - { 414, -2 }, /* (430) select_item ::= common_expression column_alias */ - { 414, -3 }, /* (431) select_item ::= common_expression AS column_alias */ - { 414, -3 }, /* (432) select_item ::= table_name NK_DOT NK_STAR */ - { 372, 0 }, /* (433) where_clause_opt ::= */ - { 372, -2 }, /* (434) where_clause_opt ::= WHERE search_condition */ - { 407, 0 }, /* (435) partition_by_clause_opt ::= */ - { 407, -3 }, /* (436) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 411, 0 }, /* (437) twindow_clause_opt ::= */ - { 411, -6 }, /* (438) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 411, -4 }, /* (439) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ - { 411, -6 }, /* (440) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 411, -8 }, /* (441) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 359, 0 }, /* (442) sliding_opt ::= */ - { 359, -4 }, /* (443) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 410, 0 }, /* (444) fill_opt ::= */ - { 410, -4 }, /* (445) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 410, -6 }, /* (446) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 415, -1 }, /* (447) fill_mode ::= NONE */ - { 415, -1 }, /* (448) fill_mode ::= PREV */ - { 415, -1 }, /* (449) fill_mode ::= NULL */ - { 415, -1 }, /* (450) fill_mode ::= LINEAR */ - { 415, -1 }, /* (451) fill_mode ::= NEXT */ - { 412, 0 }, /* (452) group_by_clause_opt ::= */ - { 412, -3 }, /* (453) group_by_clause_opt ::= GROUP BY group_by_list */ - { 416, -1 }, /* (454) group_by_list ::= expression */ - { 416, -3 }, /* (455) group_by_list ::= group_by_list NK_COMMA expression */ - { 413, 0 }, /* (456) having_clause_opt ::= */ - { 413, -2 }, /* (457) having_clause_opt ::= HAVING search_condition */ - { 408, 0 }, /* (458) range_opt ::= */ - { 408, -6 }, /* (459) range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ - { 409, 0 }, /* (460) every_opt ::= */ - { 409, -4 }, /* (461) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 364, -4 }, /* (462) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 417, -1 }, /* (463) query_expression_body ::= query_primary */ - { 417, -4 }, /* (464) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 417, -3 }, /* (465) query_expression_body ::= query_expression_body UNION query_expression_body */ - { 421, -1 }, /* (466) query_primary ::= query_specification */ - { 421, -6 }, /* (467) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ - { 418, 0 }, /* (468) order_by_clause_opt ::= */ - { 418, -3 }, /* (469) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 419, 0 }, /* (470) slimit_clause_opt ::= */ - { 419, -2 }, /* (471) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 419, -4 }, /* (472) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 419, -4 }, /* (473) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 420, 0 }, /* (474) limit_clause_opt ::= */ - { 420, -2 }, /* (475) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 420, -4 }, /* (476) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 420, -4 }, /* (477) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 382, -3 }, /* (478) subquery ::= NK_LP query_expression NK_RP */ - { 403, -1 }, /* (479) search_condition ::= common_expression */ - { 422, -1 }, /* (480) sort_specification_list ::= sort_specification */ - { 422, -3 }, /* (481) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 423, -3 }, /* (482) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 424, 0 }, /* (483) ordering_specification_opt ::= */ - { 424, -1 }, /* (484) ordering_specification_opt ::= ASC */ - { 424, -1 }, /* (485) ordering_specification_opt ::= DESC */ - { 425, 0 }, /* (486) null_ordering_opt ::= */ - { 425, -2 }, /* (487) null_ordering_opt ::= NULLS FIRST */ - { 425, -2 }, /* (488) null_ordering_opt ::= NULLS LAST */ + { 309, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 309, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 310, 0 }, /* (2) account_options ::= */ + { 310, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 310, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 310, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 310, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 310, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 310, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 310, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 310, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 310, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 311, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 311, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 313, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 313, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 313, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 313, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 313, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 313, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 313, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 313, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 313, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 313, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 309, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + { 309, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 309, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + { 309, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + { 309, -3 }, /* (28) cmd ::= DROP USER user_name */ + { 315, 0 }, /* (29) sysinfo_opt ::= */ + { 315, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + { 309, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ + { 309, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ + { 316, -1 }, /* (33) privileges ::= ALL */ + { 316, -1 }, /* (34) privileges ::= priv_type_list */ + { 318, -1 }, /* (35) priv_type_list ::= priv_type */ + { 318, -3 }, /* (36) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 319, -1 }, /* (37) priv_type ::= READ */ + { 319, -1 }, /* (38) priv_type ::= WRITE */ + { 317, -3 }, /* (39) priv_level ::= NK_STAR NK_DOT NK_STAR */ + { 317, -3 }, /* (40) priv_level ::= db_name NK_DOT NK_STAR */ + { 309, -3 }, /* (41) cmd ::= CREATE DNODE dnode_endpoint */ + { 309, -5 }, /* (42) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 309, -3 }, /* (43) cmd ::= DROP DNODE NK_INTEGER */ + { 309, -3 }, /* (44) cmd ::= DROP DNODE dnode_endpoint */ + { 309, -4 }, /* (45) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 309, -5 }, /* (46) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 309, -4 }, /* (47) cmd ::= ALTER ALL DNODES NK_STRING */ + { 309, -5 }, /* (48) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 321, -1 }, /* (49) dnode_endpoint ::= NK_STRING */ + { 321, -1 }, /* (50) dnode_endpoint ::= NK_ID */ + { 321, -1 }, /* (51) dnode_endpoint ::= NK_IPTOKEN */ + { 309, -3 }, /* (52) cmd ::= ALTER LOCAL NK_STRING */ + { 309, -4 }, /* (53) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 309, -5 }, /* (54) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (55) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (56) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (57) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (58) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (59) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (60) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (61) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 309, -5 }, /* (62) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 309, -4 }, /* (63) cmd ::= DROP DATABASE exists_opt db_name */ + { 309, -2 }, /* (64) cmd ::= USE db_name */ + { 309, -4 }, /* (65) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 309, -3 }, /* (66) cmd ::= FLUSH DATABASE db_name */ + { 309, -3 }, /* (67) cmd ::= TRIM DATABASE db_name */ + { 322, -3 }, /* (68) not_exists_opt ::= IF NOT EXISTS */ + { 322, 0 }, /* (69) not_exists_opt ::= */ + { 324, -2 }, /* (70) exists_opt ::= IF EXISTS */ + { 324, 0 }, /* (71) exists_opt ::= */ + { 323, 0 }, /* (72) db_options ::= */ + { 323, -3 }, /* (73) db_options ::= db_options BUFFER NK_INTEGER */ + { 323, -3 }, /* (74) db_options ::= db_options CACHEMODEL NK_STRING */ + { 323, -3 }, /* (75) db_options ::= db_options CACHESIZE NK_INTEGER */ + { 323, -3 }, /* (76) db_options ::= db_options COMP NK_INTEGER */ + { 323, -3 }, /* (77) db_options ::= db_options DURATION NK_INTEGER */ + { 323, -3 }, /* (78) db_options ::= db_options DURATION NK_VARIABLE */ + { 323, -3 }, /* (79) db_options ::= db_options MAXROWS NK_INTEGER */ + { 323, -3 }, /* (80) db_options ::= db_options MINROWS NK_INTEGER */ + { 323, -3 }, /* (81) db_options ::= db_options KEEP integer_list */ + { 323, -3 }, /* (82) db_options ::= db_options KEEP variable_list */ + { 323, -3 }, /* (83) db_options ::= db_options PAGES NK_INTEGER */ + { 323, -3 }, /* (84) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 323, -3 }, /* (85) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + { 323, -3 }, /* (86) db_options ::= db_options PRECISION NK_STRING */ + { 323, -3 }, /* (87) db_options ::= db_options REPLICA NK_INTEGER */ + { 323, -3 }, /* (88) db_options ::= db_options STRICT NK_STRING */ + { 323, -3 }, /* (89) db_options ::= db_options VGROUPS NK_INTEGER */ + { 323, -3 }, /* (90) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 323, -3 }, /* (91) db_options ::= db_options RETENTIONS retention_list */ + { 323, -3 }, /* (92) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 323, -3 }, /* (93) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + { 323, -3 }, /* (94) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + { 323, -3 }, /* (95) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + { 323, -4 }, /* (96) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 323, -3 }, /* (97) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + { 323, -4 }, /* (98) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 323, -3 }, /* (99) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + { 323, -3 }, /* (100) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + { 323, -3 }, /* (101) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + { 323, -3 }, /* (102) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ + { 323, -3 }, /* (103) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ + { 325, -1 }, /* (104) alter_db_options ::= alter_db_option */ + { 325, -2 }, /* (105) alter_db_options ::= alter_db_options alter_db_option */ + { 329, -2 }, /* (106) alter_db_option ::= CACHEMODEL NK_STRING */ + { 329, -2 }, /* (107) alter_db_option ::= CACHESIZE NK_INTEGER */ + { 329, -2 }, /* (108) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + { 329, -2 }, /* (109) alter_db_option ::= KEEP integer_list */ + { 329, -2 }, /* (110) alter_db_option ::= KEEP variable_list */ + { 329, -2 }, /* (111) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + { 329, -2 }, /* (112) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + { 326, -1 }, /* (113) integer_list ::= NK_INTEGER */ + { 326, -3 }, /* (114) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 327, -1 }, /* (115) variable_list ::= NK_VARIABLE */ + { 327, -3 }, /* (116) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 328, -1 }, /* (117) retention_list ::= retention */ + { 328, -3 }, /* (118) retention_list ::= retention_list NK_COMMA retention */ + { 330, -3 }, /* (119) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 309, -9 }, /* (120) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 309, -3 }, /* (121) cmd ::= CREATE TABLE multi_create_clause */ + { 309, -9 }, /* (122) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 309, -3 }, /* (123) cmd ::= DROP TABLE multi_drop_clause */ + { 309, -4 }, /* (124) cmd ::= DROP STABLE exists_opt full_table_name */ + { 309, -3 }, /* (125) cmd ::= ALTER TABLE alter_table_clause */ + { 309, -3 }, /* (126) cmd ::= ALTER STABLE alter_table_clause */ + { 338, -2 }, /* (127) alter_table_clause ::= full_table_name alter_table_options */ + { 338, -5 }, /* (128) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 338, -4 }, /* (129) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 338, -5 }, /* (130) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 338, -5 }, /* (131) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 338, -5 }, /* (132) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 338, -4 }, /* (133) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 338, -5 }, /* (134) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 338, -5 }, /* (135) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 338, -6 }, /* (136) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 335, -1 }, /* (137) multi_create_clause ::= create_subtable_clause */ + { 335, -2 }, /* (138) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 343, -10 }, /* (139) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + { 337, -1 }, /* (140) multi_drop_clause ::= drop_table_clause */ + { 337, -2 }, /* (141) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 346, -2 }, /* (142) drop_table_clause ::= exists_opt full_table_name */ + { 344, 0 }, /* (143) specific_cols_opt ::= */ + { 344, -3 }, /* (144) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + { 331, -1 }, /* (145) full_table_name ::= table_name */ + { 331, -3 }, /* (146) full_table_name ::= db_name NK_DOT table_name */ + { 332, -1 }, /* (147) column_def_list ::= column_def */ + { 332, -3 }, /* (148) column_def_list ::= column_def_list NK_COMMA column_def */ + { 349, -2 }, /* (149) column_def ::= column_name type_name */ + { 349, -4 }, /* (150) column_def ::= column_name type_name COMMENT NK_STRING */ + { 341, -1 }, /* (151) type_name ::= BOOL */ + { 341, -1 }, /* (152) type_name ::= TINYINT */ + { 341, -1 }, /* (153) type_name ::= SMALLINT */ + { 341, -1 }, /* (154) type_name ::= INT */ + { 341, -1 }, /* (155) type_name ::= INTEGER */ + { 341, -1 }, /* (156) type_name ::= BIGINT */ + { 341, -1 }, /* (157) type_name ::= FLOAT */ + { 341, -1 }, /* (158) type_name ::= DOUBLE */ + { 341, -4 }, /* (159) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 341, -1 }, /* (160) type_name ::= TIMESTAMP */ + { 341, -4 }, /* (161) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 341, -2 }, /* (162) type_name ::= TINYINT UNSIGNED */ + { 341, -2 }, /* (163) type_name ::= SMALLINT UNSIGNED */ + { 341, -2 }, /* (164) type_name ::= INT UNSIGNED */ + { 341, -2 }, /* (165) type_name ::= BIGINT UNSIGNED */ + { 341, -1 }, /* (166) type_name ::= JSON */ + { 341, -4 }, /* (167) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 341, -1 }, /* (168) type_name ::= MEDIUMBLOB */ + { 341, -1 }, /* (169) type_name ::= BLOB */ + { 341, -4 }, /* (170) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 341, -1 }, /* (171) type_name ::= DECIMAL */ + { 341, -4 }, /* (172) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 341, -6 }, /* (173) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 333, 0 }, /* (174) tags_def_opt ::= */ + { 333, -1 }, /* (175) tags_def_opt ::= tags_def */ + { 336, -4 }, /* (176) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 334, 0 }, /* (177) table_options ::= */ + { 334, -3 }, /* (178) table_options ::= table_options COMMENT NK_STRING */ + { 334, -3 }, /* (179) table_options ::= table_options MAX_DELAY duration_list */ + { 334, -3 }, /* (180) table_options ::= table_options WATERMARK duration_list */ + { 334, -5 }, /* (181) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 334, -3 }, /* (182) table_options ::= table_options TTL NK_INTEGER */ + { 334, -5 }, /* (183) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 339, -1 }, /* (184) alter_table_options ::= alter_table_option */ + { 339, -2 }, /* (185) alter_table_options ::= alter_table_options alter_table_option */ + { 352, -2 }, /* (186) alter_table_option ::= COMMENT NK_STRING */ + { 352, -2 }, /* (187) alter_table_option ::= TTL NK_INTEGER */ + { 350, -1 }, /* (188) duration_list ::= duration_literal */ + { 350, -3 }, /* (189) duration_list ::= duration_list NK_COMMA duration_literal */ + { 351, -1 }, /* (190) rollup_func_list ::= rollup_func_name */ + { 351, -3 }, /* (191) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 354, -1 }, /* (192) rollup_func_name ::= function_name */ + { 354, -1 }, /* (193) rollup_func_name ::= FIRST */ + { 354, -1 }, /* (194) rollup_func_name ::= LAST */ + { 347, -1 }, /* (195) col_name_list ::= col_name */ + { 347, -3 }, /* (196) col_name_list ::= col_name_list NK_COMMA col_name */ + { 356, -1 }, /* (197) col_name ::= column_name */ + { 309, -2 }, /* (198) cmd ::= SHOW DNODES */ + { 309, -2 }, /* (199) cmd ::= SHOW USERS */ + { 309, -2 }, /* (200) cmd ::= SHOW DATABASES */ + { 309, -4 }, /* (201) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 309, -4 }, /* (202) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 309, -3 }, /* (203) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 309, -2 }, /* (204) cmd ::= SHOW MNODES */ + { 309, -2 }, /* (205) cmd ::= SHOW MODULES */ + { 309, -2 }, /* (206) cmd ::= SHOW QNODES */ + { 309, -2 }, /* (207) cmd ::= SHOW FUNCTIONS */ + { 309, -5 }, /* (208) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 309, -2 }, /* (209) cmd ::= SHOW STREAMS */ + { 309, -2 }, /* (210) cmd ::= SHOW ACCOUNTS */ + { 309, -2 }, /* (211) cmd ::= SHOW APPS */ + { 309, -2 }, /* (212) cmd ::= SHOW CONNECTIONS */ + { 309, -2 }, /* (213) cmd ::= SHOW LICENCES */ + { 309, -2 }, /* (214) cmd ::= SHOW GRANTS */ + { 309, -4 }, /* (215) cmd ::= SHOW CREATE DATABASE db_name */ + { 309, -4 }, /* (216) cmd ::= SHOW CREATE TABLE full_table_name */ + { 309, -4 }, /* (217) cmd ::= SHOW CREATE STABLE full_table_name */ + { 309, -2 }, /* (218) cmd ::= SHOW QUERIES */ + { 309, -2 }, /* (219) cmd ::= SHOW SCORES */ + { 309, -2 }, /* (220) cmd ::= SHOW TOPICS */ + { 309, -2 }, /* (221) cmd ::= SHOW VARIABLES */ + { 309, -3 }, /* (222) cmd ::= SHOW LOCAL VARIABLES */ + { 309, -4 }, /* (223) cmd ::= SHOW DNODE NK_INTEGER VARIABLES */ + { 309, -2 }, /* (224) cmd ::= SHOW BNODES */ + { 309, -2 }, /* (225) cmd ::= SHOW SNODES */ + { 309, -2 }, /* (226) cmd ::= SHOW CLUSTER */ + { 309, -2 }, /* (227) cmd ::= SHOW TRANSACTIONS */ + { 309, -4 }, /* (228) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 309, -2 }, /* (229) cmd ::= SHOW CONSUMERS */ + { 309, -2 }, /* (230) cmd ::= SHOW SUBSCRIPTIONS */ + { 309, -5 }, /* (231) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 309, -3 }, /* (232) cmd ::= SHOW VNODES NK_INTEGER */ + { 309, -3 }, /* (233) cmd ::= SHOW VNODES NK_STRING */ + { 357, 0 }, /* (234) db_name_cond_opt ::= */ + { 357, -2 }, /* (235) db_name_cond_opt ::= db_name NK_DOT */ + { 358, 0 }, /* (236) like_pattern_opt ::= */ + { 358, -2 }, /* (237) like_pattern_opt ::= LIKE NK_STRING */ + { 359, -1 }, /* (238) table_name_cond ::= table_name */ + { 360, 0 }, /* (239) from_db_opt ::= */ + { 360, -2 }, /* (240) from_db_opt ::= FROM db_name */ + { 309, -8 }, /* (241) cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ + { 309, -4 }, /* (242) cmd ::= DROP INDEX exists_opt full_table_name */ + { 361, -10 }, /* (243) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 361, -12 }, /* (244) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 362, -1 }, /* (245) func_list ::= func */ + { 362, -3 }, /* (246) func_list ::= func_list NK_COMMA func */ + { 365, -4 }, /* (247) func ::= function_name NK_LP expression_list NK_RP */ + { 364, 0 }, /* (248) sma_stream_opt ::= */ + { 364, -3 }, /* (249) sma_stream_opt ::= stream_options WATERMARK duration_literal */ + { 364, -3 }, /* (250) sma_stream_opt ::= stream_options MAX_DELAY duration_literal */ + { 309, -6 }, /* (251) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ + { 309, -7 }, /* (252) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 309, -9 }, /* (253) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 309, -7 }, /* (254) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 309, -9 }, /* (255) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 309, -4 }, /* (256) cmd ::= DROP TOPIC exists_opt topic_name */ + { 309, -7 }, /* (257) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 309, -2 }, /* (258) cmd ::= DESC full_table_name */ + { 309, -2 }, /* (259) cmd ::= DESCRIBE full_table_name */ + { 309, -3 }, /* (260) cmd ::= RESET QUERY CACHE */ + { 309, -4 }, /* (261) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ + { 370, 0 }, /* (262) analyze_opt ::= */ + { 370, -1 }, /* (263) analyze_opt ::= ANALYZE */ + { 371, 0 }, /* (264) explain_options ::= */ + { 371, -3 }, /* (265) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 371, -3 }, /* (266) explain_options ::= explain_options RATIO NK_FLOAT */ + { 309, -10 }, /* (267) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 309, -4 }, /* (268) cmd ::= DROP FUNCTION exists_opt function_name */ + { 372, 0 }, /* (269) agg_func_opt ::= */ + { 372, -1 }, /* (270) agg_func_opt ::= AGGREGATE */ + { 373, 0 }, /* (271) bufsize_opt ::= */ + { 373, -2 }, /* (272) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 309, -9 }, /* (273) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name AS query_expression */ + { 309, -4 }, /* (274) cmd ::= DROP STREAM exists_opt stream_name */ + { 366, 0 }, /* (275) stream_options ::= */ + { 366, -3 }, /* (276) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 366, -3 }, /* (277) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 366, -4 }, /* (278) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 366, -3 }, /* (279) stream_options ::= stream_options WATERMARK duration_literal */ + { 366, -4 }, /* (280) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 309, -3 }, /* (281) cmd ::= KILL CONNECTION NK_INTEGER */ + { 309, -3 }, /* (282) cmd ::= KILL QUERY NK_STRING */ + { 309, -3 }, /* (283) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 309, -2 }, /* (284) cmd ::= BALANCE VGROUP */ + { 309, -4 }, /* (285) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 309, -4 }, /* (286) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 309, -3 }, /* (287) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 375, -2 }, /* (288) dnode_list ::= DNODE NK_INTEGER */ + { 375, -3 }, /* (289) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 309, -4 }, /* (290) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 309, -1 }, /* (291) cmd ::= query_expression */ + { 309, -7 }, /* (292) cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_expression */ + { 309, -4 }, /* (293) cmd ::= INSERT INTO full_table_name query_expression */ + { 312, -1 }, /* (294) literal ::= NK_INTEGER */ + { 312, -1 }, /* (295) literal ::= NK_FLOAT */ + { 312, -1 }, /* (296) literal ::= NK_STRING */ + { 312, -1 }, /* (297) literal ::= NK_BOOL */ + { 312, -2 }, /* (298) literal ::= TIMESTAMP NK_STRING */ + { 312, -1 }, /* (299) literal ::= duration_literal */ + { 312, -1 }, /* (300) literal ::= NULL */ + { 312, -1 }, /* (301) literal ::= NK_QUESTION */ + { 353, -1 }, /* (302) duration_literal ::= NK_VARIABLE */ + { 377, -1 }, /* (303) signed ::= NK_INTEGER */ + { 377, -2 }, /* (304) signed ::= NK_PLUS NK_INTEGER */ + { 377, -2 }, /* (305) signed ::= NK_MINUS NK_INTEGER */ + { 377, -1 }, /* (306) signed ::= NK_FLOAT */ + { 377, -2 }, /* (307) signed ::= NK_PLUS NK_FLOAT */ + { 377, -2 }, /* (308) signed ::= NK_MINUS NK_FLOAT */ + { 342, -1 }, /* (309) signed_literal ::= signed */ + { 342, -1 }, /* (310) signed_literal ::= NK_STRING */ + { 342, -1 }, /* (311) signed_literal ::= NK_BOOL */ + { 342, -2 }, /* (312) signed_literal ::= TIMESTAMP NK_STRING */ + { 342, -1 }, /* (313) signed_literal ::= duration_literal */ + { 342, -1 }, /* (314) signed_literal ::= NULL */ + { 342, -1 }, /* (315) signed_literal ::= literal_func */ + { 342, -1 }, /* (316) signed_literal ::= NK_QUESTION */ + { 379, -1 }, /* (317) literal_list ::= signed_literal */ + { 379, -3 }, /* (318) literal_list ::= literal_list NK_COMMA signed_literal */ + { 320, -1 }, /* (319) db_name ::= NK_ID */ + { 348, -1 }, /* (320) table_name ::= NK_ID */ + { 340, -1 }, /* (321) column_name ::= NK_ID */ + { 355, -1 }, /* (322) function_name ::= NK_ID */ + { 380, -1 }, /* (323) table_alias ::= NK_ID */ + { 381, -1 }, /* (324) column_alias ::= NK_ID */ + { 314, -1 }, /* (325) user_name ::= NK_ID */ + { 367, -1 }, /* (326) topic_name ::= NK_ID */ + { 374, -1 }, /* (327) stream_name ::= NK_ID */ + { 369, -1 }, /* (328) cgroup_name ::= NK_ID */ + { 382, -1 }, /* (329) expression ::= literal */ + { 382, -1 }, /* (330) expression ::= pseudo_column */ + { 382, -1 }, /* (331) expression ::= column_reference */ + { 382, -1 }, /* (332) expression ::= function_expression */ + { 382, -1 }, /* (333) expression ::= subquery */ + { 382, -3 }, /* (334) expression ::= NK_LP expression NK_RP */ + { 382, -2 }, /* (335) expression ::= NK_PLUS expression */ + { 382, -2 }, /* (336) expression ::= NK_MINUS expression */ + { 382, -3 }, /* (337) expression ::= expression NK_PLUS expression */ + { 382, -3 }, /* (338) expression ::= expression NK_MINUS expression */ + { 382, -3 }, /* (339) expression ::= expression NK_STAR expression */ + { 382, -3 }, /* (340) expression ::= expression NK_SLASH expression */ + { 382, -3 }, /* (341) expression ::= expression NK_REM expression */ + { 382, -3 }, /* (342) expression ::= column_reference NK_ARROW NK_STRING */ + { 382, -3 }, /* (343) expression ::= expression NK_BITAND expression */ + { 382, -3 }, /* (344) expression ::= expression NK_BITOR expression */ + { 345, -1 }, /* (345) expression_list ::= expression */ + { 345, -3 }, /* (346) expression_list ::= expression_list NK_COMMA expression */ + { 384, -1 }, /* (347) column_reference ::= column_name */ + { 384, -3 }, /* (348) column_reference ::= table_name NK_DOT column_name */ + { 383, -1 }, /* (349) pseudo_column ::= ROWTS */ + { 383, -1 }, /* (350) pseudo_column ::= TBNAME */ + { 383, -3 }, /* (351) pseudo_column ::= table_name NK_DOT TBNAME */ + { 383, -1 }, /* (352) pseudo_column ::= QSTART */ + { 383, -1 }, /* (353) pseudo_column ::= QEND */ + { 383, -1 }, /* (354) pseudo_column ::= QDURATION */ + { 383, -1 }, /* (355) pseudo_column ::= WSTART */ + { 383, -1 }, /* (356) pseudo_column ::= WEND */ + { 383, -1 }, /* (357) pseudo_column ::= WDURATION */ + { 385, -4 }, /* (358) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 385, -4 }, /* (359) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 385, -6 }, /* (360) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + { 385, -1 }, /* (361) function_expression ::= literal_func */ + { 378, -3 }, /* (362) literal_func ::= noarg_func NK_LP NK_RP */ + { 378, -1 }, /* (363) literal_func ::= NOW */ + { 389, -1 }, /* (364) noarg_func ::= NOW */ + { 389, -1 }, /* (365) noarg_func ::= TODAY */ + { 389, -1 }, /* (366) noarg_func ::= TIMEZONE */ + { 389, -1 }, /* (367) noarg_func ::= DATABASE */ + { 389, -1 }, /* (368) noarg_func ::= CLIENT_VERSION */ + { 389, -1 }, /* (369) noarg_func ::= SERVER_VERSION */ + { 389, -1 }, /* (370) noarg_func ::= SERVER_STATUS */ + { 389, -1 }, /* (371) noarg_func ::= CURRENT_USER */ + { 389, -1 }, /* (372) noarg_func ::= USER */ + { 387, -1 }, /* (373) star_func ::= COUNT */ + { 387, -1 }, /* (374) star_func ::= FIRST */ + { 387, -1 }, /* (375) star_func ::= LAST */ + { 387, -1 }, /* (376) star_func ::= LAST_ROW */ + { 388, -1 }, /* (377) star_func_para_list ::= NK_STAR */ + { 388, -1 }, /* (378) star_func_para_list ::= other_para_list */ + { 390, -1 }, /* (379) other_para_list ::= star_func_para */ + { 390, -3 }, /* (380) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 391, -1 }, /* (381) star_func_para ::= expression */ + { 391, -3 }, /* (382) star_func_para ::= table_name NK_DOT NK_STAR */ + { 392, -3 }, /* (383) predicate ::= expression compare_op expression */ + { 392, -5 }, /* (384) predicate ::= expression BETWEEN expression AND expression */ + { 392, -6 }, /* (385) predicate ::= expression NOT BETWEEN expression AND expression */ + { 392, -3 }, /* (386) predicate ::= expression IS NULL */ + { 392, -4 }, /* (387) predicate ::= expression IS NOT NULL */ + { 392, -3 }, /* (388) predicate ::= expression in_op in_predicate_value */ + { 393, -1 }, /* (389) compare_op ::= NK_LT */ + { 393, -1 }, /* (390) compare_op ::= NK_GT */ + { 393, -1 }, /* (391) compare_op ::= NK_LE */ + { 393, -1 }, /* (392) compare_op ::= NK_GE */ + { 393, -1 }, /* (393) compare_op ::= NK_NE */ + { 393, -1 }, /* (394) compare_op ::= NK_EQ */ + { 393, -1 }, /* (395) compare_op ::= LIKE */ + { 393, -2 }, /* (396) compare_op ::= NOT LIKE */ + { 393, -1 }, /* (397) compare_op ::= MATCH */ + { 393, -1 }, /* (398) compare_op ::= NMATCH */ + { 393, -1 }, /* (399) compare_op ::= CONTAINS */ + { 394, -1 }, /* (400) in_op ::= IN */ + { 394, -2 }, /* (401) in_op ::= NOT IN */ + { 395, -3 }, /* (402) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 396, -1 }, /* (403) boolean_value_expression ::= boolean_primary */ + { 396, -2 }, /* (404) boolean_value_expression ::= NOT boolean_primary */ + { 396, -3 }, /* (405) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 396, -3 }, /* (406) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 397, -1 }, /* (407) boolean_primary ::= predicate */ + { 397, -3 }, /* (408) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 398, -1 }, /* (409) common_expression ::= expression */ + { 398, -1 }, /* (410) common_expression ::= boolean_value_expression */ + { 399, 0 }, /* (411) from_clause_opt ::= */ + { 399, -2 }, /* (412) from_clause_opt ::= FROM table_reference_list */ + { 400, -1 }, /* (413) table_reference_list ::= table_reference */ + { 400, -3 }, /* (414) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 401, -1 }, /* (415) table_reference ::= table_primary */ + { 401, -1 }, /* (416) table_reference ::= joined_table */ + { 402, -2 }, /* (417) table_primary ::= table_name alias_opt */ + { 402, -4 }, /* (418) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 402, -2 }, /* (419) table_primary ::= subquery alias_opt */ + { 402, -1 }, /* (420) table_primary ::= parenthesized_joined_table */ + { 404, 0 }, /* (421) alias_opt ::= */ + { 404, -1 }, /* (422) alias_opt ::= table_alias */ + { 404, -2 }, /* (423) alias_opt ::= AS table_alias */ + { 405, -3 }, /* (424) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 405, -3 }, /* (425) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 403, -6 }, /* (426) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 406, 0 }, /* (427) join_type ::= */ + { 406, -1 }, /* (428) join_type ::= INNER */ + { 408, -12 }, /* (429) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 409, 0 }, /* (430) set_quantifier_opt ::= */ + { 409, -1 }, /* (431) set_quantifier_opt ::= DISTINCT */ + { 409, -1 }, /* (432) set_quantifier_opt ::= ALL */ + { 410, -1 }, /* (433) select_list ::= select_item */ + { 410, -3 }, /* (434) select_list ::= select_list NK_COMMA select_item */ + { 418, -1 }, /* (435) select_item ::= NK_STAR */ + { 418, -1 }, /* (436) select_item ::= common_expression */ + { 418, -2 }, /* (437) select_item ::= common_expression column_alias */ + { 418, -3 }, /* (438) select_item ::= common_expression AS column_alias */ + { 418, -3 }, /* (439) select_item ::= table_name NK_DOT NK_STAR */ + { 376, 0 }, /* (440) where_clause_opt ::= */ + { 376, -2 }, /* (441) where_clause_opt ::= WHERE search_condition */ + { 411, 0 }, /* (442) partition_by_clause_opt ::= */ + { 411, -3 }, /* (443) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 415, 0 }, /* (444) twindow_clause_opt ::= */ + { 415, -6 }, /* (445) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 415, -4 }, /* (446) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + { 415, -6 }, /* (447) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 415, -8 }, /* (448) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 363, 0 }, /* (449) sliding_opt ::= */ + { 363, -4 }, /* (450) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 414, 0 }, /* (451) fill_opt ::= */ + { 414, -4 }, /* (452) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 414, -6 }, /* (453) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 419, -1 }, /* (454) fill_mode ::= NONE */ + { 419, -1 }, /* (455) fill_mode ::= PREV */ + { 419, -1 }, /* (456) fill_mode ::= NULL */ + { 419, -1 }, /* (457) fill_mode ::= LINEAR */ + { 419, -1 }, /* (458) fill_mode ::= NEXT */ + { 416, 0 }, /* (459) group_by_clause_opt ::= */ + { 416, -3 }, /* (460) group_by_clause_opt ::= GROUP BY group_by_list */ + { 420, -1 }, /* (461) group_by_list ::= expression */ + { 420, -3 }, /* (462) group_by_list ::= group_by_list NK_COMMA expression */ + { 417, 0 }, /* (463) having_clause_opt ::= */ + { 417, -2 }, /* (464) having_clause_opt ::= HAVING search_condition */ + { 412, 0 }, /* (465) range_opt ::= */ + { 412, -6 }, /* (466) range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ + { 413, 0 }, /* (467) every_opt ::= */ + { 413, -4 }, /* (468) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 368, -4 }, /* (469) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 421, -1 }, /* (470) query_expression_body ::= query_primary */ + { 421, -4 }, /* (471) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 421, -3 }, /* (472) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 425, -1 }, /* (473) query_primary ::= query_specification */ + { 425, -6 }, /* (474) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ + { 422, 0 }, /* (475) order_by_clause_opt ::= */ + { 422, -3 }, /* (476) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 423, 0 }, /* (477) slimit_clause_opt ::= */ + { 423, -2 }, /* (478) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 423, -4 }, /* (479) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 423, -4 }, /* (480) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 424, 0 }, /* (481) limit_clause_opt ::= */ + { 424, -2 }, /* (482) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 424, -4 }, /* (483) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 424, -4 }, /* (484) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 386, -3 }, /* (485) subquery ::= NK_LP query_expression NK_RP */ + { 407, -1 }, /* (486) search_condition ::= common_expression */ + { 426, -1 }, /* (487) sort_specification_list ::= sort_specification */ + { 426, -3 }, /* (488) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 427, -3 }, /* (489) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 428, 0 }, /* (490) ordering_specification_opt ::= */ + { 428, -1 }, /* (491) ordering_specification_opt ::= ASC */ + { 428, -1 }, /* (492) ordering_specification_opt ::= DESC */ + { 429, 0 }, /* (493) null_ordering_opt ::= */ + { 429, -2 }, /* (494) null_ordering_opt ::= NULLS FIRST */ + { 429, -2 }, /* (495) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3367,11 +3429,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,306,&yymsp[0].minor); + yy_destructor(yypParser,310,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,307,&yymsp[0].minor); + yy_destructor(yypParser,311,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3385,20 +3447,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,306,&yymsp[-2].minor); +{ yy_destructor(yypParser,310,&yymsp[-2].minor); { } - yy_destructor(yypParser,308,&yymsp[0].minor); + yy_destructor(yypParser,312,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,309,&yymsp[0].minor); +{ yy_destructor(yypParser,313,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,307,&yymsp[-1].minor); +{ yy_destructor(yypParser,311,&yymsp[-1].minor); { } - yy_destructor(yypParser,309,&yymsp[0].minor); + yy_destructor(yypParser,313,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3412,72 +3474,72 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,308,&yymsp[0].minor); + yy_destructor(yypParser,312,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy209, &yymsp[-1].minor.yy0, yymsp[0].minor.yy59); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy149, &yymsp[-1].minor.yy0, yymsp[0].minor.yy363); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy209, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy149, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy209, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy149, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy209, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy149, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy149); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy59 = 1; } +{ yymsp[1].minor.yy363 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy59 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy363 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy69, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy49, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149); } break; case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy69, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy49, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy69 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy49 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 35: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==35); -{ yylhsminor.yy69 = yymsp[0].minor.yy69; } - yymsp[0].minor.yy69 = yylhsminor.yy69; +{ yylhsminor.yy49 = yymsp[0].minor.yy49; } + yymsp[0].minor.yy49 = yylhsminor.yy49; break; case 36: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy69 = yymsp[-2].minor.yy69 | yymsp[0].minor.yy69; } - yymsp[-2].minor.yy69 = yylhsminor.yy69; +{ yylhsminor.yy49 = yymsp[-2].minor.yy49 | yymsp[0].minor.yy49; } + yymsp[-2].minor.yy49 = yylhsminor.yy49; break; case 37: /* priv_type ::= READ */ -{ yymsp[0].minor.yy69 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy49 = PRIVILEGE_TYPE_READ; } break; case 38: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy69 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy49 = PRIVILEGE_TYPE_WRITE; } break; case 39: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy209 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy209 = yylhsminor.yy209; +{ yylhsminor.yy149 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy149 = yylhsminor.yy149; break; case 40: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy209 = yymsp[-2].minor.yy209; } - yymsp[-2].minor.yy209 = yylhsminor.yy209; +{ yylhsminor.yy149 = yymsp[-2].minor.yy149; } + yymsp[-2].minor.yy149 = yylhsminor.yy149; break; case 41: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy209, NULL); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy149, NULL); } break; case 42: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0); } break; case 43: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; case 44: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy149); } break; case 45: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } @@ -3494,31 +3556,31 @@ static YYACTIONTYPE yy_reduce( case 49: /* dnode_endpoint ::= NK_STRING */ case 50: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==50); case 51: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==51); - case 312: /* db_name ::= NK_ID */ yytestcase(yyruleno==312); - case 313: /* table_name ::= NK_ID */ yytestcase(yyruleno==313); - case 314: /* column_name ::= NK_ID */ yytestcase(yyruleno==314); - case 315: /* function_name ::= NK_ID */ yytestcase(yyruleno==315); - case 316: /* table_alias ::= NK_ID */ yytestcase(yyruleno==316); - case 317: /* column_alias ::= NK_ID */ yytestcase(yyruleno==317); - case 318: /* user_name ::= NK_ID */ yytestcase(yyruleno==318); - case 319: /* topic_name ::= NK_ID */ yytestcase(yyruleno==319); - case 320: /* stream_name ::= NK_ID */ yytestcase(yyruleno==320); - case 321: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==321); - case 357: /* noarg_func ::= NOW */ yytestcase(yyruleno==357); - case 358: /* noarg_func ::= TODAY */ yytestcase(yyruleno==358); - case 359: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==359); - case 360: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==360); - case 361: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==361); - case 362: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==362); - case 363: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==363); - case 364: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==364); - case 365: /* noarg_func ::= USER */ yytestcase(yyruleno==365); - case 366: /* star_func ::= COUNT */ yytestcase(yyruleno==366); - case 367: /* star_func ::= FIRST */ yytestcase(yyruleno==367); - case 368: /* star_func ::= LAST */ yytestcase(yyruleno==368); - case 369: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==369); -{ yylhsminor.yy209 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy209 = yylhsminor.yy209; + case 319: /* db_name ::= NK_ID */ yytestcase(yyruleno==319); + case 320: /* table_name ::= NK_ID */ yytestcase(yyruleno==320); + case 321: /* column_name ::= NK_ID */ yytestcase(yyruleno==321); + case 322: /* function_name ::= NK_ID */ yytestcase(yyruleno==322); + case 323: /* table_alias ::= NK_ID */ yytestcase(yyruleno==323); + case 324: /* column_alias ::= NK_ID */ yytestcase(yyruleno==324); + case 325: /* user_name ::= NK_ID */ yytestcase(yyruleno==325); + case 326: /* topic_name ::= NK_ID */ yytestcase(yyruleno==326); + case 327: /* stream_name ::= NK_ID */ yytestcase(yyruleno==327); + case 328: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==328); + case 364: /* noarg_func ::= NOW */ yytestcase(yyruleno==364); + case 365: /* noarg_func ::= TODAY */ yytestcase(yyruleno==365); + case 366: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==366); + case 367: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==367); + case 368: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==368); + case 369: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==369); + case 370: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==370); + case 371: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==371); + case 372: /* noarg_func ::= USER */ yytestcase(yyruleno==372); + case 373: /* star_func ::= COUNT */ yytestcase(yyruleno==373); + case 374: /* star_func ::= FIRST */ yytestcase(yyruleno==374); + case 375: /* star_func ::= LAST */ yytestcase(yyruleno==375); + case 376: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==376); +{ yylhsminor.yy149 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy149 = yylhsminor.yy149; break; case 52: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } @@ -3551,1261 +3613,1286 @@ static YYACTIONTYPE yy_reduce( { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; case 62: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy293, &yymsp[-1].minor.yy209, yymsp[0].minor.yy272); } +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy497, &yymsp[-1].minor.yy149, yymsp[0].minor.yy312); } break; case 63: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy293, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy497, &yymsp[0].minor.yy149); } break; case 64: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy149); } break; case 65: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy209, yymsp[0].minor.yy272); } +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy149, yymsp[0].minor.yy312); } break; case 66: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy149); } break; case 67: /* cmd ::= TRIM DATABASE db_name */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[0].minor.yy209); } +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[0].minor.yy149); } break; case 68: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy293 = true; } +{ yymsp[-2].minor.yy497 = true; } break; case 69: /* not_exists_opt ::= */ case 71: /* exists_opt ::= */ yytestcase(yyruleno==71); - case 255: /* analyze_opt ::= */ yytestcase(yyruleno==255); - case 262: /* agg_func_opt ::= */ yytestcase(yyruleno==262); - case 423: /* set_quantifier_opt ::= */ yytestcase(yyruleno==423); -{ yymsp[1].minor.yy293 = false; } + case 262: /* analyze_opt ::= */ yytestcase(yyruleno==262); + case 269: /* agg_func_opt ::= */ yytestcase(yyruleno==269); + case 430: /* set_quantifier_opt ::= */ yytestcase(yyruleno==430); +{ yymsp[1].minor.yy497 = false; } break; case 70: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy293 = true; } +{ yymsp[-1].minor.yy497 = true; } break; case 72: /* db_options ::= */ -{ yymsp[1].minor.yy272 = createDefaultDatabaseOptions(pCxt); } +{ yymsp[1].minor.yy312 = createDefaultDatabaseOptions(pCxt); } break; case 73: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 74: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 75: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 76: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 77: /* db_options ::= db_options DURATION NK_INTEGER */ case 78: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==78); -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 79: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 80: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 81: /* db_options ::= db_options KEEP integer_list */ case 82: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==82); -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_KEEP, yymsp[0].minor.yy172); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_KEEP, yymsp[0].minor.yy824); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 83: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; case 84: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 85: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 85: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 86: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 86: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 87: /* db_options ::= db_options STRICT NK_STRING */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 87: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 88: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 88: /* db_options ::= db_options STRICT NK_STRING */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 89: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 89: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 90: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_RETENTIONS, yymsp[0].minor.yy172); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 90: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 91: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 91: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_RETENTIONS, yymsp[0].minor.yy824); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 92: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 92: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 93: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 93: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 94: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 94: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 95: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + case 95: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 96: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-3].minor.yy272, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-3].minor.yy312, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 96: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 97: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 97: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + case 98: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-3].minor.yy272, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-3].minor.yy312, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; - break; - case 98: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 99: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy272 = setDatabaseOption(pCxt, yymsp[-2].minor.yy272, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 100: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy272 = createAlterDatabaseOptions(pCxt); yylhsminor.yy272 = setAlterDatabaseOption(pCxt, yylhsminor.yy272, &yymsp[0].minor.yy5); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 101: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy272 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy272, &yymsp[0].minor.yy5); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; - break; - case 102: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy5.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy5.val = yymsp[0].minor.yy0; } - break; - case 103: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy5.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy5.val = yymsp[0].minor.yy0; } - break; - case 104: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy5.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy5.val = yymsp[0].minor.yy0; } - break; - case 105: /* alter_db_option ::= KEEP integer_list */ - case 106: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==106); -{ yymsp[-1].minor.yy5.type = DB_OPTION_KEEP; yymsp[-1].minor.yy5.pList = yymsp[0].minor.yy172; } - break; - case 107: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy5.type = DB_OPTION_WAL; yymsp[-1].minor.yy5.val = yymsp[0].minor.yy0; } - break; - case 108: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy172 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy172 = yylhsminor.yy172; - break; - case 109: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 282: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==282); -{ yylhsminor.yy172 = addNodeToList(pCxt, yymsp[-2].minor.yy172, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy172 = yylhsminor.yy172; - break; - case 110: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy172 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy172 = yylhsminor.yy172; - break; - case 111: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy172 = addNodeToList(pCxt, yymsp[-2].minor.yy172, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy172 = yylhsminor.yy172; - break; - case 112: /* retention_list ::= retention */ - case 132: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==132); - case 135: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==135); - case 142: /* column_def_list ::= column_def */ yytestcase(yyruleno==142); - case 185: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==185); - case 190: /* col_name_list ::= col_name */ yytestcase(yyruleno==190); - case 238: /* func_list ::= func */ yytestcase(yyruleno==238); - case 310: /* literal_list ::= signed_literal */ yytestcase(yyruleno==310); - case 372: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==372); - case 426: /* select_list ::= select_item */ yytestcase(yyruleno==426); - case 480: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==480); -{ yylhsminor.yy172 = createNodeList(pCxt, yymsp[0].minor.yy272); } - yymsp[0].minor.yy172 = yylhsminor.yy172; - break; - case 113: /* retention_list ::= retention_list NK_COMMA retention */ - case 143: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==143); - case 186: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==186); - case 191: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==191); - case 239: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==239); - case 311: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==311); - case 373: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==373); - case 427: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==427); - case 481: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==481); -{ yylhsminor.yy172 = addNodeToList(pCxt, yymsp[-2].minor.yy172, yymsp[0].minor.yy272); } - yymsp[-2].minor.yy172 = yylhsminor.yy172; - break; - case 114: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy272 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 115: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 117: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==117); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy293, yymsp[-5].minor.yy272, yymsp[-3].minor.yy172, yymsp[-1].minor.yy172, yymsp[0].minor.yy272); } - break; - case 116: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy172); } - break; - case 118: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy172); } - break; - case 119: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy293, yymsp[0].minor.yy272); } - break; - case 120: /* cmd ::= ALTER TABLE alter_table_clause */ - case 284: /* cmd ::= query_expression */ yytestcase(yyruleno==284); -{ pCxt->pRootNode = yymsp[0].minor.yy272; } - break; - case 121: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy272); } - break; - case 122: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy272 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; - break; - case 123: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy272 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy272, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy209, yymsp[0].minor.yy616); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; - break; - case 124: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy272 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy272, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy209); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; - break; - case 125: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy272 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy272, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy209, yymsp[0].minor.yy616); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + yymsp[-3].minor.yy312 = yylhsminor.yy312; + break; + case 99: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 100: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 101: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 102: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 103: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ +{ yylhsminor.yy312 = setDatabaseOption(pCxt, yymsp[-2].minor.yy312, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 104: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy312 = createAlterDatabaseOptions(pCxt); yylhsminor.yy312 = setAlterDatabaseOption(pCxt, yylhsminor.yy312, &yymsp[0].minor.yy405); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 105: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy312 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy312, &yymsp[0].minor.yy405); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; + break; + case 106: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy405.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } + break; + case 107: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy405.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } + break; + case 108: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy405.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } + break; + case 109: /* alter_db_option ::= KEEP integer_list */ + case 110: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==110); +{ yymsp[-1].minor.yy405.type = DB_OPTION_KEEP; yymsp[-1].minor.yy405.pList = yymsp[0].minor.yy824; } + break; + case 111: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +{ yymsp[-1].minor.yy405.type = DB_OPTION_WAL; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } + break; + case 112: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +{ yymsp[-1].minor.yy405.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } + break; + case 113: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy824 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy824 = yylhsminor.yy824; + break; + case 114: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 289: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==289); +{ yylhsminor.yy824 = addNodeToList(pCxt, yymsp[-2].minor.yy824, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy824 = yylhsminor.yy824; + break; + case 115: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy824 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy824 = yylhsminor.yy824; + break; + case 116: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy824 = addNodeToList(pCxt, yymsp[-2].minor.yy824, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy824 = yylhsminor.yy824; + break; + case 117: /* retention_list ::= retention */ + case 137: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==137); + case 140: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==140); + case 147: /* column_def_list ::= column_def */ yytestcase(yyruleno==147); + case 190: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==190); + case 195: /* col_name_list ::= col_name */ yytestcase(yyruleno==195); + case 245: /* func_list ::= func */ yytestcase(yyruleno==245); + case 317: /* literal_list ::= signed_literal */ yytestcase(yyruleno==317); + case 379: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==379); + case 433: /* select_list ::= select_item */ yytestcase(yyruleno==433); + case 487: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==487); +{ yylhsminor.yy824 = createNodeList(pCxt, yymsp[0].minor.yy312); } + yymsp[0].minor.yy824 = yylhsminor.yy824; + break; + case 118: /* retention_list ::= retention_list NK_COMMA retention */ + case 148: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==148); + case 191: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==191); + case 196: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==196); + case 246: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==246); + case 318: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==318); + case 380: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==380); + case 434: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==434); + case 488: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==488); +{ yylhsminor.yy824 = addNodeToList(pCxt, yymsp[-2].minor.yy824, yymsp[0].minor.yy312); } + yymsp[-2].minor.yy824 = yylhsminor.yy824; + break; + case 119: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy312 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 120: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 122: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==122); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy497, yymsp[-5].minor.yy312, yymsp[-3].minor.yy824, yymsp[-1].minor.yy824, yymsp[0].minor.yy312); } + break; + case 121: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy824); } + break; + case 123: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy824); } + break; + case 124: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy497, yymsp[0].minor.yy312); } + break; + case 125: /* cmd ::= ALTER TABLE alter_table_clause */ + case 291: /* cmd ::= query_expression */ yytestcase(yyruleno==291); +{ pCxt->pRootNode = yymsp[0].minor.yy312; } + break; + case 126: /* cmd ::= ALTER STABLE alter_table_clause */ +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy312); } + break; + case 127: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy312 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; + break; + case 128: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy312 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy312, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy149, yymsp[0].minor.yy84); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; + break; + case 129: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy312 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy312, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy149); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; + break; + case 130: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy312 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy312, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy149, yymsp[0].minor.yy84); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 126: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy272 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy272, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy209, &yymsp[0].minor.yy209); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + case 131: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy312 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy312, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 127: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy272 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy272, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy209, yymsp[0].minor.yy616); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; - break; - case 128: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy272 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy272, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy209); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 132: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy312 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy312, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy149, yymsp[0].minor.yy84); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; + break; + case 133: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy312 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy312, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy149); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 129: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy272 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy272, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy209, yymsp[0].minor.yy616); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + case 134: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy312 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy312, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy149, yymsp[0].minor.yy84); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 130: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy272 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy272, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy209, &yymsp[0].minor.yy209); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + case 135: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy312 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy312, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 131: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy272 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy272, &yymsp[-2].minor.yy209, yymsp[0].minor.yy272); } - yymsp[-5].minor.yy272 = yylhsminor.yy272; + case 136: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy312 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy312, &yymsp[-2].minor.yy149, yymsp[0].minor.yy312); } + yymsp[-5].minor.yy312 = yylhsminor.yy312; break; - case 133: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 136: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==136); -{ yylhsminor.yy172 = addNodeToList(pCxt, yymsp[-1].minor.yy172, yymsp[0].minor.yy272); } - yymsp[-1].minor.yy172 = yylhsminor.yy172; + case 138: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 141: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==141); +{ yylhsminor.yy824 = addNodeToList(pCxt, yymsp[-1].minor.yy824, yymsp[0].minor.yy312); } + yymsp[-1].minor.yy824 = yylhsminor.yy824; break; - case 134: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy272 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy293, yymsp[-8].minor.yy272, yymsp[-6].minor.yy272, yymsp[-5].minor.yy172, yymsp[-2].minor.yy172, yymsp[0].minor.yy272); } - yymsp[-9].minor.yy272 = yylhsminor.yy272; + case 139: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ +{ yylhsminor.yy312 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy497, yymsp[-8].minor.yy312, yymsp[-6].minor.yy312, yymsp[-5].minor.yy824, yymsp[-2].minor.yy824, yymsp[0].minor.yy312); } + yymsp[-9].minor.yy312 = yylhsminor.yy312; break; - case 137: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy272 = createDropTableClause(pCxt, yymsp[-1].minor.yy293, yymsp[0].minor.yy272); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 142: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy312 = createDropTableClause(pCxt, yymsp[-1].minor.yy497, yymsp[0].minor.yy312); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 138: /* specific_cols_opt ::= */ - case 169: /* tags_def_opt ::= */ yytestcase(yyruleno==169); - case 435: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==435); - case 452: /* group_by_clause_opt ::= */ yytestcase(yyruleno==452); - case 468: /* order_by_clause_opt ::= */ yytestcase(yyruleno==468); -{ yymsp[1].minor.yy172 = NULL; } + case 143: /* specific_cols_opt ::= */ + case 174: /* tags_def_opt ::= */ yytestcase(yyruleno==174); + case 442: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==442); + case 459: /* group_by_clause_opt ::= */ yytestcase(yyruleno==459); + case 475: /* order_by_clause_opt ::= */ yytestcase(yyruleno==475); +{ yymsp[1].minor.yy824 = NULL; } break; - case 139: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy172 = yymsp[-1].minor.yy172; } + case 144: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy824 = yymsp[-1].minor.yy824; } break; - case 140: /* full_table_name ::= table_name */ -{ yylhsminor.yy272 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy209, NULL); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 145: /* full_table_name ::= table_name */ +{ yylhsminor.yy312 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy149, NULL); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 141: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy272 = createRealTableNode(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy209, NULL); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 146: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy312 = createRealTableNode(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149, NULL); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 144: /* column_def ::= column_name type_name */ -{ yylhsminor.yy272 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy209, yymsp[0].minor.yy616, NULL); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 149: /* column_def ::= column_name type_name */ +{ yylhsminor.yy312 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy149, yymsp[0].minor.yy84, NULL); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 145: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy272 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy209, yymsp[-2].minor.yy616, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 150: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy312 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy149, yymsp[-2].minor.yy84, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 146: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 151: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 147: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 152: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 148: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 153: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 149: /* type_name ::= INT */ - case 150: /* type_name ::= INTEGER */ yytestcase(yyruleno==150); -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_INT); } + case 154: /* type_name ::= INT */ + case 155: /* type_name ::= INTEGER */ yytestcase(yyruleno==155); +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 151: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 156: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 152: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 157: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 153: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 158: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 154: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy616 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 159: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy84 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 155: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 160: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 156: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy616 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 161: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy84 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 157: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy616 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 162: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy84 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 158: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy616 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 163: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy84 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 159: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy616 = createDataType(TSDB_DATA_TYPE_UINT); } + case 164: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy84 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 160: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy616 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 165: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy84 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 161: /* type_name ::= JSON */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_JSON); } + case 166: /* type_name ::= JSON */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 162: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy616 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 167: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy84 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 163: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 168: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 164: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 169: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 165: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy616 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 170: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy84 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 166: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy616 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 171: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy84 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 167: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy616 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 172: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy84 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 168: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy616 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 173: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy84 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 170: /* tags_def_opt ::= tags_def */ - case 371: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==371); -{ yylhsminor.yy172 = yymsp[0].minor.yy172; } - yymsp[0].minor.yy172 = yylhsminor.yy172; + case 175: /* tags_def_opt ::= tags_def */ + case 378: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==378); +{ yylhsminor.yy824 = yymsp[0].minor.yy824; } + yymsp[0].minor.yy824 = yylhsminor.yy824; break; - case 171: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy172 = yymsp[-1].minor.yy172; } + case 176: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy824 = yymsp[-1].minor.yy824; } break; - case 172: /* table_options ::= */ -{ yymsp[1].minor.yy272 = createDefaultTableOptions(pCxt); } + case 177: /* table_options ::= */ +{ yymsp[1].minor.yy312 = createDefaultTableOptions(pCxt); } break; - case 173: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-2].minor.yy272, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 178: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-2].minor.yy312, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 174: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-2].minor.yy272, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy172); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 179: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-2].minor.yy312, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy824); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 175: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-2].minor.yy272, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy172); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 180: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-2].minor.yy312, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy824); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 176: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-4].minor.yy272, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy172); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + case 181: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-4].minor.yy312, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy824); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 177: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-2].minor.yy272, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 182: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-2].minor.yy312, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 178: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-4].minor.yy272, TABLE_OPTION_SMA, yymsp[-1].minor.yy172); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + case 183: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-4].minor.yy312, TABLE_OPTION_SMA, yymsp[-1].minor.yy824); } + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 179: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy272 = createAlterTableOptions(pCxt); yylhsminor.yy272 = setTableOption(pCxt, yylhsminor.yy272, yymsp[0].minor.yy5.type, &yymsp[0].minor.yy5.val); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 184: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy312 = createAlterTableOptions(pCxt); yylhsminor.yy312 = setTableOption(pCxt, yylhsminor.yy312, yymsp[0].minor.yy405.type, &yymsp[0].minor.yy405.val); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 180: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy272 = setTableOption(pCxt, yymsp[-1].minor.yy272, yymsp[0].minor.yy5.type, &yymsp[0].minor.yy5.val); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 185: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy312 = setTableOption(pCxt, yymsp[-1].minor.yy312, yymsp[0].minor.yy405.type, &yymsp[0].minor.yy405.val); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 181: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy5.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy5.val = yymsp[0].minor.yy0; } + case 186: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy405.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } break; - case 182: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy5.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy5.val = yymsp[0].minor.yy0; } + case 187: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy405.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy405.val = yymsp[0].minor.yy0; } break; - case 183: /* duration_list ::= duration_literal */ - case 338: /* expression_list ::= expression */ yytestcase(yyruleno==338); -{ yylhsminor.yy172 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy272)); } - yymsp[0].minor.yy172 = yylhsminor.yy172; + case 188: /* duration_list ::= duration_literal */ + case 345: /* expression_list ::= expression */ yytestcase(yyruleno==345); +{ yylhsminor.yy824 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy312)); } + yymsp[0].minor.yy824 = yylhsminor.yy824; break; - case 184: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 339: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==339); -{ yylhsminor.yy172 = addNodeToList(pCxt, yymsp[-2].minor.yy172, releaseRawExprNode(pCxt, yymsp[0].minor.yy272)); } - yymsp[-2].minor.yy172 = yylhsminor.yy172; + case 189: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 346: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==346); +{ yylhsminor.yy824 = addNodeToList(pCxt, yymsp[-2].minor.yy824, releaseRawExprNode(pCxt, yymsp[0].minor.yy312)); } + yymsp[-2].minor.yy824 = yylhsminor.yy824; break; - case 187: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy272 = createFunctionNode(pCxt, &yymsp[0].minor.yy209, NULL); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 192: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy312 = createFunctionNode(pCxt, &yymsp[0].minor.yy149, NULL); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 188: /* rollup_func_name ::= FIRST */ - case 189: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==189); -{ yylhsminor.yy272 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 193: /* rollup_func_name ::= FIRST */ + case 194: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==194); +{ yylhsminor.yy312 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 192: /* col_name ::= column_name */ -{ yylhsminor.yy272 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy209); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 197: /* col_name ::= column_name */ +{ yylhsminor.yy312 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy149); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 193: /* cmd ::= SHOW DNODES */ + case 198: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 194: /* cmd ::= SHOW USERS */ + case 199: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 195: /* cmd ::= SHOW DATABASES */ + case 200: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 196: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy272, yymsp[0].minor.yy272, OP_TYPE_LIKE); } + case 201: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy312, yymsp[0].minor.yy312, OP_TYPE_LIKE); } break; - case 197: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy272, yymsp[0].minor.yy272, OP_TYPE_LIKE); } + case 202: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy312, yymsp[0].minor.yy312, OP_TYPE_LIKE); } break; - case 198: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy272, NULL, OP_TYPE_LIKE); } + case 203: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy312, NULL, OP_TYPE_LIKE); } break; - case 199: /* cmd ::= SHOW MNODES */ + case 204: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 200: /* cmd ::= SHOW MODULES */ + case 205: /* cmd ::= SHOW MODULES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT); } break; - case 201: /* cmd ::= SHOW QNODES */ + case 206: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 202: /* cmd ::= SHOW FUNCTIONS */ + case 207: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 203: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy272, yymsp[-1].minor.yy272, OP_TYPE_EQUAL); } + case 208: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy312, yymsp[-1].minor.yy312, OP_TYPE_EQUAL); } break; - case 204: /* cmd ::= SHOW STREAMS */ + case 209: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 205: /* cmd ::= SHOW ACCOUNTS */ + case 210: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 206: /* cmd ::= SHOW APPS */ + case 211: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 207: /* cmd ::= SHOW CONNECTIONS */ + case 212: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 208: /* cmd ::= SHOW LICENCES */ - case 209: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==209); + case 213: /* cmd ::= SHOW LICENCES */ + case 214: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==214); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 210: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy209); } + case 215: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy149); } break; - case 211: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy272); } + case 216: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy312); } break; - case 212: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy272); } + case 217: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy312); } break; - case 213: /* cmd ::= SHOW QUERIES */ + case 218: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 214: /* cmd ::= SHOW SCORES */ + case 219: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 215: /* cmd ::= SHOW TOPICS */ + case 220: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 216: /* cmd ::= SHOW VARIABLES */ + case 221: /* cmd ::= SHOW VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 217: /* cmd ::= SHOW LOCAL VARIABLES */ + case 222: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 218: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES */ + case 223: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES */ { pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-1].minor.yy0)); } break; - case 219: /* cmd ::= SHOW BNODES */ + case 224: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 220: /* cmd ::= SHOW SNODES */ + case 225: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 221: /* cmd ::= SHOW CLUSTER */ + case 226: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 222: /* cmd ::= SHOW TRANSACTIONS */ + case 227: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 223: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy272); } + case 228: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy312); } break; - case 224: /* cmd ::= SHOW CONSUMERS */ + case 229: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 225: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 230: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 226: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy272, yymsp[-1].minor.yy272, OP_TYPE_EQUAL); } + case 231: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy312, yymsp[-1].minor.yy312, OP_TYPE_EQUAL); } + break; + case 232: /* cmd ::= SHOW VNODES NK_INTEGER */ +{ pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } + break; + case 233: /* cmd ::= SHOW VNODES NK_STRING */ +{ pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 227: /* db_name_cond_opt ::= */ - case 232: /* from_db_opt ::= */ yytestcase(yyruleno==232); -{ yymsp[1].minor.yy272 = createDefaultDatabaseCondValue(pCxt); } + case 234: /* db_name_cond_opt ::= */ + case 239: /* from_db_opt ::= */ yytestcase(yyruleno==239); +{ yymsp[1].minor.yy312 = createDefaultDatabaseCondValue(pCxt); } break; - case 228: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy209); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 235: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy149); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 229: /* like_pattern_opt ::= */ - case 404: /* from_clause_opt ::= */ yytestcase(yyruleno==404); - case 433: /* where_clause_opt ::= */ yytestcase(yyruleno==433); - case 437: /* twindow_clause_opt ::= */ yytestcase(yyruleno==437); - case 442: /* sliding_opt ::= */ yytestcase(yyruleno==442); - case 444: /* fill_opt ::= */ yytestcase(yyruleno==444); - case 456: /* having_clause_opt ::= */ yytestcase(yyruleno==456); - case 458: /* range_opt ::= */ yytestcase(yyruleno==458); - case 460: /* every_opt ::= */ yytestcase(yyruleno==460); - case 470: /* slimit_clause_opt ::= */ yytestcase(yyruleno==470); - case 474: /* limit_clause_opt ::= */ yytestcase(yyruleno==474); -{ yymsp[1].minor.yy272 = NULL; } + case 236: /* like_pattern_opt ::= */ + case 411: /* from_clause_opt ::= */ yytestcase(yyruleno==411); + case 440: /* where_clause_opt ::= */ yytestcase(yyruleno==440); + case 444: /* twindow_clause_opt ::= */ yytestcase(yyruleno==444); + case 449: /* sliding_opt ::= */ yytestcase(yyruleno==449); + case 451: /* fill_opt ::= */ yytestcase(yyruleno==451); + case 463: /* having_clause_opt ::= */ yytestcase(yyruleno==463); + case 465: /* range_opt ::= */ yytestcase(yyruleno==465); + case 467: /* every_opt ::= */ yytestcase(yyruleno==467); + case 477: /* slimit_clause_opt ::= */ yytestcase(yyruleno==477); + case 481: /* limit_clause_opt ::= */ yytestcase(yyruleno==481); +{ yymsp[1].minor.yy312 = NULL; } break; - case 230: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 237: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 231: /* table_name_cond ::= table_name */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy209); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 238: /* table_name_cond ::= table_name */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy149); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 233: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy209); } + case 240: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy149); } break; - case 234: /* cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy293, yymsp[-3].minor.yy272, yymsp[-1].minor.yy272, NULL, yymsp[0].minor.yy272); } + case 241: /* cmd ::= CREATE SMA INDEX not_exists_opt full_table_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy497, yymsp[-3].minor.yy312, yymsp[-1].minor.yy312, NULL, yymsp[0].minor.yy312); } break; - case 235: /* cmd ::= DROP INDEX exists_opt full_table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy293, yymsp[0].minor.yy272); } + case 242: /* cmd ::= DROP INDEX exists_opt full_table_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy497, yymsp[0].minor.yy312); } break; - case 236: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy272 = createIndexOption(pCxt, yymsp[-7].minor.yy172, releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), NULL, yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } + case 243: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy312 = createIndexOption(pCxt, yymsp[-7].minor.yy824, releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), NULL, yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } break; - case 237: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy272 = createIndexOption(pCxt, yymsp[-9].minor.yy172, releaseRawExprNode(pCxt, yymsp[-5].minor.yy272), releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } + case 244: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy312 = createIndexOption(pCxt, yymsp[-9].minor.yy824, releaseRawExprNode(pCxt, yymsp[-5].minor.yy312), releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } break; - case 240: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy272 = createFunctionNode(pCxt, &yymsp[-3].minor.yy209, yymsp[-1].minor.yy172); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 247: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy312 = createFunctionNode(pCxt, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy824); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 241: /* sma_stream_opt ::= */ - case 268: /* stream_options ::= */ yytestcase(yyruleno==268); -{ yymsp[1].minor.yy272 = createStreamOptions(pCxt); } + case 248: /* sma_stream_opt ::= */ + case 275: /* stream_options ::= */ yytestcase(yyruleno==275); +{ yymsp[1].minor.yy312 = createStreamOptions(pCxt); } break; - case 242: /* sma_stream_opt ::= stream_options WATERMARK duration_literal */ - case 272: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==272); -{ ((SStreamOptions*)yymsp[-2].minor.yy272)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy272); yylhsminor.yy272 = yymsp[-2].minor.yy272; } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 249: /* sma_stream_opt ::= stream_options WATERMARK duration_literal */ + case 279: /* stream_options ::= stream_options WATERMARK duration_literal */ yytestcase(yyruleno==279); +{ ((SStreamOptions*)yymsp[-2].minor.yy312)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy312); yylhsminor.yy312 = yymsp[-2].minor.yy312; } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 243: /* sma_stream_opt ::= stream_options MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy272)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy272); yylhsminor.yy272 = yymsp[-2].minor.yy272; } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 250: /* sma_stream_opt ::= stream_options MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy312)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy312); yylhsminor.yy312 = yymsp[-2].minor.yy312; } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 244: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy293, &yymsp[-2].minor.yy209, yymsp[0].minor.yy272); } + case 251: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy497, &yymsp[-2].minor.yy149, yymsp[0].minor.yy312); } break; - case 245: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy293, &yymsp[-3].minor.yy209, &yymsp[0].minor.yy209, false); } + case 252: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy497, &yymsp[-3].minor.yy149, &yymsp[0].minor.yy149, false); } break; - case 246: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy293, &yymsp[-5].minor.yy209, &yymsp[0].minor.yy209, true); } + case 253: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy497, &yymsp[-5].minor.yy149, &yymsp[0].minor.yy149, true); } break; - case 247: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy293, &yymsp[-3].minor.yy209, yymsp[0].minor.yy272, false); } + case 254: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy497, &yymsp[-3].minor.yy149, yymsp[0].minor.yy312, false); } break; - case 248: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy293, &yymsp[-5].minor.yy209, yymsp[0].minor.yy272, true); } + case 255: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy497, &yymsp[-5].minor.yy149, yymsp[0].minor.yy312, true); } break; - case 249: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy293, &yymsp[0].minor.yy209); } + case 256: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy497, &yymsp[0].minor.yy149); } break; - case 250: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy293, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy209); } + case 257: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy497, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149); } break; - case 251: /* cmd ::= DESC full_table_name */ - case 252: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==252); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy272); } + case 258: /* cmd ::= DESC full_table_name */ + case 259: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==259); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy312); } break; - case 253: /* cmd ::= RESET QUERY CACHE */ + case 260: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 254: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy293, yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } + case 261: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy497, yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } break; - case 256: /* analyze_opt ::= ANALYZE */ - case 263: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==263); - case 424: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==424); -{ yymsp[0].minor.yy293 = true; } + case 263: /* analyze_opt ::= ANALYZE */ + case 270: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==270); + case 431: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==431); +{ yymsp[0].minor.yy497 = true; } break; - case 257: /* explain_options ::= */ -{ yymsp[1].minor.yy272 = createDefaultExplainOptions(pCxt); } + case 264: /* explain_options ::= */ +{ yymsp[1].minor.yy312 = createDefaultExplainOptions(pCxt); } break; - case 258: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy272 = setExplainVerbose(pCxt, yymsp[-2].minor.yy272, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 265: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy312 = setExplainVerbose(pCxt, yymsp[-2].minor.yy312, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 259: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy272 = setExplainRatio(pCxt, yymsp[-2].minor.yy272, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 266: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy312 = setExplainRatio(pCxt, yymsp[-2].minor.yy312, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 260: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy293, yymsp[-8].minor.yy293, &yymsp[-5].minor.yy209, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy616, yymsp[0].minor.yy232); } + case 267: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy497, yymsp[-8].minor.yy497, &yymsp[-5].minor.yy149, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy84, yymsp[0].minor.yy160); } break; - case 261: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy293, &yymsp[0].minor.yy209); } + case 268: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy497, &yymsp[0].minor.yy149); } break; - case 264: /* bufsize_opt ::= */ -{ yymsp[1].minor.yy232 = 0; } + case 271: /* bufsize_opt ::= */ +{ yymsp[1].minor.yy160 = 0; } break; - case 265: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy232 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + case 272: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ +{ yymsp[-1].minor.yy160 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 266: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name AS query_expression */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-6].minor.yy293, &yymsp[-5].minor.yy209, yymsp[-2].minor.yy272, yymsp[-4].minor.yy272, yymsp[0].minor.yy272); } + case 273: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name AS query_expression */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-6].minor.yy497, &yymsp[-5].minor.yy149, yymsp[-2].minor.yy312, yymsp[-4].minor.yy312, yymsp[0].minor.yy312); } break; - case 267: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy293, &yymsp[0].minor.yy209); } + case 274: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy497, &yymsp[0].minor.yy149); } break; - case 269: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy272)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy272 = yymsp[-2].minor.yy272; } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 276: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy312)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy312 = yymsp[-2].minor.yy312; } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 270: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy272)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy272 = yymsp[-2].minor.yy272; } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 277: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy312)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy312 = yymsp[-2].minor.yy312; } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 271: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy272)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy272)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy272); yylhsminor.yy272 = yymsp[-3].minor.yy272; } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 278: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-3].minor.yy312)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy312)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy312); yylhsminor.yy312 = yymsp[-3].minor.yy312; } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 273: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ ((SStreamOptions*)yymsp[-3].minor.yy272)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy272 = yymsp[-3].minor.yy272; } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 280: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ ((SStreamOptions*)yymsp[-3].minor.yy312)->ignoreExpired = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); yylhsminor.yy312 = yymsp[-3].minor.yy312; } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 274: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 281: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 275: /* cmd ::= KILL QUERY NK_STRING */ + case 282: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 276: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 283: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 277: /* cmd ::= BALANCE VGROUP */ + case 284: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 278: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 285: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 279: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy172); } + case 286: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy824); } break; - case 280: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 287: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 281: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy172 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 283: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } - break; - case 285: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_expression */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy272, yymsp[-2].minor.yy172, yymsp[0].minor.yy272); } - break; - case 286: /* cmd ::= INSERT INTO full_table_name query_expression */ -{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy272, NULL, yymsp[0].minor.yy272); } - break; - case 287: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 288: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 289: /* literal ::= NK_STRING */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 290: /* literal ::= NK_BOOL */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 291: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; - break; - case 292: /* literal ::= duration_literal */ - case 302: /* signed_literal ::= signed */ yytestcase(yyruleno==302); - case 322: /* expression ::= literal */ yytestcase(yyruleno==322); - case 323: /* expression ::= pseudo_column */ yytestcase(yyruleno==323); - case 324: /* expression ::= column_reference */ yytestcase(yyruleno==324); - case 325: /* expression ::= function_expression */ yytestcase(yyruleno==325); - case 326: /* expression ::= subquery */ yytestcase(yyruleno==326); - case 354: /* function_expression ::= literal_func */ yytestcase(yyruleno==354); - case 396: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==396); - case 400: /* boolean_primary ::= predicate */ yytestcase(yyruleno==400); - case 402: /* common_expression ::= expression */ yytestcase(yyruleno==402); - case 403: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==403); - case 406: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==406); - case 408: /* table_reference ::= table_primary */ yytestcase(yyruleno==408); - case 409: /* table_reference ::= joined_table */ yytestcase(yyruleno==409); - case 413: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==413); - case 463: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==463); - case 466: /* query_primary ::= query_specification */ yytestcase(yyruleno==466); -{ yylhsminor.yy272 = yymsp[0].minor.yy272; } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 293: /* literal ::= NULL */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 294: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 295: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 296: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 297: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 298: /* signed ::= NK_MINUS NK_INTEGER */ + case 288: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy824 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 290: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } + break; + case 292: /* cmd ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_expression */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-4].minor.yy312, yymsp[-2].minor.yy824, yymsp[0].minor.yy312); } + break; + case 293: /* cmd ::= INSERT INTO full_table_name query_expression */ +{ pCxt->pRootNode = createInsertStmt(pCxt, yymsp[-1].minor.yy312, NULL, yymsp[0].minor.yy312); } + break; + case 294: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 295: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 296: /* literal ::= NK_STRING */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 297: /* literal ::= NK_BOOL */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 298: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; + break; + case 299: /* literal ::= duration_literal */ + case 309: /* signed_literal ::= signed */ yytestcase(yyruleno==309); + case 329: /* expression ::= literal */ yytestcase(yyruleno==329); + case 330: /* expression ::= pseudo_column */ yytestcase(yyruleno==330); + case 331: /* expression ::= column_reference */ yytestcase(yyruleno==331); + case 332: /* expression ::= function_expression */ yytestcase(yyruleno==332); + case 333: /* expression ::= subquery */ yytestcase(yyruleno==333); + case 361: /* function_expression ::= literal_func */ yytestcase(yyruleno==361); + case 403: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==403); + case 407: /* boolean_primary ::= predicate */ yytestcase(yyruleno==407); + case 409: /* common_expression ::= expression */ yytestcase(yyruleno==409); + case 410: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==410); + case 413: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==413); + case 415: /* table_reference ::= table_primary */ yytestcase(yyruleno==415); + case 416: /* table_reference ::= joined_table */ yytestcase(yyruleno==416); + case 420: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==420); + case 470: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==470); + case 473: /* query_primary ::= query_specification */ yytestcase(yyruleno==473); +{ yylhsminor.yy312 = yymsp[0].minor.yy312; } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 300: /* literal ::= NULL */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 301: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 302: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 303: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 304: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 305: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 299: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 306: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 300: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 307: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 301: /* signed ::= NK_MINUS NK_FLOAT */ + case 308: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 303: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 310: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 304: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 311: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 305: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + case 312: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 306: /* signed_literal ::= duration_literal */ - case 308: /* signed_literal ::= literal_func */ yytestcase(yyruleno==308); - case 374: /* star_func_para ::= expression */ yytestcase(yyruleno==374); - case 429: /* select_item ::= common_expression */ yytestcase(yyruleno==429); - case 479: /* search_condition ::= common_expression */ yytestcase(yyruleno==479); -{ yylhsminor.yy272 = releaseRawExprNode(pCxt, yymsp[0].minor.yy272); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 313: /* signed_literal ::= duration_literal */ + case 315: /* signed_literal ::= literal_func */ yytestcase(yyruleno==315); + case 381: /* star_func_para ::= expression */ yytestcase(yyruleno==381); + case 436: /* select_item ::= common_expression */ yytestcase(yyruleno==436); + case 486: /* search_condition ::= common_expression */ yytestcase(yyruleno==486); +{ yylhsminor.yy312 = releaseRawExprNode(pCxt, yymsp[0].minor.yy312); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 307: /* signed_literal ::= NULL */ -{ yylhsminor.yy272 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 314: /* signed_literal ::= NULL */ +{ yylhsminor.yy312 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 309: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy272 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 316: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy312 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 327: /* expression ::= NK_LP expression NK_RP */ - case 401: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==401); -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy272)); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 334: /* expression ::= NK_LP expression NK_RP */ + case 408: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==408); +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy312)); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 328: /* expression ::= NK_PLUS expression */ + case 335: /* expression ::= NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy272)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy312)); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 329: /* expression ::= NK_MINUS expression */ + case 336: /* expression ::= NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy272), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy312), NULL)); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 330: /* expression ::= expression NK_PLUS expression */ + case 337: /* expression ::= expression NK_PLUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 331: /* expression ::= expression NK_MINUS expression */ + case 338: /* expression ::= expression NK_MINUS expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 332: /* expression ::= expression NK_STAR expression */ + case 339: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 333: /* expression ::= expression NK_SLASH expression */ + case 340: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 334: /* expression ::= expression NK_REM expression */ + case 341: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 335: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 342: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 336: /* expression ::= expression NK_BITAND expression */ + case 343: /* expression ::= expression NK_BITAND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 337: /* expression ::= expression NK_BITOR expression */ + case 344: /* expression ::= expression NK_BITOR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 340: /* column_reference ::= column_name */ -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy209, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy209)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 341: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy209, createColumnNode(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy209)); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 342: /* pseudo_column ::= ROWTS */ - case 343: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==343); - case 345: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==345); - case 346: /* pseudo_column ::= QEND */ yytestcase(yyruleno==346); - case 347: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==347); - case 348: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==348); - case 349: /* pseudo_column ::= WEND */ yytestcase(yyruleno==349); - case 350: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==350); - case 356: /* literal_func ::= NOW */ yytestcase(yyruleno==356); -{ yylhsminor.yy272 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy272 = yylhsminor.yy272; - break; - case 344: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy209)))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 351: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 352: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==352); -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy209, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy209, yymsp[-1].minor.yy172)); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; - break; - case 353: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), yymsp[-1].minor.yy616)); } - yymsp[-5].minor.yy272 = yylhsminor.yy272; - break; - case 355: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy209, NULL)); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 370: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy172 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy172 = yylhsminor.yy172; - break; - case 375: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 432: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==432); -{ yylhsminor.yy272 = createColumnNode(pCxt, &yymsp[-2].minor.yy209, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; - break; - case 376: /* predicate ::= expression compare_op expression */ - case 381: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==381); + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 347: /* column_reference ::= column_name */ +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy149, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy149)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 348: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149, createColumnNode(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy149)); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 349: /* pseudo_column ::= ROWTS */ + case 350: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==350); + case 352: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==352); + case 353: /* pseudo_column ::= QEND */ yytestcase(yyruleno==353); + case 354: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==354); + case 355: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==355); + case 356: /* pseudo_column ::= WEND */ yytestcase(yyruleno==356); + case 357: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==357); + case 363: /* literal_func ::= NOW */ yytestcase(yyruleno==363); +{ yylhsminor.yy312 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy312 = yylhsminor.yy312; + break; + case 351: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy149)))); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 358: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 359: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==359); +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy149, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy149, yymsp[-1].minor.yy824)); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; + break; + case 360: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), yymsp[-1].minor.yy84)); } + yymsp[-5].minor.yy312 = yylhsminor.yy312; + break; + case 362: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy149, NULL)); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 377: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy824 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy824 = yylhsminor.yy824; + break; + case 382: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 439: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==439); +{ yylhsminor.yy312 = createColumnNode(pCxt, &yymsp[-2].minor.yy149, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; + break; + case 383: /* predicate ::= expression compare_op expression */ + case 388: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==388); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy392, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy320, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 377: /* predicate ::= expression BETWEEN expression AND expression */ + case 384: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy272), releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy312), releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-4].minor.yy272 = yylhsminor.yy272; + yymsp[-4].minor.yy312 = yylhsminor.yy312; break; - case 378: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 385: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy272), releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy312), releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-5].minor.yy272 = yylhsminor.yy272; + yymsp[-5].minor.yy312 = yylhsminor.yy312; break; - case 379: /* predicate ::= expression IS NULL */ + case 386: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), NULL)); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 380: /* predicate ::= expression IS NOT NULL */ + case 387: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), NULL)); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 382: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy392 = OP_TYPE_LOWER_THAN; } + case 389: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy320 = OP_TYPE_LOWER_THAN; } break; - case 383: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy392 = OP_TYPE_GREATER_THAN; } + case 390: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy320 = OP_TYPE_GREATER_THAN; } break; - case 384: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy392 = OP_TYPE_LOWER_EQUAL; } + case 391: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy320 = OP_TYPE_LOWER_EQUAL; } break; - case 385: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy392 = OP_TYPE_GREATER_EQUAL; } + case 392: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy320 = OP_TYPE_GREATER_EQUAL; } break; - case 386: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy392 = OP_TYPE_NOT_EQUAL; } + case 393: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy320 = OP_TYPE_NOT_EQUAL; } break; - case 387: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy392 = OP_TYPE_EQUAL; } + case 394: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy320 = OP_TYPE_EQUAL; } break; - case 388: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy392 = OP_TYPE_LIKE; } + case 395: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy320 = OP_TYPE_LIKE; } break; - case 389: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy392 = OP_TYPE_NOT_LIKE; } + case 396: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy320 = OP_TYPE_NOT_LIKE; } break; - case 390: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy392 = OP_TYPE_MATCH; } + case 397: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy320 = OP_TYPE_MATCH; } break; - case 391: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy392 = OP_TYPE_NMATCH; } + case 398: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy320 = OP_TYPE_NMATCH; } break; - case 392: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy392 = OP_TYPE_JSON_CONTAINS; } + case 399: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy320 = OP_TYPE_JSON_CONTAINS; } break; - case 393: /* in_op ::= IN */ -{ yymsp[0].minor.yy392 = OP_TYPE_IN; } + case 400: /* in_op ::= IN */ +{ yymsp[0].minor.yy320 = OP_TYPE_IN; } break; - case 394: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy392 = OP_TYPE_NOT_IN; } + case 401: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy320 = OP_TYPE_NOT_IN; } break; - case 395: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy172)); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 402: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy824)); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 397: /* boolean_value_expression ::= NOT boolean_primary */ + case 404: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy272), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy312), NULL)); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 398: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 405: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 399: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 406: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy272); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy272); - yylhsminor.yy272 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy312); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy312); + yylhsminor.yy312 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 405: /* from_clause_opt ::= FROM table_reference_list */ - case 434: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==434); - case 457: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==457); -{ yymsp[-1].minor.yy272 = yymsp[0].minor.yy272; } + case 412: /* from_clause_opt ::= FROM table_reference_list */ + case 441: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==441); + case 464: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==464); +{ yymsp[-1].minor.yy312 = yymsp[0].minor.yy312; } break; - case 407: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy272 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy272, yymsp[0].minor.yy272, NULL); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 414: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy312 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy312, yymsp[0].minor.yy312, NULL); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 410: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy272 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy209, &yymsp[0].minor.yy209); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 417: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy312 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 411: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy272 = createRealTableNode(pCxt, &yymsp[-3].minor.yy209, &yymsp[-1].minor.yy209, &yymsp[0].minor.yy209); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 418: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy312 = createRealTableNode(pCxt, &yymsp[-3].minor.yy149, &yymsp[-1].minor.yy149, &yymsp[0].minor.yy149); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 412: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy272 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy272), &yymsp[0].minor.yy209); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 419: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy312 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy312), &yymsp[0].minor.yy149); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 414: /* alias_opt ::= */ -{ yymsp[1].minor.yy209 = nil_token; } + case 421: /* alias_opt ::= */ +{ yymsp[1].minor.yy149 = nil_token; } break; - case 415: /* alias_opt ::= table_alias */ -{ yylhsminor.yy209 = yymsp[0].minor.yy209; } - yymsp[0].minor.yy209 = yylhsminor.yy209; + case 422: /* alias_opt ::= table_alias */ +{ yylhsminor.yy149 = yymsp[0].minor.yy149; } + yymsp[0].minor.yy149 = yylhsminor.yy149; break; - case 416: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy209 = yymsp[0].minor.yy209; } + case 423: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy149 = yymsp[0].minor.yy149; } break; - case 417: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 418: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==418); -{ yymsp[-2].minor.yy272 = yymsp[-1].minor.yy272; } + case 424: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 425: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==425); +{ yymsp[-2].minor.yy312 = yymsp[-1].minor.yy312; } break; - case 419: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy272 = createJoinTableNode(pCxt, yymsp[-4].minor.yy156, yymsp[-5].minor.yy272, yymsp[-2].minor.yy272, yymsp[0].minor.yy272); } - yymsp[-5].minor.yy272 = yylhsminor.yy272; + case 426: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy312 = createJoinTableNode(pCxt, yymsp[-4].minor.yy832, yymsp[-5].minor.yy312, yymsp[-2].minor.yy312, yymsp[0].minor.yy312); } + yymsp[-5].minor.yy312 = yylhsminor.yy312; break; - case 420: /* join_type ::= */ -{ yymsp[1].minor.yy156 = JOIN_TYPE_INNER; } + case 427: /* join_type ::= */ +{ yymsp[1].minor.yy832 = JOIN_TYPE_INNER; } break; - case 421: /* join_type ::= INNER */ -{ yymsp[0].minor.yy156 = JOIN_TYPE_INNER; } + case 428: /* join_type ::= INNER */ +{ yymsp[0].minor.yy832 = JOIN_TYPE_INNER; } break; - case 422: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 429: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy272 = createSelectStmt(pCxt, yymsp[-10].minor.yy293, yymsp[-9].minor.yy172, yymsp[-8].minor.yy272); - yymsp[-11].minor.yy272 = addWhereClause(pCxt, yymsp[-11].minor.yy272, yymsp[-7].minor.yy272); - yymsp[-11].minor.yy272 = addPartitionByClause(pCxt, yymsp[-11].minor.yy272, yymsp[-6].minor.yy172); - yymsp[-11].minor.yy272 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy272, yymsp[-2].minor.yy272); - yymsp[-11].minor.yy272 = addGroupByClause(pCxt, yymsp[-11].minor.yy272, yymsp[-1].minor.yy172); - yymsp[-11].minor.yy272 = addHavingClause(pCxt, yymsp[-11].minor.yy272, yymsp[0].minor.yy272); - yymsp[-11].minor.yy272 = addRangeClause(pCxt, yymsp[-11].minor.yy272, yymsp[-5].minor.yy272); - yymsp[-11].minor.yy272 = addEveryClause(pCxt, yymsp[-11].minor.yy272, yymsp[-4].minor.yy272); - yymsp[-11].minor.yy272 = addFillClause(pCxt, yymsp[-11].minor.yy272, yymsp[-3].minor.yy272); + yymsp[-11].minor.yy312 = createSelectStmt(pCxt, yymsp[-10].minor.yy497, yymsp[-9].minor.yy824, yymsp[-8].minor.yy312); + yymsp[-11].minor.yy312 = addWhereClause(pCxt, yymsp[-11].minor.yy312, yymsp[-7].minor.yy312); + yymsp[-11].minor.yy312 = addPartitionByClause(pCxt, yymsp[-11].minor.yy312, yymsp[-6].minor.yy824); + yymsp[-11].minor.yy312 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy312, yymsp[-2].minor.yy312); + yymsp[-11].minor.yy312 = addGroupByClause(pCxt, yymsp[-11].minor.yy312, yymsp[-1].minor.yy824); + yymsp[-11].minor.yy312 = addHavingClause(pCxt, yymsp[-11].minor.yy312, yymsp[0].minor.yy312); + yymsp[-11].minor.yy312 = addRangeClause(pCxt, yymsp[-11].minor.yy312, yymsp[-5].minor.yy312); + yymsp[-11].minor.yy312 = addEveryClause(pCxt, yymsp[-11].minor.yy312, yymsp[-4].minor.yy312); + yymsp[-11].minor.yy312 = addFillClause(pCxt, yymsp[-11].minor.yy312, yymsp[-3].minor.yy312); } break; - case 425: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy293 = false; } + case 432: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy497 = false; } break; - case 428: /* select_item ::= NK_STAR */ -{ yylhsminor.yy272 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy272 = yylhsminor.yy272; + case 435: /* select_item ::= NK_STAR */ +{ yylhsminor.yy312 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy312 = yylhsminor.yy312; break; - case 430: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy272 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy272), &yymsp[0].minor.yy209); } - yymsp[-1].minor.yy272 = yylhsminor.yy272; + case 437: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy312 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy312), &yymsp[0].minor.yy149); } + yymsp[-1].minor.yy312 = yylhsminor.yy312; break; - case 431: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy272 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), &yymsp[0].minor.yy209); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 438: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy312 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), &yymsp[0].minor.yy149); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 436: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 453: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==453); - case 469: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==469); -{ yymsp[-2].minor.yy172 = yymsp[0].minor.yy172; } + case 443: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 460: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==460); + case 476: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==476); +{ yymsp[-2].minor.yy824 = yymsp[0].minor.yy824; } break; - case 438: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy272 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), releaseRawExprNode(pCxt, yymsp[-1].minor.yy272)); } + case 445: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy312 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), releaseRawExprNode(pCxt, yymsp[-1].minor.yy312)); } break; - case 439: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ -{ yymsp[-3].minor.yy272 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy272)); } + case 446: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ +{ yymsp[-3].minor.yy312 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy312)); } break; - case 440: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy272 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), NULL, yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } + case 447: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy312 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), NULL, yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } break; - case 441: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy272 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy272), releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), yymsp[-1].minor.yy272, yymsp[0].minor.yy272); } + case 448: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy312 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy312), releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), yymsp[-1].minor.yy312, yymsp[0].minor.yy312); } break; - case 443: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - case 461: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==461); -{ yymsp[-3].minor.yy272 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy272); } + case 450: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + case 468: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==468); +{ yymsp[-3].minor.yy312 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy312); } break; - case 445: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy272 = createFillNode(pCxt, yymsp[-1].minor.yy186, NULL); } + case 452: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy312 = createFillNode(pCxt, yymsp[-1].minor.yy134, NULL); } break; - case 446: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy272 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy172)); } + case 453: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy312 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy824)); } break; - case 447: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy186 = FILL_MODE_NONE; } + case 454: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy134 = FILL_MODE_NONE; } break; - case 448: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy186 = FILL_MODE_PREV; } + case 455: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy134 = FILL_MODE_PREV; } break; - case 449: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy186 = FILL_MODE_NULL; } + case 456: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy134 = FILL_MODE_NULL; } break; - case 450: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy186 = FILL_MODE_LINEAR; } + case 457: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy134 = FILL_MODE_LINEAR; } break; - case 451: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy186 = FILL_MODE_NEXT; } + case 458: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy134 = FILL_MODE_NEXT; } break; - case 454: /* group_by_list ::= expression */ -{ yylhsminor.yy172 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); } - yymsp[0].minor.yy172 = yylhsminor.yy172; + case 461: /* group_by_list ::= expression */ +{ yylhsminor.yy824 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } + yymsp[0].minor.yy824 = yylhsminor.yy824; break; - case 455: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy172 = addNodeToList(pCxt, yymsp[-2].minor.yy172, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy272))); } - yymsp[-2].minor.yy172 = yylhsminor.yy172; + case 462: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy824 = addNodeToList(pCxt, yymsp[-2].minor.yy824, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy312))); } + yymsp[-2].minor.yy824 = yylhsminor.yy824; break; - case 459: /* range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ -{ yymsp[-5].minor.yy272 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy272), releaseRawExprNode(pCxt, yymsp[-1].minor.yy272)); } + case 466: /* range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ +{ yymsp[-5].minor.yy312 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy312), releaseRawExprNode(pCxt, yymsp[-1].minor.yy312)); } break; - case 462: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 469: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy272 = addOrderByClause(pCxt, yymsp[-3].minor.yy272, yymsp[-2].minor.yy172); - yylhsminor.yy272 = addSlimitClause(pCxt, yylhsminor.yy272, yymsp[-1].minor.yy272); - yylhsminor.yy272 = addLimitClause(pCxt, yylhsminor.yy272, yymsp[0].minor.yy272); + yylhsminor.yy312 = addOrderByClause(pCxt, yymsp[-3].minor.yy312, yymsp[-2].minor.yy824); + yylhsminor.yy312 = addSlimitClause(pCxt, yylhsminor.yy312, yymsp[-1].minor.yy312); + yylhsminor.yy312 = addLimitClause(pCxt, yylhsminor.yy312, yymsp[0].minor.yy312); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 464: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy272 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy272, yymsp[0].minor.yy272); } - yymsp[-3].minor.yy272 = yylhsminor.yy272; + case 471: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy312 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy312, yymsp[0].minor.yy312); } + yymsp[-3].minor.yy312 = yylhsminor.yy312; break; - case 465: /* query_expression_body ::= query_expression_body UNION query_expression_body */ -{ yylhsminor.yy272 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy272, yymsp[0].minor.yy272); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 472: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy312 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy312, yymsp[0].minor.yy312); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 467: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ + case 474: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ { - yymsp[-5].minor.yy272 = addOrderByClause(pCxt, yymsp[-4].minor.yy272, yymsp[-3].minor.yy172); - yymsp[-5].minor.yy272 = addSlimitClause(pCxt, yymsp[-5].minor.yy272, yymsp[-2].minor.yy272); - yymsp[-5].minor.yy272 = addLimitClause(pCxt, yymsp[-5].minor.yy272, yymsp[-1].minor.yy272); + yymsp[-5].minor.yy312 = addOrderByClause(pCxt, yymsp[-4].minor.yy312, yymsp[-3].minor.yy824); + yymsp[-5].minor.yy312 = addSlimitClause(pCxt, yymsp[-5].minor.yy312, yymsp[-2].minor.yy312); + yymsp[-5].minor.yy312 = addLimitClause(pCxt, yymsp[-5].minor.yy312, yymsp[-1].minor.yy312); } break; - case 471: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 475: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==475); -{ yymsp[-1].minor.yy272 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 478: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 482: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==482); +{ yymsp[-1].minor.yy312 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 472: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 476: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==476); -{ yymsp[-3].minor.yy272 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 479: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 483: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==483); +{ yymsp[-3].minor.yy312 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 473: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 477: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==477); -{ yymsp[-3].minor.yy272 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 480: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 484: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==484); +{ yymsp[-3].minor.yy312 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 478: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy272 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy272); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 485: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy312 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy312); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 482: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy272 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy272), yymsp[-1].minor.yy818, yymsp[0].minor.yy493); } - yymsp[-2].minor.yy272 = yylhsminor.yy272; + case 489: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy312 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy312), yymsp[-1].minor.yy158, yymsp[0].minor.yy417); } + yymsp[-2].minor.yy312 = yylhsminor.yy312; break; - case 483: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy818 = ORDER_ASC; } + case 490: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy158 = ORDER_ASC; } break; - case 484: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy818 = ORDER_ASC; } + case 491: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy158 = ORDER_ASC; } break; - case 485: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy818 = ORDER_DESC; } + case 492: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy158 = ORDER_DESC; } break; - case 486: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy493 = NULL_ORDER_DEFAULT; } + case 493: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy417 = NULL_ORDER_DEFAULT; } break; - case 487: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy493 = NULL_ORDER_FIRST; } + case 494: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy417 = NULL_ORDER_FIRST; } break; - case 488: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy493 = NULL_ORDER_LAST; } + case 495: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy417 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index cd7a9d549a1faab8994f71e7bf659c3a45f2cc01..7725674200c9e9c931f4640a1eecbed7e16b55f2 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -32,100 +32,59 @@ namespace { void generateInformationSchema(MockCatalogService* mcs) { - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODES, TSDB_SYSTEM_TABLE, 1) - .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MNODES, TSDB_SYSTEM_TABLE, 1) - .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MODULES, TSDB_SYSTEM_TABLE, 1) - .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_QNODES, TSDB_SYSTEM_TABLE, 1) - .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DATABASES, TSDB_SYSTEM_TABLE, 1) - .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_FUNCTIONS, TSDB_SYSTEM_TABLE, 1) - .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_INDEXES, TSDB_SYSTEM_TABLE, 3) - .addColumn("index_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) - .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES, TSDB_SYSTEM_TABLE, 2) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) - .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 2) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) - .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 1) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USERS, TSDB_SYSTEM_TABLE, 1) - .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS, TSDB_SYSTEM_TABLE, 1) - .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CONFIGS, TSDB_SYSTEM_TABLE, 1) - .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_CONFIG_OPTION_LEN); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODE_VARIABLES, TSDB_SYSTEM_TABLE, 1) - .addColumn("dnode_id", TSDB_DATA_TYPE_INT); - builder.done(); - } - { - ITableBuilder& builder = - mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CLUSTER, TSDB_SYSTEM_TABLE, 1) - .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN); - builder.done(); - } + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODES, TSDB_SYSTEM_TABLE, 1) + .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MNODES, TSDB_SYSTEM_TABLE, 1) + .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_MODULES, TSDB_SYSTEM_TABLE, 1) + .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_QNODES, TSDB_SYSTEM_TABLE, 1) + .addColumn("endpoint", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DATABASES, TSDB_SYSTEM_TABLE, 1) + .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_FUNCTIONS, TSDB_SYSTEM_TABLE, 1) + .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_FUNC_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_INDEXES, TSDB_SYSTEM_TABLE, 3) + .addColumn("index_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES, TSDB_SYSTEM_TABLE, 2) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .addColumn("stable_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLES, TSDB_SYSTEM_TABLE, 2) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .addColumn("table_name", TSDB_DATA_TYPE_BINARY, TSDB_TABLE_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_TABLE_DISTRIBUTED, TSDB_SYSTEM_TABLE, 1) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_USERS, TSDB_SYSTEM_TABLE, 1) + .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_USER_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VGROUPS, TSDB_SYSTEM_TABLE, 1) + .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CONFIGS, TSDB_SYSTEM_TABLE, 1) + .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_CONFIG_OPTION_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_DNODE_VARIABLES, TSDB_SYSTEM_TABLE, 1) + .addColumn("dnode_id", TSDB_DATA_TYPE_INT) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_CLUSTER, TSDB_SYSTEM_TABLE, 1) + .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_CLUSTER_ID_LEN) + .done(); + mcs->createTableBuilder(TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_VNODES, TSDB_SYSTEM_TABLE, 2) + .addColumn("dnode_id", TSDB_DATA_TYPE_INT) + .addColumn("dnode_ep", TSDB_DATA_TYPE_BINARY, TSDB_EP_LEN) + .done(); } void generatePerformanceSchema(MockCatalogService* mcs) { diff --git a/source/libs/parser/test/parAlterToBalanceTest.cpp b/source/libs/parser/test/parAlterToBalanceTest.cpp index 1caba6eab0384019b01f0c4957a6b9b9ffa1a5d1..3a08ef97564c426c834e26ed0569f77a29184d76 100644 --- a/source/libs/parser/test/parAlterToBalanceTest.cpp +++ b/source/libs/parser/test/parAlterToBalanceTest.cpp @@ -88,6 +88,7 @@ TEST_F(ParserInitialATest, alterDnode) { * | REPLICA int_value -- todo: enum 1, 3, default 1, unit replica * | STRICT {'off' | 'on'} -- todo: default 'off' * | WAL_LEVEL int_value -- enum 1, 2, default 1 + * | SST_TRIGGER int_value -- rang [1, 16], default 8 * } */ TEST_F(ParserInitialATest, alterDatabase) { @@ -112,6 +113,7 @@ TEST_F(ParserInitialATest, alterDatabase) { expect.cacheLast = -1; expect.cacheLastSize = -1; expect.replications = -1; + expect.sstTrigger = -1; }; auto setAlterDbBuffer = [&](int32_t buffer) { expect.buffer = buffer; }; auto setAlterDbPageSize = [&](int32_t pageSize) { expect.pageSize = pageSize; }; @@ -128,6 +130,7 @@ TEST_F(ParserInitialATest, alterDatabase) { auto setAlterDbStrict = [&](int8_t strict) { expect.strict = strict; }; auto setAlterDbCacheModel = [&](int8_t cacheModel) { expect.cacheLast = cacheModel; }; auto setAlterDbReplica = [&](int8_t replications) { expect.replications = replications; }; + auto setAlterDbSstTrigger = [&](int8_t sstTrigger) { expect.sstTrigger = sstTrigger; }; setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_DATABASE_STMT); @@ -146,6 +149,7 @@ TEST_F(ParserInitialATest, alterDatabase) { ASSERT_EQ(req.strict, expect.strict); ASSERT_EQ(req.cacheLast, expect.cacheLast); ASSERT_EQ(req.replications, expect.replications); + ASSERT_EQ(req.sstTrigger, expect.sstTrigger); }); const int32_t MINUTE_PER_DAY = MILLISECOND_PER_DAY / MILLISECOND_PER_MINUTE; @@ -157,7 +161,8 @@ TEST_F(ParserInitialATest, alterDatabase) { setAlterDbFsync(200); setAlterDbWal(1); setAlterDbCacheModel(TSDB_CACHE_MODEL_LAST_ROW); - run("ALTER DATABASE test CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 WAL_LEVEL 1"); + setAlterDbSstTrigger(16); + run("ALTER DATABASE test CACHEMODEL 'last_row' CACHESIZE 32 WAL_FSYNC_PERIOD 200 KEEP 10 WAL_LEVEL 1 STT_TRIGGER 16"); clearAlterDbReq(); initAlterDb("test"); @@ -231,6 +236,8 @@ TEST_F(ParserInitialATest, alterDatabaseSemanticCheck) { run("ALTER DATABASE test KEEP 1w", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 0", TSDB_CODE_PAR_INVALID_DB_OPTION); run("ALTER DATABASE test WAL_LEVEL 3", TSDB_CODE_PAR_INVALID_DB_OPTION); + run("ALTER DATABASE test STT_TRIGGER 0", TSDB_CODE_PAR_INVALID_DB_OPTION); + run("ALTER DATABASE test STT_TRIGGER 17", TSDB_CODE_PAR_INVALID_DB_OPTION); // Regardless of the specific sentence run("ALTER DATABASE db WAL_LEVEL 0 # td-14436", TSDB_CODE_PAR_SYNTAX_ERROR, PARSER_STAGE_PARSE); } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 68c4ac3706e2a42fd370a96a85d1adf6df162774..a159d797faed1fccf1194c54ecbe5506531ff0cc 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -111,10 +111,14 @@ TEST_F(ParserInitialCTest, createDatabase) { expect.numOfVgroups = TSDB_DEFAULT_VN_PER_DB; expect.numOfStables = TSDB_DEFAULT_DB_SINGLE_STABLE; expect.schemaless = TSDB_DEFAULT_DB_SCHEMALESS; - expect.walRetentionPeriod = TSDB_DEFAULT_DB_WAL_RETENTION_PERIOD; - expect.walRetentionSize = TSDB_DEFAULT_DB_WAL_RETENTION_SIZE; - expect.walRollPeriod = TSDB_DEFAULT_DB_WAL_ROLL_PERIOD; + expect.walRetentionPeriod = TSDB_REP_DEF_DB_WAL_RET_PERIOD; + expect.walRetentionSize = TSDB_REP_DEF_DB_WAL_RET_SIZE; + expect.walRollPeriod = TSDB_REP_DEF_DB_WAL_ROLL_PERIOD; expect.walSegmentSize = TSDB_DEFAULT_DB_WAL_SEGMENT_SIZE; + expect.sstTrigger = TSDB_DEFAULT_SST_TRIGGER; + expect.hashPrefix = TSDB_DEFAULT_HASH_PREFIX; + expect.hashSuffix = TSDB_DEFAULT_HASH_SUFFIX; + expect.tsdbPageSize = TSDB_DEFAULT_TSDB_PAGESIZE; }; auto setDbBufferFunc = [&](int32_t buffer) { expect.buffer = buffer; }; @@ -155,6 +159,10 @@ TEST_F(ParserInitialCTest, createDatabase) { auto setDbWalRetentionSize = [&](int32_t walRetentionSize) { expect.walRetentionSize = walRetentionSize; }; auto setDbWalRollPeriod = [&](int32_t walRollPeriod) { expect.walRollPeriod = walRollPeriod; }; auto setDbWalSegmentSize = [&](int32_t walSegmentSize) { expect.walSegmentSize = walSegmentSize; }; + auto setDbSstTrigger = [&](int32_t sstTrigger) { expect.sstTrigger = sstTrigger; }; + auto setDbHashPrefix = [&](int32_t hashPrefix) { expect.hashPrefix = hashPrefix; }; + auto setDbHashSuffix = [&](int32_t hashSuffix) { expect.hashSuffix = hashSuffix; }; + auto setDbTsdbPageSize = [&](int32_t tsdbPageSize) { expect.tsdbPageSize = tsdbPageSize; }; setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_DATABASE_STMT); @@ -185,7 +193,10 @@ TEST_F(ParserInitialCTest, createDatabase) { ASSERT_EQ(req.walRetentionSize, expect.walRetentionSize); ASSERT_EQ(req.walRollPeriod, expect.walRollPeriod); ASSERT_EQ(req.walSegmentSize, expect.walSegmentSize); - // ASSERT_EQ(req.schemaless, expect.schemaless); + ASSERT_EQ(req.sstTrigger, expect.sstTrigger); + ASSERT_EQ(req.hashPrefix, expect.hashPrefix); + ASSERT_EQ(req.hashSuffix, expect.hashSuffix); + ASSERT_EQ(req.tsdbPageSize, expect.tsdbPageSize); ASSERT_EQ(req.ignoreExist, expect.ignoreExist); ASSERT_EQ(req.numOfRetensions, expect.numOfRetensions); if (expect.numOfRetensions > 0) { @@ -233,6 +244,10 @@ TEST_F(ParserInitialCTest, createDatabase) { setDbWalRetentionSize(-1); setDbWalRollPeriod(10); setDbWalSegmentSize(20); + setDbSstTrigger(16); + setDbHashPrefix(3); + setDbHashSuffix(4); + setDbTsdbPageSize(32); run("CREATE DATABASE IF NOT EXISTS wxy_db " "BUFFER 64 " "CACHEMODEL 'last_value' " @@ -256,7 +271,11 @@ TEST_F(ParserInitialCTest, createDatabase) { "WAL_RETENTION_PERIOD -1 " "WAL_RETENTION_SIZE -1 " "WAL_ROLL_PERIOD 10 " - "WAL_SEGMENT_SIZE 20"); + "WAL_SEGMENT_SIZE 20 " + "STT_TRIGGER 16 " + "TABLE_PREFIX 3 " + "TABLE_SUFFIX 4 " + "TSDB_PAGESIZE 32"); clearCreateDbReq(); setCreateDbReqFunc("wxy_db", 1); @@ -266,6 +285,14 @@ TEST_F(ParserInitialCTest, createDatabase) { "DURATION 100m " "KEEP 1440m,300h,400d "); clearCreateDbReq(); + + setCreateDbReqFunc("wxy_db", 1); + setDbReplicaFunc(3); + setDbWalRetentionPeriod(TSDB_REPS_DEF_DB_WAL_RET_PERIOD); + setDbWalRetentionSize(TSDB_REPS_DEF_DB_WAL_RET_SIZE); + setDbWalRollPeriod(TSDB_REPS_DEF_DB_WAL_ROLL_PERIOD); + run("CREATE DATABASE IF NOT EXISTS wxy_db REPLICA 3"); + clearCreateDbReq(); } TEST_F(ParserInitialCTest, createDatabaseSemanticCheck) { diff --git a/source/libs/parser/test/parShowToUse.cpp b/source/libs/parser/test/parShowToUse.cpp index 6590378565849e8b39bab100a324823e2d665848..e33252c072fb1b34e5801098e27c2a51bef51c68 100644 --- a/source/libs/parser/test/parShowToUse.cpp +++ b/source/libs/parser/test/parShowToUse.cpp @@ -218,7 +218,13 @@ TEST_F(ParserShowToUseTest, showVgroups) { run("SHOW test.vgroups"); } -// todo SHOW vnodes +TEST_F(ParserShowToUseTest, showVnodes) { + useDb("root", "test"); + + run("SHOW VNODES 1"); + + run("SHOW VNODES 'node1:7030'"); +} TEST_F(ParserShowToUseTest, splitVgroup) { useDb("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index bf72f5210577d6f43f8ae97d098091b3020aeb16..fea2be55f962af0ff1d37b94ea4ddbe244eb3363 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -1372,9 +1372,21 @@ static int32_t createDeleteAggLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* pD } int32_t code = nodesListMakeStrictAppend(&pAgg->pAggFuncs, nodesCloneNode(pDelete->pCountFunc)); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pAgg->pAggFuncs, nodesCloneNode(pDelete->pFirstFunc)); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListStrictAppend(pAgg->pAggFuncs, nodesCloneNode(pDelete->pLastFunc)); + } if (TSDB_CODE_SUCCESS == code) { code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pCountFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pFirstFunc); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExpr(pAgg->pAggFuncs, &pDelete->pLastFunc); + } // set the output if (TSDB_CODE_SUCCESS == code) { code = createColumnByRewriteExprs(pAgg->pAggFuncs, &pAgg->node.pTargets); @@ -1405,7 +1417,9 @@ static int32_t createVnodeModifLogicNodeByDelete(SLogicPlanContext* pCxt, SDelet strcpy(pModify->tsColName, pRealTable->pMeta->schema->name); pModify->deleteTimeRange = pDelete->timeRange; pModify->pAffectedRows = nodesCloneNode(pDelete->pCountFunc); - if (NULL == pModify->pAffectedRows) { + pModify->pStartTs = nodesCloneNode(pDelete->pFirstFunc); + pModify->pEndTs = nodesCloneNode(pDelete->pLastFunc); + if (NULL == pModify->pAffectedRows || NULL == pModify->pStartTs || NULL == pModify->pEndTs) { nodesDestroyNode((SNode*)pModify); return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index cafae18dbe812546f6ef931d804ca4a9e5c1a6fa..b0177e61ed290e8ef687e5de8c54c824545d8103 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -1323,8 +1323,9 @@ static int32_t createSortPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren static int32_t createPartitionPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChildren, SPartitionLogicNode* pPartLogicNode, SPhysiNode** pPhyNode) { - SPartitionPhysiNode* pPart = - (SPartitionPhysiNode*)makePhysiNode(pCxt, (SLogicNode*)pPartLogicNode, QUERY_NODE_PHYSICAL_PLAN_PARTITION); + SPartitionPhysiNode* pPart = (SPartitionPhysiNode*)makePhysiNode( + pCxt, (SLogicNode*)pPartLogicNode, + pCxt->pPlanCxt->streamQuery ? QUERY_NODE_PHYSICAL_PLAN_STREAM_PARTITION : QUERY_NODE_PHYSICAL_PLAN_PARTITION); if (NULL == pPart) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -1669,6 +1670,12 @@ static int32_t createDataDeleter(SPhysiPlanContext* pCxt, SVnodeModifyLogicNode* int32_t code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pAffectedRows, &pDeleter->pAffectedRows); + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pStartTs, &pDeleter->pStartTs); + } + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pRoot->pOutputDataBlockDesc->dataBlockId, -1, pModify->pEndTs, &pDeleter->pEndTs); + } if (TSDB_CODE_SUCCESS == code) { pDeleter->sink.pInputDataBlockDesc = (SDataBlockDescNode*)nodesCloneNode((SNode*)pRoot->pOutputDataBlockDesc); if (NULL == pDeleter->sink.pInputDataBlockDesc) { diff --git a/source/libs/planner/src/planner.c b/source/libs/planner/src/planner.c index baa1d1074c7d4bea0df280649777db4a659247cb..35903d45b16175c1f9c21904b96ced434178ec51 100644 --- a/source/libs/planner/src/planner.c +++ b/source/libs/planner/src/planner.c @@ -123,6 +123,21 @@ int32_t qSubPlanToString(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { int32_t qStringToSubplan(const char* pStr, SSubplan** pSubplan) { return nodesStringToNode(pStr, (SNode**)pSubplan); } +int32_t qSubPlanToMsg(const SSubplan* pSubplan, char** pStr, int32_t* pLen) { + if (SUBPLAN_TYPE_MODIFY == pSubplan->subplanType && NULL == pSubplan->pNode) { + SDataInserterNode* insert = (SDataInserterNode*)pSubplan->pDataSink; + *pLen = insert->size; + *pStr = insert->pData; + insert->pData = NULL; + return TSDB_CODE_SUCCESS; + } + return nodesNodeToMsg((const SNode*)pSubplan, pStr, pLen); +} + +int32_t qMsgToSubplan(const char* pStr, int32_t len, SSubplan** pSubplan) { + return nodesMsgToNode(pStr, len, (SNode**)pSubplan); +} + char* qQueryPlanToString(const SQueryPlan* pPlan) { char* pStr = NULL; int32_t len = 0; diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 6c5b760564b1e73f09c92fc03d30a958c898c2fc..c2a0aee847f8ad1f6a03cf675c01fabdab3c4eff 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -35,6 +35,8 @@ TEST_F(PlanOptimizeTest, scanPath) { run("SELECT LAST(c1) FROM t1 WHERE ts BETWEEN '2022-7-29 11:10:10' AND '2022-7-30 11:10:10' INTERVAL(10S) " "FILL(LINEAR)"); + + run("SELECT COUNT(TBNAME) FROM t1"); } TEST_F(PlanOptimizeTest, pushDownCondition) { diff --git a/source/libs/planner/test/planTestUtil.cpp b/source/libs/planner/test/planTestUtil.cpp index 96f7d29230bafc94639be35fcc56550c029ffbac..bf19c7a2221ea36d565f9a2031a51a2a3b6dbfff 100644 --- a/source/libs/planner/test/planTestUtil.cpp +++ b/source/libs/planner/test/planTestUtil.cpp @@ -19,6 +19,7 @@ #include #include +#include #include "cmdnodes.h" #include "mockCatalogService.h" @@ -251,6 +252,7 @@ class PlannerTestBaseImpl { string splitLogicPlan_; string scaledLogicPlan_; string physiPlan_; + string physiPlanMsg_; vector physiSubplans_; }; @@ -274,6 +276,7 @@ class PlannerTestBaseImpl { res_.splitLogicPlan_.clear(); res_.scaledLogicPlan_.clear(); res_.physiPlan_.clear(); + res_.physiPlanMsg_.clear(); res_.physiSubplans_.clear(); } @@ -408,6 +411,8 @@ class PlannerTestBaseImpl { SNode* pSubplan; FOREACH(pSubplan, ((SNodeListNode*)pNode)->pNodeList) { res_.physiSubplans_.push_back(toString(pSubplan)); } } + res_.physiPlanMsg_ = toMsg((SNode*)(*pPlan)); + cout << "json len: " << res_.physiPlan_.length() << ", msg len: " << res_.physiPlanMsg_.length() << endl; } void setPlanContext(SQuery* pQuery, SPlanContext* pCxt) { @@ -446,12 +451,50 @@ class PlannerTestBaseImpl { string toString(const SNode* pRoot) { char* pStr = NULL; int32_t len = 0; + + auto start = chrono::steady_clock::now(); DO_WITH_THROW(nodesNodeToString, pRoot, false, &pStr, &len) + if (QUERY_NODE_PHYSICAL_PLAN == nodeType(pRoot)) { + cout << "nodesNodeToString: " + << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; + } + string str(pStr); taosMemoryFreeClear(pStr); return str; } + string toMsg(const SNode* pRoot) { + char* pStr = NULL; + int32_t len = 0; + + auto start = chrono::steady_clock::now(); + DO_WITH_THROW(nodesNodeToMsg, pRoot, &pStr, &len) + cout << "nodesNodeToMsg: " + << chrono::duration_cast(chrono::steady_clock::now() - start).count() << "us" << endl; + + SNode* pNode = NULL; + char* pNewStr = NULL; + int32_t newlen = 0; + DO_WITH_THROW(nodesMsgToNode, pStr, len, &pNode) + DO_WITH_THROW(nodesNodeToMsg, pNode, &pNewStr, &newlen) + if (newlen != len || 0 != memcmp(pStr, pNewStr, len)) { + cout << "nodesNodeToMsg error!!!!!!!!!!!!!! len = " << len << ", newlen = " << newlen << endl; + taosMemoryFreeClear(pNewStr); + DO_WITH_THROW(nodesNodeToString, pRoot, false, &pNewStr, &newlen) + cout << "orac node: " << pNewStr << endl; + taosMemoryFreeClear(pNewStr); + DO_WITH_THROW(nodesNodeToString, pNode, false, &pNewStr, &newlen) + cout << "new node: " << pNewStr << endl; + } + nodesDestroyNode(pNode); + taosMemoryFreeClear(pNewStr); + + string str(pStr, len); + taosMemoryFreeClear(pStr); + return str; + } + caseEnv caseEnv_; stmtEnv stmtEnv_; stmtRes res_; diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index d848016e46482614972d5e85469e4297136d6cc0..8162b922cec8fd779907df851bc1d8b545cafc28 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -134,8 +134,7 @@ int32_t taosAsyncExec(__async_exec_fn_t execFn, void* execParam, int32_t* code) schedMsg.thandle = execParam; schedMsg.msg = code; - taosScheduleTask(&pTaskQueue, &schedMsg); - return 0; + return taosScheduleTask(&pTaskQueue, &schedMsg); } void destroySendMsgInfo(SMsgSendInfo* pMsgBody) { @@ -472,5 +471,3 @@ int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { return TSDB_CODE_SUCCESS; } - - diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index e2d3ac1583926da6fe9d9aff82392c4fcc3c2d65..e54937114cd1bc0e011e8dc9d8ed44a710bc1807 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -38,6 +38,8 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { pOut->dbVgroup->vgVersion = usedbRsp->vgVersion; pOut->dbVgroup->hashMethod = usedbRsp->hashMethod; + pOut->dbVgroup->hashPrefix = usedbRsp->hashPrefix; + pOut->dbVgroup->hashSuffix = usedbRsp->hashSuffix; qDebug("Got %d vgroup for db %s", usedbRsp->vgNum, usedbRsp->db); diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index a5eab730869fb7fcd37f82d3a3006f5f4b1b2017..f174a163c90df817211cd40eaba7d798af977bbf 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -572,7 +572,7 @@ int32_t qwProcessQuery(QW_FPARAMS_DEF, SQWMsg *qwMsg, char *sql) { // QW_TASK_DLOGL("subplan json string, len:%d, %s", qwMsg->msgLen, qwMsg->msg); - code = qStringToSubplan(qwMsg->msg, &plan); + code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan); if (TSDB_CODE_SUCCESS != code) { code = TSDB_CODE_INVALID_MSG; QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); @@ -969,7 +969,7 @@ int32_t qwProcessDelete(QW_FPARAMS_DEF, SQWMsg *qwMsg, SDeleteRes *pRes) { DataSinkHandle sinkHandle = NULL; SQWTaskCtx ctx = {0}; - code = qStringToSubplan(qwMsg->msg, &plan); + code = qMsgToSubplan(qwMsg->msg, qwMsg->msgLen, &plan); if (TSDB_CODE_SUCCESS != code) { code = TSDB_CODE_INVALID_MSG; QW_TASK_ELOG("task physical plan to subplan failed, code:%x - %s", code, tstrerror(code)); diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index e00cda399b5b2de5887b6394d05b960d295a0827..14776fc92754a03068d695a246b8f40128981c21 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -285,8 +285,9 @@ typedef struct SSchJob { } SSchJob; typedef struct SSchTaskCtx { - int64_t jobRid; + int64_t jobRid; SSchTask *pTask; + bool asyncLaunch; } SSchTaskCtx; extern SSchedulerMgmt schMgmt; diff --git a/source/libs/scheduler/src/schRemote.c b/source/libs/scheduler/src/schRemote.c index b2b824c33d3af08f632aac3cb83ca2c2a9e3bff7..4b15629407b793f78b8995633ad4422e3d9a5ffc 100644 --- a/source/libs/scheduler/src/schRemote.c +++ b/source/libs/scheduler/src/schRemote.c @@ -414,7 +414,7 @@ int32_t schHandleCallback(void *param, SDataBuf *pMsg, int32_t rspCode) { tstrerror(rspCode)); SCH_ERR_JRET(schProcessOnCbBegin(&pJob, &pTask, pParam->queryId, pParam->refId, pParam->taskId)); - + code = schHandleResponseMsg(pJob, pTask, pParam->execId, pMsg, rspCode); pMsg->pData = NULL; diff --git a/source/libs/scheduler/src/schTask.c b/source/libs/scheduler/src/schTask.c index 05c24288c4af1498715b40e42b23671d0c0ae291..70da8d4a16ad13d51926642430019d0f9b8134d4 100644 --- a/source/libs/scheduler/src/schTask.c +++ b/source/libs/scheduler/src/schTask.c @@ -53,7 +53,7 @@ void schInitTaskRetryTimes(SSchJob *pJob, SSchTask *pTask, SSchLevel *pLevel) { int32_t nodeNum = taosArrayGetSize(pJob->nodeList); pTask->maxRetryTimes = TMAX(nodeNum, SCH_DEFAULT_MAX_RETRY_NUM); } - + pTask->maxExecTimes = pTask->maxRetryTimes * (pLevel->level + 1); } @@ -143,14 +143,10 @@ int32_t schUpdateTaskExecNode(SSchJob *pJob, SSchTask *pTask, void *handle, int3 return TSDB_CODE_SUCCESS; } - if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it - SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); - return TSDB_CODE_SUCCESS; - } - SSchNodeInfo *nodeInfo = taosHashGet(pTask->execNodes, &execId, sizeof(execId)); if (NULL == nodeInfo) { // ignore it - SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + SCH_TASK_DLOG("handle not updated since execId %d already not exist, current execId %d, waitRetry %d", execId, + pTask->execId, pTask->waitRetry); return TSDB_CODE_SUCCESS; } @@ -165,11 +161,16 @@ int32_t schUpdateTaskHandle(SSchJob *pJob, SSchTask *pTask, bool dropExecNode, v if (dropExecNode) { SCH_RET(schDropTaskExecNode(pJob, pTask, handle, execId)); } + + schUpdateTaskExecNode(pJob, pTask, handle, execId); + if ((execId != pTask->execId) || pTask->waitRetry) { // ignore it + SCH_TASK_DLOG("handle not updated since execId %d is already not current execId %d, waitRetry %d", execId, pTask->execId, pTask->waitRetry); + SCH_ERR_RET(TSDB_CODE_SCH_IGNORE_ERROR); + } + SCH_SET_TASK_HANDLE(pTask, handle); - schUpdateTaskExecNode(pJob, pTask, handle, execId); - return TSDB_CODE_SUCCESS; } @@ -319,7 +320,7 @@ int32_t schRescheduleTask(SSchJob *pJob, SSchTask *pTask) { if (!schMgmt.cfg.enableReSchedule) { return TSDB_CODE_SUCCESS; } - + if (SCH_IS_DATA_BIND_TASK(pTask)) { return TSDB_CODE_SUCCESS; } @@ -346,7 +347,8 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 } if (((pTask->execId + 1) >= pTask->maxExecTimes) || ((pTask->retryTimes + 1) > pTask->maxRetryTimes)) { - SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, pTask->maxExecTimes, pTask->execId); + SCH_TASK_DLOG("task no more retry since reach max times %d:%d, execId %d", pTask->maxRetryTimes, + pTask->maxExecTimes, pTask->execId); schHandleJobFailure(pJob, rspCode); return TSDB_CODE_SUCCESS; } @@ -354,7 +356,7 @@ int32_t schDoTaskRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 pTask->waitRetry = true; schDropTaskOnExecNode(pJob, pTask); taosHashClear(pTask->execNodes); - SCH_ERR_JRET(schRemoveTaskFromExecList(pJob, pTask)); + schRemoveTaskFromExecList(pJob, pTask); schDeregisterTaskHb(pJob, pTask); atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); taosMemoryFreeClear(pTask->msg); @@ -432,12 +434,14 @@ int32_t schHandleRedirect(SSchJob *pJob, SSchTask *pTask, SDataBuf *pData, int32 code = schDoTaskRedirect(pJob, pTask, pData, rspCode); taosMemoryFree(pData->pData); + taosMemoryFree(pData->pEpSet); SCH_RET(code); _return: taosMemoryFree(pData->pData); + taosMemoryFree(pData->pEpSet); SCH_RET(schProcessOnTaskFailure(pJob, pTask, code)); } @@ -553,7 +557,8 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo if ((pTask->retryTimes + 1) > pTask->maxRetryTimes) { *needRetry = false; - SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes, pTask->maxRetryTimes); + SCH_TASK_DLOG("task no more retry since reach max retry times, retryTimes:%d/%d", pTask->retryTimes, + pTask->maxRetryTimes); return TSDB_CODE_SUCCESS; } @@ -569,25 +574,25 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo return TSDB_CODE_SUCCESS; } -/* - if (SCH_IS_DATA_BIND_TASK(pTask)) { - if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { - *needRetry = false; - SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId, - SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); - return TSDB_CODE_SUCCESS; - } - } else { - int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); + /* + if (SCH_IS_DATA_BIND_TASK(pTask)) { + if ((pTask->execId + 1) >= SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since all ep tried, execId:%d, epNum:%d", pTask->execId, + SCH_TASK_NUM_OF_EPS(&pTask->plan->execNode)); + return TSDB_CODE_SUCCESS; + } + } else { + int32_t candidateNum = taosArrayGetSize(pTask->candidateAddrs); - if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) { - *needRetry = false; - SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", - pTask->candidateIdx, candidateNum); - return TSDB_CODE_SUCCESS; + if ((pTask->candidateIdx + 1) >= candidateNum && (TSDB_CODE_SCH_TIMEOUT_ERROR != errCode)) { + *needRetry = false; + SCH_TASK_DLOG("task no more retry since all candiates tried, candidateIdx:%d, candidateNum:%d", + pTask->candidateIdx, candidateNum); + return TSDB_CODE_SUCCESS; + } } - } -*/ + */ *needRetry = true; SCH_TASK_DLOG("task need the %dth retry, errCode:%x - %s", pTask->execId + 1, errCode, tstrerror(errCode)); @@ -598,7 +603,7 @@ int32_t schTaskCheckSetRetry(SSchJob *pJob, SSchTask *pTask, int32_t errCode, bo int32_t schHandleTaskRetry(SSchJob *pJob, SSchTask *pTask) { atomic_sub_fetch_32(&pTask->level->taskLaunchedNum, 1); - SCH_ERR_RET(schRemoveTaskFromExecList(pJob, pTask)); + schRemoveTaskFromExecList(pJob, pTask); SCH_SET_TASK_STATUS(pTask, JOB_TASK_STATUS_INIT); if (SCH_TASK_NEED_FLOW_CTRL(pJob, pTask)) { @@ -635,8 +640,9 @@ int32_t schSetAddrsFromNodeList(SSchJob *pJob, SSchTask *pTask) { SCH_ERR_RET(TSDB_CODE_QRY_OUT_OF_MEMORY); } - SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, naddr->epSet.inUse, naddr->epSet.numOfEps, - SCH_GET_CUR_EP(naddr)->fqdn, SCH_GET_CUR_EP(naddr)->port); + SCH_TASK_TLOG("set %dth candidate addr, id %d, inUse:%d/%d, fqdn:%s, port:%d", i, naddr->nodeId, + naddr->epSet.inUse, naddr->epSet.numOfEps, SCH_GET_CUR_EP(naddr)->fqdn, + SCH_GET_CUR_EP(naddr)->port); ++addNum; } @@ -716,10 +722,10 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { if (candidateNum <= 1) { goto _return; } - + switch (schMgmt.cfg.schPolicy) { case SCH_LOAD_SEQ: - case SCH_ALL: + case SCH_ALL: default: if (++pTask->candidateIdx >= candidateNum) { pTask->candidateIdx = 0; @@ -737,15 +743,14 @@ int32_t schSwitchTaskCandidateAddr(SSchJob *pJob, SSchTask *pTask) { _return: SCH_TASK_DLOG("switch task candiateIdx to %d/%d", pTask->candidateIdx, candidateNum); - + return TSDB_CODE_SUCCESS; } int32_t schRemoveTaskFromExecList(SSchJob *pJob, SSchTask *pTask) { int32_t code = taosHashRemove(pJob->execTasks, &pTask->taskId, sizeof(pTask->taskId)); if (code) { - SCH_TASK_ELOG("task failed to rm from execTask list, code:%x", code); - SCH_ERR_RET(TSDB_CODE_SCH_INTERNAL_ERROR); + SCH_TASK_WLOG("task already not in execTask list, code:%x", code); } return TSDB_CODE_SUCCESS; @@ -764,7 +769,7 @@ void schDropTaskOnExecNode(SSchJob *pJob, SSchTask *pTask) { return; } - int32_t i = 0; + int32_t i = 0; SSchNodeInfo *nodeInfo = taosHashIterate(pTask->execNodes, NULL); while (nodeInfo) { if (nodeInfo->handle) { @@ -876,7 +881,7 @@ int32_t schLaunchRemoteTask(SSchJob *pJob, SSchTask *pTask) { int32_t code = 0; if (NULL == pTask->msg) { // TODO add more detailed reason for failure - code = qSubPlanToString(plan, &pTask->msg, &pTask->msgLen); + code = qSubPlanToMsg(plan, &pTask->msg, &pTask->msgLen); if (TSDB_CODE_SUCCESS != code) { SCH_TASK_ELOG("failed to create physical plan, code:%s, msg:%p, len:%d", tstrerror(code), pTask->msg, pTask->msgLen); @@ -925,16 +930,21 @@ int32_t schLaunchLocalTask(SSchJob *pJob, SSchTask *pTask) { int32_t schLaunchTaskImpl(void *param) { SSchTaskCtx *pCtx = (SSchTaskCtx *)param; - SSchJob *pJob = schAcquireJob(pCtx->jobRid); + SSchJob *pJob = schAcquireJob(pCtx->jobRid); if (NULL == pJob) { - taosMemoryFree(param); qDebug("job refId 0x%" PRIx64 " already not exist", pCtx->jobRid); + taosMemoryFree(param); SCH_RET(TSDB_CODE_SCH_JOB_IS_DROPPING); } - + SSchTask *pTask = pCtx->pTask; - int8_t status = 0; - int32_t code = 0; + + if (pCtx->asyncLaunch) { + SCH_LOCK_TASK(pTask); + } + + int8_t status = 0; + int32_t code = 0; atomic_add_fetch_32(&pTask->level->taskLaunchedNum, 1); pTask->execId++; @@ -965,8 +975,6 @@ int32_t schLaunchTaskImpl(void *param) { _return: - taosMemoryFree(param); - if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { if (code) { code = schProcessOnTaskFailure(pJob, pTask, code); @@ -976,26 +984,33 @@ _return: } } + if (pCtx->asyncLaunch) { + SCH_UNLOCK_TASK(pTask); + } + schReleaseJob(pJob->refId); + taosMemoryFree(param); + SCH_RET(code); } -int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { +int32_t schAsyncLaunchTaskImpl(SSchJob *pJob, SSchTask *pTask) { SSchTaskCtx *param = taosMemoryCalloc(1, sizeof(SSchTaskCtx)); if (NULL == param) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - + param->jobRid = pJob->refId; param->pTask = pTask; if (pJob->taskNum >= SCH_MIN_AYSNC_EXEC_NUM) { + param->asyncLaunch = true; taosAsyncExec(schLaunchTaskImpl, param, NULL); } else { SCH_ERR_RET(schLaunchTaskImpl(param)); } - + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 9d4010f60e5fcb222e235181a2ce12b8d4dc4102..7cdb7c0db95cd582fad03174d0fa6927cb1fd668 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -243,6 +243,39 @@ FAIL: return 0; } +int32_t streamSearchAndAddBlock(SStreamTask* pTask, SStreamDispatchReq* pReqs, SSDataBlock* pDataBlock, int32_t vgSz, + int64_t groupId) { + char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, groupId); + SArray* vgInfo = pTask->shuffleDispatcher.dbInfo.pVgroupInfos; + + /*uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName));*/ + SUseDbRsp* pDbInfo = &pTask->shuffleDispatcher.dbInfo; + uint32_t hashValue = + taosGetTbHashVal(ctbName, strlen(ctbName), pDbInfo->hashMethod, pDbInfo->hashPrefix, pDbInfo->hashSuffix); + taosMemoryFree(ctbName); + + bool found = false; + // TODO: optimize search + int32_t j; + for (j = 0; j < vgSz; j++) { + SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); + ASSERT(pVgInfo->vgId > 0); + if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { + return -1; + } + if (pReqs[j].blockNum == 0) { + atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); + } + pReqs[j].blockNum++; + found = true; + break; + } + } + ASSERT(found); + return 0; +} + int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pData) { int32_t code = -1; int32_t blockNum = taosArrayGetSize(pData->blocks); @@ -317,20 +350,10 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat for (int32_t i = 0; i < blockNum; i++) { SSDataBlock* pDataBlock = taosArrayGet(pData->blocks, i); - char* ctbName = buildCtbNameByGroupId(pTask->shuffleDispatcher.stbFullName, pDataBlock->info.groupId); - - // TODO: get hash function by hashMethod - uint32_t hashValue = MurmurHash3_32(ctbName, strlen(ctbName)); - - taosMemoryFree(ctbName); - bool found = false; - // TODO: optimize search - int32_t j; - for (j = 0; j < vgSz; j++) { - SVgroupInfo* pVgInfo = taosArrayGet(vgInfo, j); - ASSERT(pVgInfo->vgId > 0); - if (hashValue >= pVgInfo->hashBegin && hashValue <= pVgInfo->hashEnd) { + // TODO: do not use broadcast + if (pDataBlock->info.type == STREAM_DELETE_RESULT) { + for (int32_t j = 0; j < vgSz; j++) { if (streamAddBlockToDispatchMsg(pDataBlock, &pReqs[j]) < 0) { goto FAIL_SHUFFLE_DISPATCH; } @@ -338,11 +361,13 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat atomic_add_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); } pReqs[j].blockNum++; - found = true; - break; } + continue; + } + + if (streamSearchAndAddBlock(pTask, pReqs, pDataBlock, vgSz, pDataBlock->info.groupId) < 0) { + goto FAIL_SHUFFLE_DISPATCH; } - ASSERT(found); } for (int32_t i = 0; i < vgSz; i++) { diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index dfd6f012cc4f64d252f75a20f761c6f87fc05b78..6cd5132bb9d9021d5c0fe590224414430d36af40 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -35,6 +35,10 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask) { goto _err; } + if (tdbTbOpen("func.state.db", sizeof(STupleKey), -1, STupleKeyCmpr, pState->db, &pState->pFuncStateDb) < 0) { + goto _err; + } + if (streamStateBegin(pState) < 0) { goto _err; } @@ -44,8 +48,9 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask) { return pState; _err: - if (pState->pStateDb) tdbTbClose(pState->pStateDb); - if (pState->db) tdbClose(pState->db); + tdbTbClose(pState->pStateDb); + tdbTbClose(pState->pFuncStateDb); + tdbClose(pState->db); taosMemoryFree(pState); return NULL; } @@ -53,6 +58,7 @@ _err: void streamStateClose(SStreamState* pState) { tdbCommit(pState->db, &pState->txn); tdbTbClose(pState->pStateDb); + tdbTbClose(pState->pFuncStateDb); tdbClose(pState->db); taosMemoryFree(pState); @@ -101,6 +107,17 @@ int32_t streamStateAbort(SStreamState* pState) { return 0; } +int32_t streamStateFuncPut(SStreamState* pState, const STupleKey* key, const void* value, int32_t vLen) { + return tdbTbUpsert(pState->pFuncStateDb, key, sizeof(STupleKey), value, vLen, &pState->txn); +} +int32_t streamStateFuncGet(SStreamState* pState, const STupleKey* key, void** pVal, int32_t* pVLen) { + return tdbTbGet(pState->pFuncStateDb, key, sizeof(STupleKey), pVal, pVLen); +} + +int32_t streamStateFuncDel(SStreamState* pState, const STupleKey* key) { + return tdbTbDelete(pState->pFuncStateDb, key, sizeof(STupleKey), &pState->txn); +} + int32_t streamStatePut(SStreamState* pState, const SWinKey* key, const void* value, int32_t vLen) { return tdbTbUpsert(pState->pStateDb, key, sizeof(SWinKey), value, vLen, &pState->txn); } @@ -112,6 +129,29 @@ int32_t streamStateDel(SStreamState* pState, const SWinKey* key) { return tdbTbDelete(pState->pStateDb, key, sizeof(SWinKey), &pState->txn); } +int32_t streamStateAddIfNotExist(SStreamState* pState, const SWinKey* key, void** pVal, int32_t* pVLen) { + // todo refactor + int32_t size = *pVLen; + if (streamStateGet(pState, key, pVal, pVLen) == 0) { + return 0; + } + void* tmp = taosMemoryCalloc(1, size); + if (streamStatePut(pState, key, &tmp, size) == 0) { + taosMemoryFree(tmp); + int32_t code = streamStateGet(pState, key, pVal, pVLen); + ASSERT(code == 0); + return code; + } + taosMemoryFree(tmp); + return -1; +} + +int32_t streamStateReleaseBuf(SStreamState* pState, const SWinKey* key, void* pVal) { + // todo refactor + streamFreeVal(pVal); + return 0; +} + SStreamStateCur* streamStateGetCur(SStreamState* pState, const SWinKey* key) { SStreamStateCur* pCur = taosMemoryCalloc(1, sizeof(SStreamStateCur)); if (pCur == NULL) return NULL; diff --git a/source/libs/stream/src/streamUpdate.c b/source/libs/stream/src/streamUpdate.c index d053662bd30287d5d9589a3881c8588fd3eb82ec..332f7ad2fd7be60f532b1394eb2d72adf985b82a 100644 --- a/source/libs/stream/src/streamUpdate.c +++ b/source/libs/stream/src/streamUpdate.c @@ -170,8 +170,17 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { if (ts < maxTs - pInfo->watermark) { // this window has been closed. if (pInfo->pCloseWinSBF) { - return tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY)); + res = tScalableBfPut(pInfo->pCloseWinSBF, &ts, sizeof(TSKEY)); + if (res == TSDB_CODE_SUCCESS) { + return false; + } else { + qDebug("===stream===Update close window sbf. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, + maxTs, *pMapMaxTs, ts); + return true; + } } + qDebug("===stream===Update close window. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, + maxTs, *pMapMaxTs, ts); return true; } @@ -193,7 +202,7 @@ bool updateInfoIsUpdated(SUpdateInfo *pInfo, uint64_t tableId, TSKEY ts) { } if (ts < pInfo->minTS) { - qDebug("===stream===Update. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, + qDebug("===stream===Update min ts. tableId:%" PRIu64 ", maxTs:%" PRIu64 ", mapMaxTs:%" PRIu64 ", ts:%" PRIu64, tableId, maxTs, *pMapMaxTs, ts); return true; } else if (res == TSDB_CODE_SUCCESS) { diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 51098374b03531142c9c12443fa5b02efddc3aca..6f29b54f806f1113ec69dface7bdcbb4b0c42afc 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -2766,8 +2766,6 @@ const char* syncStr(ESyncState state) { } int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry) { - SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); - if (ths->state != TAOS_SYNC_STATE_FOLLOWER) { syncNodeEventLog(ths, "I am not follower, can not do leader transfer"); return 0; @@ -2799,6 +2797,8 @@ int32_t syncDoLeaderTransfer(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* p } */ + SyncLeaderTransfer* pSyncLeaderTransfer = syncLeaderTransferFromRpcMsg2(pRpcMsg); + do { char logBuf[128]; snprintf(logBuf, sizeof(logBuf), "do leader transfer, index:%ld", pEntry->index); diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index b42aba560fa1c26ef9426b55729c1d39cafa8a24..faebe5bbecb16012831ed103de520c14accc81d5 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -1992,6 +1992,313 @@ void syncAppendEntriesReplyLog2(char* s, const SyncAppendEntriesReply* pMsg) { } } +// ---- message process SyncHeartbeat---- +SyncHeartbeat* syncHeartbeatBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncHeartbeat); + SyncHeartbeat* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_HEARTBEAT; + return pMsg; +} + +void syncHeartbeatDestroy(SyncHeartbeat* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncHeartbeatSerialize(const SyncHeartbeat* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncHeartbeatDeserialize(const char* buf, uint32_t len, SyncHeartbeat* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncHeartbeatSerialize2(const SyncHeartbeat* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncHeartbeatSerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncHeartbeat* syncHeartbeatDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncHeartbeat* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncHeartbeatDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncHeartbeat2RpcMsg(const SyncHeartbeat* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncHeartbeatSerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncHeartbeatFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeat* pMsg) { + syncHeartbeatDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncHeartbeat* syncHeartbeatFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncHeartbeat* pMsg = syncHeartbeatDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncHeartbeat2Json(const SyncHeartbeat* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->commitIndex); + cJSON_AddStringToObject(pRoot, "commitIndex", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncHeartbeat", pRoot); + return pJson; +} + +char* syncHeartbeat2Str(const SyncHeartbeat* pMsg) { + cJSON* pJson = syncHeartbeat2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncHeartbeatPrint(const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + printf("syncHeartbeatPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatPrint2(char* s, const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + printf("syncHeartbeatPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatLog(const SyncHeartbeat* pMsg) { + char* serialized = syncHeartbeat2Str(pMsg); + sTrace("syncHeartbeatLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncHeartbeatLog2(char* s, const SyncHeartbeat* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncHeartbeat2Str(pMsg); + sTrace("syncHeartbeatLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + +// ---- message process SyncHeartbeatReply---- +SyncHeartbeatReply* syncHeartbeatReplyBuild(int32_t vgId) { + uint32_t bytes = sizeof(SyncHeartbeatReply); + SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes); + memset(pMsg, 0, bytes); + pMsg->bytes = bytes; + pMsg->vgId = vgId; + pMsg->msgType = TDMT_SYNC_HEARTBEAT_REPLY; + return pMsg; +} + +void syncHeartbeatReplyDestroy(SyncHeartbeatReply* pMsg) { + if (pMsg != NULL) { + taosMemoryFree(pMsg); + } +} + +void syncHeartbeatReplySerialize(const SyncHeartbeatReply* pMsg, char* buf, uint32_t bufLen) { + ASSERT(pMsg->bytes <= bufLen); + memcpy(buf, pMsg, pMsg->bytes); +} + +void syncHeartbeatReplyDeserialize(const char* buf, uint32_t len, SyncHeartbeatReply* pMsg) { + memcpy(pMsg, buf, len); + ASSERT(len == pMsg->bytes); +} + +char* syncHeartbeatReplySerialize2(const SyncHeartbeatReply* pMsg, uint32_t* len) { + char* buf = taosMemoryMalloc(pMsg->bytes); + ASSERT(buf != NULL); + syncHeartbeatReplySerialize(pMsg, buf, pMsg->bytes); + if (len != NULL) { + *len = pMsg->bytes; + } + return buf; +} + +SyncHeartbeatReply* syncHeartbeatReplyDeserialize2(const char* buf, uint32_t len) { + uint32_t bytes = *((uint32_t*)buf); + SyncHeartbeatReply* pMsg = taosMemoryMalloc(bytes); + ASSERT(pMsg != NULL); + syncHeartbeatReplyDeserialize(buf, len, pMsg); + ASSERT(len == pMsg->bytes); + return pMsg; +} + +void syncHeartbeatReply2RpcMsg(const SyncHeartbeatReply* pMsg, SRpcMsg* pRpcMsg) { + memset(pRpcMsg, 0, sizeof(*pRpcMsg)); + pRpcMsg->msgType = pMsg->msgType; + pRpcMsg->contLen = pMsg->bytes; + pRpcMsg->pCont = rpcMallocCont(pRpcMsg->contLen); + syncHeartbeatReplySerialize(pMsg, pRpcMsg->pCont, pRpcMsg->contLen); +} + +void syncHeartbeatReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncHeartbeatReply* pMsg) { + syncHeartbeatReplyDeserialize(pRpcMsg->pCont, pRpcMsg->contLen, pMsg); +} + +SyncHeartbeatReply* syncHeartbeatReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { + SyncHeartbeatReply* pMsg = syncHeartbeatReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); + ASSERT(pMsg != NULL); + return pMsg; +} + +cJSON* syncHeartbeatReply2Json(const SyncHeartbeatReply* pMsg) { + char u64buf[128] = {0}; + cJSON* pRoot = cJSON_CreateObject(); + + if (pMsg != NULL) { + cJSON_AddNumberToObject(pRoot, "bytes", pMsg->bytes); + cJSON_AddNumberToObject(pRoot, "vgId", pMsg->vgId); + cJSON_AddNumberToObject(pRoot, "msgType", pMsg->msgType); + + cJSON* pSrcId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->srcId.addr); + cJSON_AddStringToObject(pSrcId, "addr", u64buf); + { + uint64_t u64 = pMsg->srcId.addr; + cJSON* pTmp = pSrcId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pSrcId, "vgId", pMsg->srcId.vgId); + cJSON_AddItemToObject(pRoot, "srcId", pSrcId); + + cJSON* pDestId = cJSON_CreateObject(); + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->destId.addr); + cJSON_AddStringToObject(pDestId, "addr", u64buf); + { + uint64_t u64 = pMsg->destId.addr; + cJSON* pTmp = pDestId; + char host[128] = {0}; + uint16_t port; + syncUtilU642Addr(u64, host, sizeof(host), &port); + cJSON_AddStringToObject(pTmp, "addr_host", host); + cJSON_AddNumberToObject(pTmp, "addr_port", port); + } + cJSON_AddNumberToObject(pDestId, "vgId", pMsg->destId.vgId); + cJSON_AddItemToObject(pRoot, "destId", pDestId); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->privateTerm); + cJSON_AddStringToObject(pRoot, "privateTerm", u64buf); + + snprintf(u64buf, sizeof(u64buf), "%" PRIu64, pMsg->term); + cJSON_AddStringToObject(pRoot, "term", u64buf); + + cJSON_AddStringToObject(pRoot, "matchIndex", u64buf); + snprintf(u64buf, sizeof(u64buf), "%" PRId64, pMsg->startTime); + cJSON_AddStringToObject(pRoot, "startTime", u64buf); + } + + cJSON* pJson = cJSON_CreateObject(); + cJSON_AddItemToObject(pJson, "SyncHeartbeatReply", pRoot); + return pJson; +} + +char* syncHeartbeatReply2Str(const SyncHeartbeatReply* pMsg) { + cJSON* pJson = syncHeartbeatReply2Json(pMsg); + char* serialized = cJSON_Print(pJson); + cJSON_Delete(pJson); + return serialized; +} + +void syncHeartbeatReplyPrint(const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + printf("syncHeartbeatReplyPrint | len:%" PRIu64 " | %s \n", strlen(serialized), serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyPrint2(char* s, const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + printf("syncHeartbeatReplyPrint2 | len:%" PRIu64 " | %s | %s \n", strlen(serialized), s, serialized); + fflush(NULL); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyLog(const SyncHeartbeatReply* pMsg) { + char* serialized = syncHeartbeatReply2Str(pMsg); + sTrace("syncHeartbeatReplyLog | len:%" PRIu64 " | %s", strlen(serialized), serialized); + taosMemoryFree(serialized); +} + +void syncHeartbeatReplyLog2(char* s, const SyncHeartbeatReply* pMsg) { + if (gRaftDetailLog) { + char* serialized = syncHeartbeatReply2Str(pMsg); + sTrace("syncHeartbeatReplyLog2 | len:%" PRIu64 " | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); + } +} + // ---- message process SyncApplyMsg---- SyncApplyMsg* syncApplyMsgBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SyncApplyMsg) + dataLen; diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 72845d0c1d1a9378a3a189f4037c6fc646c8a536..b9cc7a391dde35e2569f30000752b3ef175fc824 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -57,6 +57,8 @@ add_executable(syncLeaderTransferTest "") add_executable(syncReconfigFinishTest "") add_executable(syncRestoreFromSnapshot "") add_executable(syncRaftCfgIndexTest "") +add_executable(syncHeartbeatTest "") +add_executable(syncHeartbeatReplyTest "") target_sources(syncTest @@ -295,6 +297,14 @@ target_sources(syncRaftCfgIndexTest PRIVATE "syncRaftCfgIndexTest.cpp" ) +target_sources(syncHeartbeatTest + PRIVATE + "syncHeartbeatTest.cpp" +) +target_sources(syncHeartbeatReplyTest + PRIVATE + "syncHeartbeatReplyTest.cpp" +) target_include_directories(syncTest @@ -592,6 +602,16 @@ target_include_directories(syncRaftCfgIndexTest "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncHeartbeatTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) +target_include_directories(syncHeartbeatReplyTest + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_link_libraries(syncTest @@ -830,6 +850,14 @@ target_link_libraries(syncRaftCfgIndexTest sync gtest_main ) +target_link_libraries(syncHeartbeatTest + sync + gtest_main +) +target_link_libraries(syncHeartbeatReplyTest + sync + gtest_main +) enable_testing() diff --git a/source/libs/sync/test/syncHeartbeatReplyTest.cpp b/source/libs/sync/test/syncHeartbeatReplyTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..0ccd7b70bb8f552fa14c0a615993abbab678a8a0 --- /dev/null +++ b/source/libs/sync/test/syncHeartbeatReplyTest.cpp @@ -0,0 +1,105 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncHeartbeatReply *createMsg() { + SyncHeartbeatReply *pMsg = syncHeartbeatReplyBuild(1000); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + + pMsg->term = 33; + pMsg->privateTerm = 44; + pMsg->startTime = taosGetTimestampMs(); + return pMsg; +} + +void test1() { + SyncHeartbeatReply *pMsg = createMsg(); + syncHeartbeatReplyLog2((char *)"test1:", pMsg); + syncHeartbeatReplyDestroy(pMsg); +} + +void test2() { + SyncHeartbeatReply *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncHeartbeatReplySerialize(pMsg, serialized, len); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000); + syncHeartbeatReplyDeserialize(serialized, len, pMsg2); + syncHeartbeatReplyLog2((char *)"test2: syncHeartbeatReplySerialize -> syncHeartbeatReplyDeserialize ", + pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test3() { + SyncHeartbeatReply *pMsg = createMsg(); + uint32_t len; + char * serialized = syncHeartbeatReplySerialize2(pMsg, &len); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyDeserialize2(serialized, len); + syncHeartbeatReplyLog2((char *)"test3: syncHeartbeatReplySerialize3 -> syncHeartbeatReplyDeserialize2 ", + pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test4() { + SyncHeartbeatReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyBuild(1000); + syncHeartbeatReplyFromRpcMsg(&rpcMsg, pMsg2); + syncHeartbeatReplyLog2((char *)"test4: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg ", + pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +void test5() { + SyncHeartbeatReply *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeatReply2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeatReply *pMsg2 = syncHeartbeatReplyFromRpcMsg2(&rpcMsg); + syncHeartbeatReplyLog2((char *)"test5: syncHeartbeatReply2RpcMsg -> syncHeartbeatReplyFromRpcMsg2 ", + pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatReplyDestroy(pMsg); + syncHeartbeatReplyDestroy(pMsg2); +} + +int main() { + gRaftDetailLog = true; + + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/sync/test/syncHeartbeatTest.cpp b/source/libs/sync/test/syncHeartbeatTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..d910c828f1e13f4e6484c7b8ad7ea6758f626750 --- /dev/null +++ b/source/libs/sync/test/syncHeartbeatTest.cpp @@ -0,0 +1,99 @@ +#include +#include +#include "syncIO.h" +#include "syncInt.h" +#include "syncMessage.h" +#include "syncUtil.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +SyncHeartbeat *createMsg() { + SyncHeartbeat *pMsg = syncHeartbeatBuild(789); + pMsg->srcId.addr = syncUtilAddr2U64("127.0.0.1", 1234); + pMsg->srcId.vgId = 100; + pMsg->destId.addr = syncUtilAddr2U64("127.0.0.1", 5678); + pMsg->destId.vgId = 100; + pMsg->term = 8; + pMsg->commitIndex = 33; + pMsg->privateTerm = 44; + return pMsg; +} + +void test1() { + SyncHeartbeat *pMsg = createMsg(); + syncHeartbeatLog2((char *)"test1:", pMsg); + syncHeartbeatDestroy(pMsg); +} + +void test2() { + SyncHeartbeat *pMsg = createMsg(); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); + syncHeartbeatSerialize(pMsg, serialized, len); + SyncHeartbeat *pMsg2 = syncHeartbeatBuild(789); + syncHeartbeatDeserialize(serialized, len, pMsg2); + syncHeartbeatLog2((char *)"test2: syncHeartbeatSerialize -> syncHeartbeatDeserialize ", pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test3() { + SyncHeartbeat *pMsg = createMsg(); + uint32_t len; + char * serialized = syncHeartbeatSerialize2(pMsg, &len); + SyncHeartbeat *pMsg2 = syncHeartbeatDeserialize2(serialized, len); + syncHeartbeatLog2((char *)"test3: syncHeartbeatSerialize2 -> syncHeartbeatDeserialize2 ", pMsg2); + + taosMemoryFree(serialized); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test4() { + SyncHeartbeat *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeat2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeat *pMsg2 = (SyncHeartbeat *)taosMemoryMalloc(rpcMsg.contLen); + syncHeartbeatFromRpcMsg(&rpcMsg, pMsg2); + syncHeartbeatLog2((char *)"test4: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +void test5() { + SyncHeartbeat *pMsg = createMsg(); + SRpcMsg rpcMsg; + syncHeartbeat2RpcMsg(pMsg, &rpcMsg); + SyncHeartbeat *pMsg2 =syncHeartbeatFromRpcMsg2(&rpcMsg); + syncHeartbeatLog2((char *)"test5: syncHeartbeat2RpcMsg -> syncHeartbeatFromRpcMsg2 ", pMsg2); + + rpcFreeCont(rpcMsg.pCont); + syncHeartbeatDestroy(pMsg); + syncHeartbeatDestroy(pMsg2); +} + +int main() { + tsAsyncLog = 0; + sDebugFlag = DEBUG_TRACE + DEBUG_SCREEN + DEBUG_FILE; + gRaftDetailLog = true; + logTest(); + + test1(); + test2(); + test3(); + test4(); + test5(); + + return 0; +} diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 1480920f908e34bb7be5b95affe64619ac042289..c6ecd37680c642feb8e961fc4b398dd3f01d0645 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -489,7 +489,7 @@ static int tdbBtreeBalanceDeeper(SBTree *pBt, SPage *pRoot, SPage **ppChild, TXN } // Copy the root page content to the child page - tdbPageCopy(pRoot, pChild); + tdbPageCopy(pRoot, pChild, 0); // Reinitialize the root page zArg.flags = TDB_BTREE_ROOT; @@ -742,7 +742,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx for (int i = 0; i < nOlds; i++) { tdbPageCreate(pOlds[0]->pageSize, &pOldsCopy[i], tdbDefaultMalloc, NULL); tdbBtreeInitPage(pOldsCopy[i], &iarg, 0); - tdbPageCopy(pOlds[i], pOldsCopy[i]); + tdbPageCopy(pOlds[i], pOldsCopy[i], 0); } iNew = 0; nNewCells = 0; @@ -840,7 +840,7 @@ static int tdbBtreeBalanceNonRoot(SBTree *pBt, SPage *pParent, int idx, TXN *pTx i8 flags = TDB_BTREE_ROOT | TDB_BTREE_PAGE_IS_LEAF(pNews[0]); // copy content to the parent page tdbBtreeInitPage(pParent, &(SBtreeInitPageArg){.flags = flags, .pBt = pBt}, 0); - tdbPageCopy(pNews[0], pParent); + tdbPageCopy(pNews[0], pParent, 1); } for (int i = 0; i < 3; i++) { diff --git a/source/libs/tdb/src/db/tdbPage.c b/source/libs/tdb/src/db/tdbPage.c index a3f376b929291780bdd57cbf99f5db6035e70aff..1e2eefabf42bd97644baf38e19b77595529aab38 100644 --- a/source/libs/tdb/src/db/tdbPage.c +++ b/source/libs/tdb/src/db/tdbPage.c @@ -229,7 +229,7 @@ int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt) { return 0; } -void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { +void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int deepCopyOvfl) { int delta, nFree; pToPage->pFreeStart = pToPage->pPageHdr + (pFromPage->pFreeStart - pFromPage->pPageHdr); @@ -250,8 +250,15 @@ void tdbPageCopy(SPage *pFromPage, SPage *pToPage) { // Copy the overflow cells for (int iOvfl = 0; iOvfl < pFromPage->nOverflow; iOvfl++) { + SCell *pNewCell = pFromPage->apOvfl[iOvfl]; + if (deepCopyOvfl) { + int szCell = (*pFromPage->xCellSize)(pFromPage, pFromPage->apOvfl[iOvfl], 0, NULL, NULL); + pNewCell = (SCell *)tdbOsMalloc(szCell); + memcpy(pNewCell, pFromPage->apOvfl[iOvfl], szCell); + } + + pToPage->apOvfl[iOvfl] = pNewCell; pToPage->aiOvfl[iOvfl] = pFromPage->aiOvfl[iOvfl]; - pToPage->apOvfl[iOvfl] = pFromPage->apOvfl[iOvfl]; } pToPage->nOverflow = pFromPage->nOverflow; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index f90c39278888c7838b0c4b1b5b434e4c06fb30a0..2cc62d3d6af466921ef10b9fc5871499d4797467 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -34,6 +34,22 @@ static int tdbPagerInitPage(SPager *pPager, SPage *pPage, int (*initPage)(SPage static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage); static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage); +static FORCE_INLINE int32_t pageCmpFn(const void *lhs, const void *rhs) { + SPage *pPageL = (SPage *)(((uint8_t *)lhs) - sizeof(SRBTreeNode)); + SPage *pPageR = (SPage *)(((uint8_t *)rhs) - sizeof(SRBTreeNode)); + + SPgno pgnoL = TDB_PAGE_PGNO(pPageL); + SPgno pgnoR = TDB_PAGE_PGNO(pPageR); + + if (pgnoL < pgnoR) { + return -1; + } else if (pgnoL > pgnoR) { + return 1; + } else { + return 0; + } +} + int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { uint8_t *pPtr; SPager *pPager; @@ -83,6 +99,8 @@ int tdbPagerOpen(SPCache *pCache, const char *fileName, SPager **ppPager) { ret = tdbGetFileSize(pPager->fd, pPager->pageSize, &(pPager->dbOrigSize)); pPager->dbFileSize = pPager->dbOrigSize; + tRBTreeCreate(&pPager->rbt, pageCmpFn); + *ppPager = pPager; return 0; } @@ -170,7 +188,7 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { // Set page as dirty pPage->isDirty = 1; - + /* // Add page to dirty list(TODO: NOT use O(n^2) algorithm) for (ppPage = &pPager->pDirty; (*ppPage) && TDB_PAGE_PGNO(*ppPage) < TDB_PAGE_PGNO(pPage); ppPage = &((*ppPage)->pDirtyNext)) { @@ -185,6 +203,8 @@ int tdbPagerWrite(SPager *pPager, SPage *pPage) { ASSERT(*ppPage == NULL || TDB_PAGE_PGNO(*ppPage) > TDB_PAGE_PGNO(pPage)); pPage->pDirtyNext = *ppPage; *ppPage = pPage; + */ + tRBTreePut(&pPager->rbt, (SRBTreeNode *)pPage); // Write page to journal if neccessary if (TDB_PAGE_PGNO(pPage) <= pPager->dbOrigSize) { @@ -228,6 +248,24 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { return 0; } + SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); + SRBTreeNode *pNode = NULL; + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; + ret = tdbPagerWritePageToDB(pPager, pPage); + if (ret < 0) { + ASSERT(0); + return -1; + } + + pPage->isDirty = 0; + + // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); + tdbPCacheRelease(pPager->pCache, pPage, pTxn); + } + + tRBTreeCreate(&pPager->rbt, pageCmpFn); + /* // loop to write the dirty pages to file for (pPage = pPager->pDirty; pPage; pPage = pPage->pDirtyNext) { // TODO: update the page footer @@ -238,9 +276,6 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { } } - tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize); - pPager->dbOrigSize = pPager->dbFileSize; - // release the page for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) { pPager->pDirty = pPage->pDirtyNext; @@ -250,6 +285,9 @@ int tdbPagerCommit(SPager *pPager, TXN *pTxn) { tdbPCacheRelease(pPager->pCache, pPage, pTxn); } + */ + tdbTrace("tdbttl commit:%p, %d", pPager, pPager->dbOrigSize); + pPager->dbOrigSize = pPager->dbFileSize; // sync the db file tdbOsFSync(pPager->fd); @@ -308,15 +346,19 @@ int tdbPagerAbort(SPager *pPager, TXN *pTxn) { } */ // 3, release the dirty pages - for (pPage = pPager->pDirty; pPage; pPage = pPager->pDirty) { - pPager->pDirty = pPage->pDirtyNext; - pPage->pDirtyNext = NULL; + SRBTreeIter iter = tRBTreeIterCreate(&pPager->rbt, 1); + SRBTreeNode *pNode = NULL; + while ((pNode = tRBTreeIterNext(&iter)) != NULL) { + pPage = (SPage *)pNode; pPage->isDirty = 0; + // tRBTreeDrop(&pPager->rbt, (SRBTreeNode *)pPage); tdbPCacheRelease(pPager->pCache, pPage, pTxn); } + tRBTreeCreate(&pPager->rbt, pageCmpFn); + // 4, remove the journal file tdbOsClose(pPager->jfd); tdbOsRemove(pPager->jFileName); @@ -497,12 +539,19 @@ static int tdbPagerWritePageToJournal(SPager *pPager, SPage *pPage) { return 0; } - +/* +struct TdFile { + TdThreadRwlock rwlock; + int refId; + int fd; + FILE *fp; +} TdFile; +*/ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { i64 offset; int ret; - offset = pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); + offset = (i64)pPage->pageSize * (TDB_PAGE_PGNO(pPage) - 1); if (tdbOsLSeek(pPager->fd, offset, SEEK_SET) < 0) { ASSERT(0); return -1; @@ -514,6 +563,7 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { return -1; } + // pwrite(pPager->fd->fd, pPage->pData, pPage->pageSize, offset); return 0; } diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 6a694cf8f1f8cce95e7fa9373e2aa2c01128a6d9..29a9665c156a4514846c5549e7a90d97a8df4984 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -19,6 +19,7 @@ #include "tdb.h" #include "tlog.h" +#include "trbtree.h" #ifdef __cplusplus extern "C" { @@ -256,6 +257,7 @@ typedef struct { #pragma pack(pop) struct SPage { + SRBTreeNode node; // must be the first field for pageCmpFn to work tdb_spinlock_t lock; int pageSize; u8 *pData; @@ -280,13 +282,13 @@ struct SPage { static inline i32 tdbRefPage(SPage *pPage) { i32 nRef = atomic_add_fetch_32(&((pPage)->nRef), 1); - tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef); + // tdbTrace("ref page %p/%d, nRef %d", pPage, pPage->id, nRef); return nRef; } static inline i32 tdbUnrefPage(SPage *pPage) { i32 nRef = atomic_sub_fetch_32(&((pPage)->nRef), 1); - tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef); + // tdbTrace("unref page %p/%d, nRef %d", pPage, pPage->id, nRef); return nRef; } @@ -331,7 +333,7 @@ void tdbPageInit(SPage *pPage, u8 szAmHdr, int (*xCellSize)(const SPage *, SCell int tdbPageInsertCell(SPage *pPage, int idx, SCell *pCell, int szCell, u8 asOvfl); int tdbPageDropCell(SPage *pPage, int idx, TXN *pTxn, SBTree *pBt); int tdbPageUpdateCell(SPage *pPage, int idx, SCell *pCell, int szCell, TXN *pTxn, SBTree *pBt); -void tdbPageCopy(SPage *pFromPage, SPage *pToPage); +void tdbPageCopy(SPage *pFromPage, SPage *pToPage, int copyOvflCells); int tdbPageCapacity(int pageSize, int amHdrSize); static inline SCell *tdbPageGetCell(SPage *pPage, int idx) { @@ -389,6 +391,7 @@ struct SPager { SPgno dbFileSize; SPgno dbOrigSize; SPage *pDirty; + SRBTree rbt; u8 inTran; SPager *pNext; // used by TDB SPager *pHashNext; // used by TDB diff --git a/source/libs/tfs/src/tfs.c b/source/libs/tfs/src/tfs.c index 62aec219df57f1a8edd11456ca631e315d553803..4600e5e568fc5644024c633e08328a54247b28b5 100644 --- a/source/libs/tfs/src/tfs.c +++ b/source/libs/tfs/src/tfs.c @@ -113,6 +113,8 @@ SDiskSize tfsGetSize(STfs *pTfs) { return size; } +int32_t tfsGetLevel(STfs *pTfs) { return pTfs->nlevel; } + int32_t tfsAllocDisk(STfs *pTfs, int32_t expLevel, SDiskID *pDiskId) { pDiskId->level = expLevel; pDiskId->id = -1; diff --git a/source/libs/transport/src/thttp.c b/source/libs/transport/src/thttp.c index 275e7b42cc10ed2a77934da84ed544a1874e0b28..1559c85e23d59fec376890433f924522df8dc761 100644 --- a/source/libs/transport/src/thttp.c +++ b/source/libs/transport/src/thttp.c @@ -126,22 +126,22 @@ _OVER: return code; } -static void destroyHttpClient(SHttpClient* cli) { +static FORCE_INLINE void destroyHttpClient(SHttpClient* cli) { taosMemoryFree(cli->wbuf); taosMemoryFree(cli->rbuf); taosMemoryFree(cli->addr); taosMemoryFree(cli); } -static void clientCloseCb(uv_handle_t* handle) { +static FORCE_INLINE void clientCloseCb(uv_handle_t* handle) { SHttpClient* cli = handle->data; destroyHttpClient(cli); } -static void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { +static FORCE_INLINE void clientAllocBuffCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf) { SHttpClient* cli = handle->data; buf->base = cli->rbuf; buf->len = HTTP_RECV_BUF_SIZE; } -static void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { +static FORCE_INLINE void clientRecvCb(uv_stream_t* handle, ssize_t nread, const uv_buf_t* buf) { SHttpClient* cli = handle->data; if (nread < 0) { uError("http-report recv error:%s", uv_err_name(nread)); @@ -173,7 +173,7 @@ static void clientConnCb(uv_connect_t* req, int32_t status) { uv_write(&cli->req, (uv_stream_t*)&cli->tcp, cli->wbuf, 2, clientSentCb); } -static int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { +static FORCE_INLINE int32_t taosBuildDstAddr(const char* server, uint16_t port, struct sockaddr_in* dest) { uint32_t ip = taosGetIpv4FromFqdn(server); if (ip == 0xffffffff) { terrno = TAOS_SYSTEM_ERROR(errno); diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 41688c733079f12fbd04683183dd80db3b65606d..dcfa93431b40209e56b0f0f04d3111ece4c893f5 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -69,11 +69,9 @@ typedef struct SCliThrd { SAsyncPool* asyncPool; uv_prepare_t* prepare; void* pool; // conn pool - + // timer handles SArray* timerList; - // msg queue - queue msg; TdThreadMutex msgMtx; SDelayQueue* delayQueue; @@ -108,7 +106,7 @@ static void cliReadTimeoutCb(uv_timer_t* handle); // register timer in each thread to clear expire conn // static void cliTimeoutCb(uv_timer_t* handle); // alloc buffer for recv -static void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); +static FORCE_INLINE void cliAllocRecvBufferCb(uv_handle_t* handle, size_t suggested_size, uv_buf_t* buf); // callback after recv nbytes from socket static void cliRecvCb(uv_stream_t* cli, ssize_t nread, const uv_buf_t* buf); // callback after send data to socket @@ -132,10 +130,10 @@ static void cliSend(SCliConn* pConn); static void cliDestroyConnMsgs(SCliConn* conn, bool destroy); // cli util func -static bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx); -static void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); +static FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx); +static FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr); -static int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); +static FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* resp); // process data read from server, add decompress etc later static void cliHandleResp(SCliConn* conn); @@ -150,13 +148,11 @@ static void cliHandleUpdate(SCliMsg* pMsg, SCliThrd* pThrd); static void (*cliAsyncHandle[])(SCliMsg* pMsg, SCliThrd* pThrd) = {cliHandleReq, cliHandleQuit, cliHandleRelease, NULL, cliHandleUpdate}; -static void cliSendQuit(SCliThrd* thrd); -static void destroyUserdata(STransMsg* userdata); +static FORCE_INLINE void destroyUserdata(STransMsg* userdata); +static FORCE_INLINE void destroyCmsg(void* cmsg); +static FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst); +static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx); -static int cliRBChoseIdx(STrans* pTransInst); - -static void destroyCmsg(void* cmsg); -static void transDestroyConnCtx(STransConnCtx* ctx); // thread obj static SCliThrd* createThrdObj(); static void destroyThrdObj(SCliThrd* pThrd); @@ -166,7 +162,7 @@ static void cliWalkCb(uv_handle_t* handle, void* arg); static void cliReleaseUnfinishedMsg(SCliConn* conn) { for (int i = 0; i < transQueueSize(&conn->cliMsgs); i++) { SCliMsg* msg = transQueueGet(&conn->cliMsgs, i); - if (msg != NULL && msg->ctx != NULL) { + if (msg != NULL && msg->ctx != NULL && msg->ctx->ahandle != (void*)0x9527) { if (conn->ctx.freeFunc != NULL && msg->ctx->ahandle != NULL) { conn->ctx.freeFunc(msg->ctx->ahandle); } @@ -200,22 +196,22 @@ static void cliReleaseUnfinishedMsg(SCliConn* conn) { #define CONN_GET_HOST_THREAD(conn) (conn ? ((SCliConn*)conn)->hostThrd : NULL) #define CONN_GET_INST_LABEL(conn) (((STrans*)(((SCliThrd*)(conn)->hostThrd)->pTransInst))->label) -#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ - do { \ - int i = 0, sz = transQueueSize(&conn->cliMsgs); \ - for (; i < sz; i++) { \ - pMsg = transQueueGet(&conn->cliMsgs, i); \ - if (pMsg != NULL && pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \ - break; \ - } \ - } \ - if (i == sz) { \ - pMsg = NULL; \ - tDebug("msg not found, %" PRIu64 "", ahandle); \ - } else { \ - pMsg = transQueueRm(&conn->cliMsgs, i); \ - tDebug("msg found, %" PRIu64 "", ahandle); \ - } \ +#define CONN_GET_MSGCTX_BY_AHANDLE(conn, ahandle) \ + do { \ + int i = 0, sz = transQueueSize(&conn->cliMsgs); \ + for (; i < sz; i++) { \ + pMsg = transQueueGet(&conn->cliMsgs, i); \ + if (pMsg->ctx != NULL && (uint64_t)pMsg->ctx->ahandle == ahandle) { \ + break; \ + } \ + } \ + if (i == sz) { \ + pMsg = NULL; \ + tDebug("msg not found, %" PRIu64 "", ahandle); \ + } else { \ + pMsg = transQueueRm(&conn->cliMsgs, i); \ + tDebug("msg found, %" PRIu64 "", ahandle); \ + } \ } while (0) #define CONN_GET_NEXT_SENDMSG(conn) \ do { \ @@ -293,7 +289,12 @@ bool cliMaySendCachedMsg(SCliConn* conn) { if (!transQueueEmpty(&conn->cliMsgs)) { SCliMsg* pCliMsg = NULL; CONN_GET_NEXT_SENDMSG(conn); - cliSend(conn); + if (pCliMsg == NULL) + return false; + else { + cliSend(conn); + return true; + } } return false; _RETURN: @@ -373,15 +374,19 @@ void cliHandleResp(SCliConn* conn) { if (pCtx == NULL && CONN_NO_PERSIST_BY_APP(conn)) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); + transFreeMsg(transMsg.pCont); return; } if (CONN_RELEASE_BY_SERVER(conn) && transMsg.info.ahandle == NULL) { tDebug("%s except, conn %p read while cli ignore it", CONN_GET_INST_LABEL(conn), conn); + transFreeMsg(transMsg.pCont); return; } - if (cliAppCb(conn, &transMsg, pMsg) != 0) { - return; + if (pMsg == NULL || (pMsg && pMsg->type != Release)) { + if (cliAppCb(conn, &transMsg, pMsg) != 0) { + return; + } } destroyCmsg(pMsg); @@ -390,7 +395,7 @@ void cliHandleResp(SCliConn* conn) { } if (CONN_NO_PERSIST_BY_APP(conn)) { - addConnToPool(pThrd->pool, conn); + return addConnToPool(pThrd->pool, conn); } uv_read_start((uv_stream_t*)conn->stream, cliAllocRecvBufferCb, cliRecvCb); @@ -429,17 +434,20 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { transMsg.info.ahandle); } } else { - transMsg.info.ahandle = pCtx ? pCtx->ahandle : NULL; + transMsg.info.ahandle = (pMsg->type != Release && pCtx) ? pCtx->ahandle : NULL; } if (pCtx == NULL || pCtx->pSem == NULL) { if (transMsg.info.ahandle == NULL) { + if (REQUEST_NO_RESP(&pMsg->msg) || pMsg->type == Release) destroyCmsg(pMsg); once = true; continue; } } - if (cliAppCb(pConn, &transMsg, pMsg) != 0) { - return; + if (pMsg == NULL || (pMsg && pMsg->type != Release)) { + if (cliAppCb(pConn, &transMsg, pMsg) != 0) { + return; + } } destroyCmsg(pMsg); tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); @@ -705,6 +713,9 @@ static bool cliHandleNoResp(SCliConn* conn) { if (cliMaySendCachedMsg(conn) == false) { SCliThrd* thrd = conn->hostThrd; addConnToPool(thrd->pool, conn); + res = false; + } else { + res = true; } } } @@ -782,7 +793,13 @@ void cliSend(SCliConn* pConn) { uv_buf_t wb = uv_buf_init((char*)pHead, msgLen); uv_write_t* req = transReqQueuePush(&pConn->wreqQueue); - uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); + + int status = uv_write(req, (uv_stream_t*)pConn->stream, &wb, 1, cliSendCb); + if (status != 0) { + tGError("%s conn %p failed to sent msg:%s, errmsg:%s", CONN_GET_INST_LABEL(pConn), pConn, TMSG_INFO(pMsg->msgType), + uv_err_name(status)); + cliHandleExcept(pConn); + } return; _RETURN: return; @@ -885,26 +902,23 @@ SCliConn* cliGetConn(SCliMsg* pMsg, SCliThrd* pThrd, bool* ignore) { } return conn; } -void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { +FORCE_INLINE void cliMayCvtFqdnToIp(SEpSet* pEpSet, SCvtAddr* pCvtAddr) { if (pCvtAddr->cvt == false) { return; } - for (int i = 0; i < pEpSet->numOfEps && pEpSet->numOfEps == 1; i++) { - if (strncmp(pEpSet->eps[i].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) { - memset(pEpSet->eps[i].fqdn, 0, TSDB_FQDN_LEN); - memcpy(pEpSet->eps[i].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN); - } + if (pEpSet->numOfEps == 1 && strncmp(pEpSet->eps[0].fqdn, pCvtAddr->fqdn, TSDB_FQDN_LEN) == 0) { + memset(pEpSet->eps[0].fqdn, 0, TSDB_FQDN_LEN); + memcpy(pEpSet->eps[0].fqdn, pCvtAddr->ip, TSDB_FQDN_LEN); } } -bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { +FORCE_INLINE bool cliIsEpsetUpdated(int32_t code, STransConnCtx* pCtx) { if (code != 0) return false; if (pCtx->retryCnt == 0) return false; if (transEpSetIsEqual(&pCtx->epSet, &pCtx->origEpSet)) return false; return true; } - -int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) { +FORCE_INLINE int32_t cliBuildExceptResp(SCliMsg* pMsg, STransMsg* pResp) { if (pMsg == NULL) return -1; memset(pResp, 0, sizeof(STransMsg)); @@ -934,7 +948,9 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { // persist conn already release by server STransMsg resp; cliBuildExceptResp(pMsg, &resp); - pTransInst->cfp(pTransInst->parent, &resp, NULL); + if (pMsg->type != Release) { + pTransInst->cfp(pTransInst->parent, &resp, NULL); + } destroyCmsg(pMsg); return; } @@ -980,6 +996,8 @@ void cliHandleReq(SCliMsg* pMsg, SCliThrd* pThrd) { return; } } + STraceId* trace = &pMsg->msg.info.traceId; + tGTrace("%s conn %p ready", pTransInst->label, conn); } static void cliAsyncCb(uv_async_t* handle) { SAsyncItem* item = handle->data; @@ -1128,14 +1146,15 @@ void* transInitClient(uint32_t ip, uint32_t port, char* label, int numOfThreads, return cli; } -static void destroyUserdata(STransMsg* userdata) { +static FORCE_INLINE void destroyUserdata(STransMsg* userdata) { if (userdata->pCont == NULL) { return; } transFreeMsg(userdata->pCont); userdata->pCont = NULL; } -static void destroyCmsg(void* arg) { + +static FORCE_INLINE void destroyCmsg(void* arg) { SCliMsg* pMsg = arg; if (pMsg == NULL) { return; @@ -1201,7 +1220,7 @@ static void destroyThrdObj(SCliThrd* pThrd) { taosMemoryFree(pThrd); } -static void transDestroyConnCtx(STransConnCtx* ctx) { +static FORCE_INLINE void transDestroyConnCtx(STransConnCtx* ctx) { // taosMemoryFree(ctx); } @@ -1220,7 +1239,7 @@ void cliWalkCb(uv_handle_t* handle, void* arg) { } } -int cliRBChoseIdx(STrans* pTransInst) { +FORCE_INLINE int cliRBChoseIdx(STrans* pTransInst) { int8_t index = pTransInst->index; if (pTransInst->numOfThreads == 0) { return -1; @@ -1230,7 +1249,7 @@ int cliRBChoseIdx(STrans* pTransInst) { } return index % pTransInst->numOfThreads; } -static void doDelayTask(void* param) { +static FORCE_INLINE void doDelayTask(void* param) { STaskArg* arg = param; SCliMsg* pMsg = arg->param1; SCliThrd* pThrd = arg->param2; @@ -1264,13 +1283,13 @@ static void cliSchedMsgToNextNode(SCliMsg* pMsg, SCliThrd* pThrd) { transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); } -void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { +FORCE_INLINE void cliCompareAndSwap(int8_t* val, int8_t exp, int8_t newVal) { if (*val != exp) { *val = newVal; } } -bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { +FORCE_INLINE bool cliTryExtractEpSet(STransMsg* pResp, SEpSet* dst) { if ((pResp == NULL || pResp->info.hasEpSet == 0)) { return false; } @@ -1300,15 +1319,11 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { STrans* pTransInst = pThrd->pTransInst; if (pMsg == NULL || pMsg->ctx == NULL) { - tTrace("%s conn %p handle resp", pTransInst->label, pConn); + tDebug("%s conn %p handle resp", pTransInst->label, pConn); pTransInst->cfp(pTransInst->parent, pResp, NULL); return 0; } - /* - * no retry - * 1. query conn - * 2. rpc thread already receive quit msg - */ + STransConnCtx* pCtx = pMsg->ctx; int32_t code = pResp->code; @@ -1406,53 +1421,57 @@ void transUnrefCliHandle(void* handle) { cliDestroyConn((SCliConn*)handle, true); } } -SCliThrd* transGetWorkThrdFromHandle(int64_t handle, bool* validHandle) { +static FORCE_INLINE SCliThrd* transGetWorkThrdFromHandle(STrans* trans, int64_t handle) { SCliThrd* pThrd = NULL; SExHandle* exh = transAcquireExHandle(transGetRefMgt(), handle); if (exh == NULL) { return NULL; } - *validHandle = true; + if (exh->pThrd == NULL && trans != NULL) { + int idx = cliRBChoseIdx(trans); + if (idx < 0) return NULL; + exh->pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; + } + pThrd = exh->pThrd; transReleaseExHandle(transGetRefMgt(), handle); return pThrd; } -SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle, bool* validHandle) { +SCliThrd* transGetWorkThrd(STrans* trans, int64_t handle) { if (handle == 0) { int idx = cliRBChoseIdx(trans); if (idx < 0) return NULL; return ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; } - SCliThrd* pThrd = transGetWorkThrdFromHandle(handle, validHandle); - if (*validHandle == true && pThrd == NULL) { - int idx = cliRBChoseIdx(trans); - if (idx < 0) return NULL; - pThrd = ((SCliObj*)trans->tcphandle)->pThreadObj[idx]; - } + SCliThrd* pThrd = transGetWorkThrdFromHandle(trans, handle); return pThrd; } int transReleaseCliHandle(void* handle) { int idx = -1; bool valid = false; - SCliThrd* pThrd = transGetWorkThrdFromHandle((int64_t)handle, &valid); + SCliThrd* pThrd = transGetWorkThrdFromHandle(NULL, (int64_t)handle); if (pThrd == NULL) { return -1; } - STransMsg tmsg = {.info.handle = handle}; + STransMsg tmsg = {.info.handle = handle, .info.ahandle = (void*)0x9527}; TRACE_SET_MSGID(&tmsg.info.traceId, tGenIdPI64()); + STransConnCtx* pCtx = taosMemoryCalloc(1, sizeof(STransConnCtx)); + pCtx->ahandle = tmsg.info.ahandle; + SCliMsg* cmsg = taosMemoryCalloc(1, sizeof(SCliMsg)); cmsg->msg = tmsg; cmsg->type = Release; + cmsg->ctx = pCtx; STraceId* trace = &tmsg.info.traceId; tGDebug("send release request at thread:%08" PRId64 "", pThrd->pid); if (0 != transAsyncSend(pThrd->asyncPool, &cmsg->q)) { - taosMemoryFree(cmsg); + destroyCmsg(cmsg); return -1; } return 0; @@ -1465,9 +1484,8 @@ int transSendRequest(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STran return -1; } - bool valid = false; - SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); - if (pThrd == NULL && valid == false) { + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { transFreeMsg(pReq->pCont); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return TSDB_CODE_RPC_BROKEN_LINK; @@ -1510,9 +1528,8 @@ int transSendRecv(void* shandle, const SEpSet* pEpSet, STransMsg* pReq, STransMs return -1; } - bool valid = false; - SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle, &valid); - if (pThrd == NULL && valid == false) { + SCliThrd* pThrd = transGetWorkThrd(pTransInst, (int64_t)pReq->info.handle); + if (pThrd == NULL) { transFreeMsg(pReq->pCont); transReleaseExHandle(transGetInstMgt(), (int64_t)shandle); return TSDB_CODE_RPC_BROKEN_LINK; @@ -1596,6 +1613,7 @@ int64_t transAllocHandle() { SExHandle* exh = taosMemoryCalloc(1, sizeof(SExHandle)); exh->refId = transAddExHandle(transGetRefMgt(), exh); tDebug("pre alloc refId %" PRId64 "", exh->refId); + return exh->refId; } #endif diff --git a/source/libs/transport/src/transComm.c b/source/libs/transport/src/transComm.c index a4d679b281512ff13757eab7c9c42a11e0edb36b..5f3171ee0e840ee7f558a13b4ad315bd9bcfb856 100644 --- a/source/libs/transport/src/transComm.c +++ b/source/libs/transport/src/transComm.c @@ -287,10 +287,10 @@ void transCtxMerge(STransCtx* dst, STransCtx* src) { STransCtxVal* sVal = (STransCtxVal*)iter; key = taosHashGetKey(sVal, &klen); - STransCtxVal* dVal = taosHashGet(dst->args, key, klen); - if (dVal) { - dst->freeFunc(dVal->val); - } + // STransCtxVal* dVal = taosHashGet(dst->args, key, klen); + // if (dVal) { + // dst->freeFunc(dVal->val); + // } taosHashPut(dst->args, key, klen, sVal, sizeof(*sVal)); iter = taosHashIterate(src->args, iter); } @@ -424,7 +424,7 @@ void transQueueDestroy(STransQueue* queue) { taosArrayDestroy(queue->q); } -static int32_t timeCompare(const HeapNode* a, const HeapNode* b) { +static FORCE_INLINE int32_t timeCompare(const HeapNode* a, const HeapNode* b) { SDelayTask* arg1 = container_of(a, SDelayTask, node); SDelayTask* arg2 = container_of(b, SDelayTask, node); if (arg1->execTime > arg2->execTime) { diff --git a/source/libs/transport/src/transSvr.c b/source/libs/transport/src/transSvr.c index 7007079f8760c944c70bbfe0798fb0fc518cf958..d3277d1cc1ad740bb9e6f01aba4690d42e07fd38 100644 --- a/source/libs/transport/src/transSvr.c +++ b/source/libs/transport/src/transSvr.c @@ -125,17 +125,17 @@ static void uvWorkAfterTask(uv_work_t* req, int status); static void uvWalkCb(uv_handle_t* handle, void* arg); static void uvFreeCb(uv_handle_t* handle); -static void uvStartSendRespImpl(SSvrMsg* smsg); +static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg); + static void uvPrepareSendData(SSvrMsg* msg, uv_buf_t* wb); static void uvStartSendResp(SSvrMsg* msg); static void uvNotifyLinkBrokenToApp(SSvrConn* conn); -static void destroySmsg(SSvrMsg* smsg); -// check whether already read complete packet -static SSvrConn* createConn(void* hThrd); -static void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); -static void destroyConnRegArg(SSvrConn* conn); +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg); +static FORCE_INLINE SSvrConn* createConn(void* hThrd); +static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear /*clear handle or not*/); +static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn); static int reallocConnRef(SSvrConn* conn); @@ -413,7 +413,7 @@ static void uvPrepareSendData(SSvrMsg* smsg, uv_buf_t* wb) { wb->len = len; } -static void uvStartSendRespImpl(SSvrMsg* smsg) { +static FORCE_INLINE void uvStartSendRespImpl(SSvrMsg* smsg) { SSvrConn* pConn = smsg->pConn; if (pConn->broken) { return; @@ -447,7 +447,7 @@ static void uvStartSendResp(SSvrMsg* smsg) { return; } -static void destroySmsg(SSvrMsg* smsg) { +static FORCE_INLINE void destroySmsg(SSvrMsg* smsg) { if (smsg == NULL) { return; } @@ -812,7 +812,7 @@ void* transWorkerThread(void* arg) { return NULL; } -static SSvrConn* createConn(void* hThrd) { +static FORCE_INLINE SSvrConn* createConn(void* hThrd) { SWorkThrd* pThrd = hThrd; SSvrConn* pConn = (SSvrConn*)taosMemoryCalloc(1, sizeof(SSvrConn)); @@ -842,7 +842,7 @@ static SSvrConn* createConn(void* hThrd) { return pConn; } -static void destroyConn(SSvrConn* conn, bool clear) { +static FORCE_INLINE void destroyConn(SSvrConn* conn, bool clear) { if (conn == NULL) { return; } @@ -854,7 +854,7 @@ static void destroyConn(SSvrConn* conn, bool clear) { } } } -static void destroyConnRegArg(SSvrConn* conn) { +static FORCE_INLINE void destroyConnRegArg(SSvrConn* conn) { if (conn->regArg.init == 1) { transFreeMsg(conn->regArg.msg.pCont); conn->regArg.init = 0; @@ -907,7 +907,7 @@ static void uvDestroyConn(uv_handle_t* handle) { } static void uvPipeListenCb(uv_stream_t* handle, int status) { if (status != 0) { - tError("server failed to init pipe"); + tError("server failed to init pipe, errmsg: %s", uv_err_name(status)); return; } @@ -945,7 +945,10 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, uv_loop_init(srv->loop); int ret = uv_pipe_init(srv->loop, &srv->pipeListen, 0); - assert(ret == 0); + if (ret != 0) { + tError("failed to init pipe, errmsg: %s", uv_err_name(ret)); + goto End; + } #ifdef WINDOWS char pipeName[64]; @@ -956,10 +959,16 @@ void* transInitServer(uint32_t ip, uint32_t port, char* label, int numOfThreads, taosGetSelfPthreadId()); #endif ret = uv_pipe_bind(&srv->pipeListen, pipeName); - assert(ret == 0); + if (ret != 0) { + tError("failed to bind pipe, errmsg: %s", uv_err_name(ret)); + goto End; + } ret = uv_listen((uv_stream_t*)&srv->pipeListen, SOMAXCONN, uvPipeListenCb); - assert(ret == 0); + if (ret != 0) { + tError("failed to listen pipe, errmsg: %s", uv_err_name(ret)); + goto End; + } for (int i = 0; i < srv->numOfThreads; i++) { SWorkThrd* thrd = (SWorkThrd*)taosMemoryCalloc(1, sizeof(SWorkThrd)); @@ -1082,12 +1091,12 @@ void transCloseServer(void* arg) { if (srv->inited) { uv_async_send(srv->pAcceptAsync); taosThreadJoin(srv->thread, NULL); - } - SRV_RELEASE_UV(srv->loop); + SRV_RELEASE_UV(srv->loop); - for (int i = 0; i < srv->numOfThreads; i++) { - sendQuitToWorkThrd(srv->pThreadObj[i]); - destroyWorkThrd(srv->pThreadObj[i]); + for (int i = 0; i < srv->numOfThreads; i++) { + sendQuitToWorkThrd(srv->pThreadObj[i]); + destroyWorkThrd(srv->pThreadObj[i]); + } } taosMemoryFree(srv->pThreadObj); diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 9db7d6c4554e3231399abd5ab470ea71f8207a37..5c437e6f7aeb942e02e79314e4140af0cdfe4323 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -168,6 +168,9 @@ static int32_t walReadChangeFile(SWalReader *pReader, int64_t fileFirstVer) { } pReader->pIdxFile = pIdxFile; + + pReader->curFileFirstVer = fileFirstVer; + return 0; } @@ -372,7 +375,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { int32_t walSkipFetchBody(SWalReader *pRead, const SWalCkHead *pHead) { int64_t code; -// ASSERT(pRead->curVersion == pHead->head.version); + // ASSERT(pRead->curVersion == pHead->head.version); code = taosLSeekFile(pRead->pLogFile, pHead->head.bodyLen, SEEK_CUR); if (code < 0) { @@ -415,7 +418,8 @@ int32_t walFetchBody(SWalReader *pRead, SWalCkHead **ppHead) { } if (walValidBodyCksum(*ppHead) != 0) { - wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, ver); + wError("vgId:%d, wal fetch body error, index:%" PRId64 ", since body checksum not passed", pRead->pWal->cfg.vgId, + ver); pRead->curInvalid = 1; terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; diff --git a/source/os/src/osDir.c b/source/os/src/osDir.c index 30aaa01dae0bf26bb930271f056d77226e808a4d..3dfb1458ad2fc802af761f68a4fe4407098fff25 100644 --- a/source/os/src/osDir.c +++ b/source/os/src/osDir.c @@ -160,6 +160,66 @@ int32_t taosMulMkDir(const char *dirname) { return code; } +int32_t taosMulModeMkDir(const char *dirname, int mode) { + if (dirname == NULL) return -1; + char temp[1024]; + char *pos = temp; + int32_t code = 0; +#ifdef WINDOWS + taosRealPath(dirname, temp, sizeof(temp)); + if (temp[1] == ':') pos += 3; +#else + strcpy(temp, dirname); +#endif + + if (taosDirExist(temp)) { + chmod(temp, mode); + return code; + } + + if (strncmp(temp, TD_DIRSEP, 1) == 0) { + pos += 1; + } else if (strncmp(temp, "." TD_DIRSEP, 2) == 0) { + pos += 2; + } + + for (; *pos != '\0'; pos++) { + if (*pos == TD_DIRSEP[0]) { + *pos = '\0'; +#ifdef WINDOWS + code = _mkdir(temp, mode); +#else + code = mkdir(temp, mode); +#endif + if (code < 0 && errno != EEXIST) { + terrno = TAOS_SYSTEM_ERROR(errno); + return code; + } + *pos = TD_DIRSEP[0]; + } + } + + if (*(pos - 1) != TD_DIRSEP[0]) { +#ifdef WINDOWS + code = _mkdir(temp, mode); +#else + code = mkdir(temp, mode); +#endif + if (code < 0 && errno != EEXIST) { + terrno = TAOS_SYSTEM_ERROR(errno); + return code; + } + } + + if (code < 0 && errno == EEXIST) { + chmod(temp, mode); + return 0; + } + + chmod(temp, mode); + return code; +} + void taosRemoveOldFiles(const char *dirname, int32_t keepDays) { TdDirPtr pDir = taosOpenDir(dirname); if (pDir == NULL) return; diff --git a/source/os/src/osSemaphore.c b/source/os/src/osSemaphore.c index 8cc6f0ef2e2b436624cc961315e3ecff6db7691b..5baba5af1e8051afd06b6f188795fb7c26e99a7e 100644 --- a/source/os/src/osSemaphore.c +++ b/source/os/src/osSemaphore.c @@ -400,6 +400,9 @@ int tsem_init(tsem_t *psem, int flags, unsigned int count) { } int tsem_destroy(tsem_t *psem) { + if (psem == NULL || *psem == NULL) return -1; + dispatch_release(*psem); + *psem = NULL; return 0; } @@ -421,13 +424,7 @@ int tsem_timewait(tsem_t *psem, int64_t nanosecs) { return 0; } -bool taosCheckPthreadValid(TdThread thread) { - int32_t ret = taosThreadKill(thread, 0); - if (ret == ESRCH) return false; - if (ret == EINVAL) return false; - // alive - return true; -} +bool taosCheckPthreadValid(TdThread thread) { return thread != 0; } int64_t taosGetSelfPthreadId() { TdThread thread = taosThreadSelf(); diff --git a/source/os/src/osSysinfo.c b/source/os/src/osSysinfo.c index 19e9568bbebf20a74e5f316bb50056efa4786c1a..6867c1373bdb03ad63c3001291feaad468090dea 100644 --- a/source/os/src/osSysinfo.c +++ b/source/os/src/osSysinfo.c @@ -344,30 +344,27 @@ int32_t taosGetCpuInfo(char *cpuModel, int32_t maxLen, float *numOfCores) { *numOfCores = si.dwNumberOfProcessors; return 0; #elif defined(_TD_DARWIN_64) - char *line = NULL; - size_t size = 0; + char buf[16]; int32_t done = 0; int32_t code = -1; - TdFilePtr pFile = taosOpenFile("/proc/cpuinfo", TD_FILE_READ | TD_FILE_STREAM); - if (pFile == NULL) return false; - - while (done != 3 && (size = taosGetLineFile(pFile, &line)) != -1) { - line[size - 1] = '\0'; - if (((done & 1) == 0) && strncmp(line, "model name", 10) == 0) { - const char *v = strchr(line, ':') + 2; - tstrncpy(cpuModel, v, maxLen); - code = 0; - done |= 1; - } else if (((done & 2) == 0) && strncmp(line, "cpu cores", 9) == 0) { - const char *v = strchr(line, ':') + 2; - *numOfCores = atof(v); - done |= 2; - } + TdCmdPtr pCmd = taosOpenCmd("sysctl -n machdep.cpu.brand_string"); + if (pCmd == NULL) return code; + if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) { + code = 0; + done |= 1; } + taosCloseCmd(&pCmd); - if (line != NULL) taosMemoryFree(line); - taosCloseFile(&pFile); + pCmd = taosOpenCmd("sysctl -n machdep.cpu.core_count"); + if (pCmd == NULL) return code; + memset(buf, 0, sizeof(buf)); + if (taosGetsCmd(pCmd, maxLen, cpuModel) > 0) { + code = 0; + done |= 2; + *numOfCores = atof(buf); + } + taosCloseCmd(&pCmd); return code; #else diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 3117152af6a3a5c4d1fb6ce08896924d3c0d6d1c..50c42ff170db351285aca7ef631f0143d3fc5167 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -288,6 +288,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_CONSUMER_NOT_READY, "Consumer not ready") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_SUBSCRIBED, "Topic subscribed cannot be dropped") TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOPIC_MUST_BE_DELETED, "Topic must be dropped first") TAOS_DEFINE_ERROR(TSDB_CODE_MND_CGROUP_USED, "Consumer group being used by some consumer") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_IN_REBALANCE, "Topic being rebalanced") // mnode-stream TAOS_DEFINE_ERROR(TSDB_CODE_MND_STREAM_ALREADY_EXIST, "Stream already exists") @@ -579,6 +580,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_NUM, "Invalid function par TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_TYPE, "Invalid function para type") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_FUNTION_PARA_VALUE, "Invalid function para value") TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_NOT_BUILTIN_FUNTION, "Not buildin function") +TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps not allowed in function") //udf TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") @@ -618,6 +620,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REMOVE_EXISTS, "Rsma remove exists" TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_FETCH_MSG_MSSED_UP, "Rsma fetch msg is messed up") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_EMPTY_INFO, "Rsma info is empty") TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_INVALID_SCHEMA, "Rsma invalid schema") +TAOS_DEFINE_ERROR(TSDB_CODE_RSMA_REGEX_MATCH, "Rsma regex match") //index TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_REBUILDING, "Index is rebuilding") diff --git a/source/util/src/trbtree.c b/source/util/src/trbtree.c index 0970485dade90bb8719a2fa39facb047e07bcfff..65f1bac60aa51c48fe65895dbe0ded241d22590d 100644 --- a/source/util/src/trbtree.c +++ b/source/util/src/trbtree.c @@ -13,179 +13,297 @@ * along with this program. If not, see . */ -#include "os.h" +#include "trbtree.h" -typedef int32_t (*tRBTreeCmprFn)(void *, void *); - -typedef struct SRBTree SRBTree; -typedef struct SRBTreeNode SRBTreeNode; -typedef struct SRBTreeIter SRBTreeIter; - -struct SRBTreeNode { - enum { RED, BLACK } color; - SRBTreeNode *parent; - SRBTreeNode *left; - SRBTreeNode *right; - uint8_t payload[]; -}; - -struct SRBTree { - tRBTreeCmprFn cmprFn; - SRBTreeNode *root; -}; - -struct SRBTreeIter { - SRBTree *pTree; -}; - -#define RBTREE_NODE_COLOR(N) ((N) ? (N)->color : BLACK) - -// APIs ================================================ -static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *pNode) { - SRBTreeNode *right = pNode->right; - - pNode->right = right->left; - if (pNode->right) { - pNode->right->parent = pNode; +static void tRBTreeRotateLeft(SRBTree *pTree, SRBTreeNode *x) { + SRBTreeNode *y = x->right; + x->right = y->left; + if (y->left != pTree->NIL) { + y->left->parent = x; } - - right->parent = pNode->parent; - if (pNode->parent == NULL) { - pTree->root = right; - } else if (pNode == pNode->parent->left) { - pNode->parent->left = right; + y->parent = x->parent; + if (x->parent == pTree->NIL) { + pTree->root = y; + } else if (x == x->parent->left) { + x->parent->left = y; } else { - pNode->parent->right = right; + x->parent->right = y; } - - right->left = pNode; - pNode->parent = right; + y->left = x; + x->parent = y; } -static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *pNode) { - SRBTreeNode *left = pNode->left; - - pNode->left = left->right; - if (pNode->left) { - pNode->left->parent = pNode; +static void tRBTreeRotateRight(SRBTree *pTree, SRBTreeNode *x) { + SRBTreeNode *y = x->left; + x->left = y->right; + if (y->right != pTree->NIL) { + y->right->parent = x; } - - left->parent = pNode->parent; - if (pNode->parent == NULL) { - pTree->root = left; - } else if (pNode == pNode->parent->left) { - pNode->parent->left = left; + y->parent = x->parent; + if (x->parent == pTree->NIL) { + pTree->root = y; + } else if (x == x->parent->right) { + x->parent->right = y; } else { - pNode->parent->right = left; + x->parent->left = y; } - - left->right = pNode; - pNode->parent = left; + y->right = x; + x->parent = y; } -#define tRBTreeCreate(compare) \ - (SRBTree) { .cmprFn = (compare), .root = NULL } - -SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *pNew) { - pNew->left = NULL; - pNew->right = NULL; - pNew->color = RED; - - // insert - if (pTree->root == NULL) { - pNew->parent = NULL; - pTree->root = pNew; - } else { - SRBTreeNode *pNode = pTree->root; - while (true) { - ASSERT(pNode); - - int32_t c = pTree->cmprFn(pNew->payload, pNode->payload); - if (c < 0) { - if (pNode->left) { - pNode = pNode->left; - } else { - pNew->parent = pNode; - pNode->left = pNew; - break; - } - } else if (c > 0) { - if (pNode->right) { - pNode = pNode->right; - } else { - pNew->parent = pNode; - pNode->right = pNew; - break; +static void tRBTreePutFix(SRBTree *pTree, SRBTreeNode *z) { + while (z->parent->color == RED) { + if (z->parent == z->parent->parent->left) { // z.parent is the left child + + SRBTreeNode *y = z->parent->parent->right; // uncle of z + + if (y->color == RED) { // case 1 + z->parent->color = BLACK; + y->color = BLACK; + z->parent->parent->color = RED; + z = z->parent->parent; + } else { // case2 or case3 + if (z == z->parent->right) { // case2 + z = z->parent; // marked z.parent as new z + tRBTreeRotateLeft(pTree, z); } + // case3 + z->parent->color = BLACK; // made parent black + z->parent->parent->color = RED; // made parent red + tRBTreeRotateRight(pTree, z->parent->parent); + } + } else { // z.parent is the right child + SRBTreeNode *y = z->parent->parent->left; // uncle of z + + if (y->color == RED) { + z->parent->color = BLACK; + y->color = BLACK; + z->parent->parent->color = RED; + z = z->parent->parent; } else { - return NULL; + if (z == z->parent->left) { + z = z->parent; // marked z.parent as new z + tRBTreeRotateRight(pTree, z); + } + z->parent->color = BLACK; // made parent black + z->parent->parent->color = RED; // made parent red + tRBTreeRotateLeft(pTree, z->parent->parent); } } } + pTree->root->color = BLACK; +} - // fix - SRBTreeNode *pNode = pNew; - while (pNode->parent && pNode->parent->color == RED) { - SRBTreeNode *p = pNode->parent; - SRBTreeNode *g = p->parent; - - if (p == g->left) { - SRBTreeNode *u = g->right; - - if (RBTREE_NODE_COLOR(u) == RED) { - p->color = BLACK; - u->color = BLACK; - g->color = RED; - pNode = g; +static void tRBTreeTransplant(SRBTree *pTree, SRBTreeNode *u, SRBTreeNode *v) { + if (u->parent == pTree->NIL) + pTree->root = v; + else if (u == u->parent->left) + u->parent->left = v; + else + u->parent->right = v; + v->parent = u->parent; +} + +static void tRBTreeDropFix(SRBTree *pTree, SRBTreeNode *x) { + while (x != pTree->root && x->color == BLACK) { + if (x == x->parent->left) { + SRBTreeNode *w = x->parent->right; + if (w->color == RED) { + w->color = BLACK; + x->parent->color = RED; + tRBTreeRotateLeft(pTree, x->parent); + w = x->parent->right; + } + if (w->left->color == BLACK && w->right->color == BLACK) { + w->color = RED; + x = x->parent; } else { - if (pNode == p->right) { - pNode = p; - tRBTreeRotateLeft(pTree, pNode); + if (w->right->color == BLACK) { + w->left->color = BLACK; + w->color = RED; + tRBTreeRotateRight(pTree, w); + w = x->parent->right; } - pNode->parent->color = BLACK; - pNode->parent->parent->color = RED; - tRBTreeRotateRight(pTree, pNode->parent->parent); + w->color = x->parent->color; + x->parent->color = BLACK; + w->right->color = BLACK; + tRBTreeRotateLeft(pTree, x->parent); + x = pTree->root; } } else { - SRBTreeNode *u = g->left; - - if (RBTREE_NODE_COLOR(u) == RED) { - p->color = BLACK; - u->color = BLACK; - g->color = RED; + SRBTreeNode *w = x->parent->left; + if (w->color == RED) { + w->color = BLACK; + x->parent->color = RED; + tRBTreeRotateRight(pTree, x->parent); + w = x->parent->left; + } + if (w->right->color == BLACK && w->left->color == BLACK) { + w->color = RED; + x = x->parent; } else { - if (pNode == p->left) { - pNode = p; - tRBTreeRotateRight(pTree, pNode); + if (w->left->color == BLACK) { + w->right->color = BLACK; + w->color = RED; + tRBTreeRotateLeft(pTree, w); + w = x->parent->left; } - pNode->parent->color = BLACK; - pNode->parent->parent->color = RED; - tRBTreeRotateLeft(pTree, pNode->parent->parent); + w->color = x->parent->color; + x->parent->color = BLACK; + w->left->color = BLACK; + tRBTreeRotateRight(pTree, x->parent); + x = pTree->root; } } } + x->color = BLACK; +} - pTree->root->color = BLACK; - return pNew; +static SRBTreeNode *tRBTreeSuccessor(SRBTree *pTree, SRBTreeNode *pNode) { + if (pNode->right != pTree->NIL) { + pNode = pNode->right; + while (pNode->left != pTree->NIL) { + pNode = pNode->left; + } + } else { + while (true) { + if (pNode->parent == pTree->NIL || pNode == pNode->parent->left) { + pNode = pNode->parent; + break; + } else { + pNode = pNode->parent; + } + } + } + + return pNode; } -SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { - SRBTreeNode *pNode = pTree->root; +static SRBTreeNode *tRBTreePredecessor(SRBTree *pTree, SRBTreeNode *pNode) { + if (pNode->left != pTree->NIL) { + pNode = pNode->left; + while (pNode->right != pTree->NIL) { + pNode = pNode->right; + } + } else { + while (true) { + if (pNode->parent == pTree->NIL || pNode == pNode->parent->right) { + pNode = pNode->parent; + break; + } else { + pNode = pNode->parent; + } + } + } - while (pNode) { - int32_t c = pTree->cmprFn(pKey, pNode->payload); + return pNode; +} +void tRBTreeCreate(SRBTree *pTree, tRBTreeCmprFn cmprFn) { + pTree->cmprFn = cmprFn; + pTree->n = 0; + pTree->NIL = &pTree->NILNODE; + pTree->NIL->color = BLACK; + pTree->NIL->parent = NULL; + pTree->NIL->left = NULL; + pTree->NIL->right = NULL; + pTree->root = pTree->NIL; + pTree->min = pTree->NIL; + pTree->max = pTree->NIL; +} + +SRBTreeNode *tRBTreePut(SRBTree *pTree, SRBTreeNode *z) { + SRBTreeNode *y = pTree->NIL; // variable for the parent of the added node + SRBTreeNode *temp = pTree->root; + + while (temp != pTree->NIL) { + y = temp; + + int32_t c = pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(temp)); if (c < 0) { - pNode = pNode->left; + temp = temp->left; } else if (c > 0) { - pNode = pNode->right; + temp = temp->right; } else { - break; + return NULL; } } + z->parent = y; + + if (y == pTree->NIL) { + pTree->root = z; + } else if (pTree->cmprFn(RBTREE_NODE_PAYLOAD(z), RBTREE_NODE_PAYLOAD(y)) < 0) { + y->left = z; + } else { + y->right = z; + } + + z->color = RED; + z->left = pTree->NIL; + z->right = pTree->NIL; + + tRBTreePutFix(pTree, z); + + // update min/max node + if (pTree->min == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->min), RBTREE_NODE_PAYLOAD(z)) > 0) { + pTree->min = z; + } + if (pTree->max == pTree->NIL || pTree->cmprFn(RBTREE_NODE_PAYLOAD(pTree->max), RBTREE_NODE_PAYLOAD(z)) < 0) { + pTree->max = z; + } + pTree->n++; + return z; +} + +void tRBTreeDrop(SRBTree *pTree, SRBTreeNode *z) { + SRBTreeNode *y = z; + SRBTreeNode *x; + ECOLOR y_orignal_color = y->color; + + // update min/max node + if (pTree->min == z) { + pTree->min = tRBTreeSuccessor(pTree, pTree->min); + } + if (pTree->max == z) { + pTree->max = tRBTreePredecessor(pTree, pTree->max); + } + + // drop impl + if (z->left == pTree->NIL) { + x = z->right; + tRBTreeTransplant(pTree, z, z->right); + } else if (z->right == pTree->NIL) { + x = z->left; + tRBTreeTransplant(pTree, z, z->left); + } else { + y = tRBTreeSuccessor(pTree, z); + y_orignal_color = y->color; + x = y->right; + if (y->parent == z) { + x->parent = z; + } else { + tRBTreeTransplant(pTree, y, y->right); + y->right = z->right; + y->right->parent = y; + } + tRBTreeTransplant(pTree, z, y); + y->left = z->left; + y->left->parent = y; + y->color = z->color; + } + + // fix + if (y_orignal_color == BLACK) { + tRBTreeDropFix(pTree, x); + } + pTree->n--; +} + +SRBTreeNode *tRBTreeDropByKey(SRBTree *pTree, void *pKey) { + SRBTreeNode *pNode = tRBTreeGet(pTree, pKey); if (pNode) { - // TODO + tRBTreeDrop(pTree, pNode); } return pNode; @@ -194,8 +312,8 @@ SRBTreeNode *tRBTreeDrop(SRBTree *pTree, void *pKey) { SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { SRBTreeNode *pNode = pTree->root; - while (pNode) { - int32_t c = pTree->cmprFn(pKey, pNode->payload); + while (pNode != pTree->NIL) { + int32_t c = pTree->cmprFn(pKey, RBTREE_NODE_PAYLOAD(pNode)); if (c < 0) { pNode = pNode->left; @@ -206,5 +324,23 @@ SRBTreeNode *tRBTreeGet(SRBTree *pTree, void *pKey) { } } - return pNode; + return (pNode == pTree->NIL) ? NULL : pNode; } + +// SRBTreeIter ================================================ +SRBTreeNode *tRBTreeIterNext(SRBTreeIter *pIter) { + SRBTreeNode *pNode = pIter->pNode; + + if (pIter->pNode != pIter->pTree->NIL) { + if (pIter->asc) { + // ascend + pIter->pNode = tRBTreeSuccessor(pIter->pTree, pIter->pNode); + } else { + // descend + pIter->pNode = tRBTreePredecessor(pIter->pTree, pIter->pNode); + } + } + +_exit: + return (pNode == pIter->pTree->NIL) ? NULL : pNode; +} \ No newline at end of file diff --git a/source/util/src/tsched.c b/source/util/src/tsched.c index 89471c4347e49152b2b1fff0313ef29da92c70ff..9cf9e2c4316a850ac31b15aaab8a864deb4d32eb 100644 --- a/source/util/src/tsched.c +++ b/source/util/src/tsched.c @@ -149,18 +149,18 @@ void *taosProcessSchedQueue(void *scheduler) { return NULL; } -void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { +int taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { SSchedQueue *pSched = (SSchedQueue *)queueScheduler; int32_t ret = 0; if (pSched == NULL) { uError("sched is not ready, msg:%p is dropped", pMsg); - return; + return -1; } if (atomic_load_8(&pSched->stop)) { uError("sched is already stopped, msg:%p is dropped", pMsg); - return; + return -1; } if ((ret = tsem_wait(&pSched->emptySem)) != 0) { @@ -185,6 +185,7 @@ void taosScheduleTask(void *queueScheduler, SSchedMsg *pMsg) { uFatal("post %s fullSem failed(%s)", pSched->label, strerror(errno)); ASSERT(0); } + return ret; } void taosCleanUpScheduler(void *param) { @@ -192,11 +193,11 @@ void taosCleanUpScheduler(void *param) { if (pSched == NULL) return; uDebug("start to cleanup %s schedQsueue", pSched->label); - + atomic_store_8(&pSched->stop, 1); taosMsleep(200); - + for (int32_t i = 0; i < pSched->numOfThreads; ++i) { if (taosCheckPthreadValid(pSched->qthread[i])) { tsem_post(&pSched->fullSem); @@ -220,7 +221,7 @@ void taosCleanUpScheduler(void *param) { if (pSched->queue) taosMemoryFree(pSched->queue); if (pSched->qthread) taosMemoryFree(pSched->qthread); - //taosMemoryFree(pSched); + // taosMemoryFree(pSched); } // for debug purpose, dump the scheduler status every 1min. diff --git a/source/util/src/tuuid.c b/source/util/src/tuuid.c index 9101aec949873eb976517581c6790e1bc2dac3b9..7460ccbc829e7c630d217a39f29a2eafa934cb53 100644 --- a/source/util/src/tuuid.c +++ b/source/util/src/tuuid.c @@ -51,11 +51,11 @@ int64_t tGenIdPI64(void) { int64_t id; while (true) { - int64_t ts = taosGetTimestampMs(); + int64_t ts = taosGetTimestampMs() >> 8; uint64_t pid = taosGetPId(); int32_t val = atomic_add_fetch_32(&tUUIDSerialNo, 1); - id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0FFF) << 40) | ((ts & 0xFFFFFF) << 16) | (val & 0xFFFF); + id = ((tUUIDHashId & 0x07FF) << 52) | ((pid & 0x0F) << 48) | ((ts & 0x3FFFFFF) << 20) | (val & 0xFFFFF); if (id) { break; } diff --git a/source/util/test/CMakeLists.txt b/source/util/test/CMakeLists.txt index d2a503e6613909c39af8cfb961dd5b51f36120c3..6e42ef7e75eac38a5a072bfef8521152fd74ae06 100644 --- a/source/util/test/CMakeLists.txt +++ b/source/util/test/CMakeLists.txt @@ -75,4 +75,12 @@ target_link_libraries(taosbsearchTest os util gtest_main) add_test( NAME taosbsearchTest COMMAND taosbsearchTest +) + +# trbtreeTest +add_executable(rbtreeTest "trbtreeTest.cpp") +target_link_libraries(rbtreeTest os util gtest_main) +add_test( + NAME rbtreeTest + COMMAND rbtreeTest ) \ No newline at end of file diff --git a/source/util/test/pageBufferTest.cpp b/source/util/test/pageBufferTest.cpp index 1a057c5875ee95de2fc3c457ca09314366fff48c..534c17758714820e9a9f2bf6b81d23ac121fcaf4 100644 --- a/source/util/test/pageBufferTest.cpp +++ b/source/util/test/pageBufferTest.cpp @@ -23,9 +23,9 @@ void simpleTest() { ASSERT_EQ(getTotalBufSize(pBuf), 1024); - SIDList list = getDataBufPagesIdList(pBuf, groupId); + SIDList list = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(list), 1); - ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); + //ASSERT_EQ(getNumOfBufGroupId(pBuf), 1); releaseBufPage(pBuf, pBufPage); @@ -98,7 +98,7 @@ void writeDownTest() { SFilePage* pBufPagex = static_cast(getBufPage(pBuf, writePageId)); ASSERT_EQ(*(int32_t*)pBufPagex->data, nx); - SArray* pa = getDataBufPagesIdList(pBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(pa), 5); destroyDiskbasedBuf(pBuf); @@ -152,7 +152,7 @@ void recyclePageTest() { SFilePage* pBufPagex1 = static_cast(getBufPage(pBuf, 1)); - SArray* pa = getDataBufPagesIdList(pBuf, groupId); + SArray* pa = getDataBufPagesIdList(pBuf); ASSERT_EQ(taosArrayGetSize(pa), 6); destroyDiskbasedBuf(pBuf); diff --git a/source/util/test/trbtreeTest.cpp b/source/util/test/trbtreeTest.cpp new file mode 100644 index 0000000000000000000000000000000000000000..cabf315df0a2d32c6f2adcf1ac81a3e1565e6e0c --- /dev/null +++ b/source/util/test/trbtreeTest.cpp @@ -0,0 +1,40 @@ +#include + +#include +#include + +#include "trbtree.h" + +static int32_t tCmprInteger(const void *p1, const void *p2) { + if (*(int *)p1 < *(int *)p2) { + return -1; + } else if (*(int *)p1 > *(int *)p2) { + return 1; + } + return 0; +} + +TEST(trbtreeTest, rbtree_test1) { +#if 0 + SRBTree rt; + tRBTreeCreate(&rt, tCmprInteger); + int a[] = {1, 3, 4, 2, 7, 5, 8}; + + for (int i = 0; i < sizeof(a) / sizeof(a[0]); i++) { + SRBTreeNode *pNode = (SRBTreeNode *)taosMemoryMalloc(sizeof(*pNode) + sizeof(int)); + *(int *)pNode->payload = a[i]; + + tRBTreePut(&rt, pNode); + } + + SRBTreeIter rti = tRBTreeIterCreate(&rt, 1); + SRBTreeNode *pNode = tRBTreeIterNext(&rti); + int la = 0; + while (pNode) { + GTEST_ASSERT_GT(*(int *)pNode->payload, la); + la = *(int *)pNode->payload; + // printf("%d\n", la); + pNode = tRBTreeIterNext(&rti); + } +#endif +} \ No newline at end of file diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py index 99e8cd36a4cf1dae08baf93ef4d6338bb08dc7bd..a44ad4c1d0b5cc8e4ba181c1bf941454929f2df6 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/demo.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/demo.py @@ -22,9 +22,9 @@ from util.dnodes import * class TDTestCase: def caseDescription(self): - ''' + """ [TD-13823] taosBenchmark test cases - ''' + """ return def init(self, conn, logSql): @@ -34,19 +34,19 @@ class TDTestCase: def getPath(self, tool="taosBenchmark"): selfPath = os.path.dirname(os.path.realpath(__file__)) - if ("community" in selfPath): - projPath = selfPath[:selfPath.find("community")] + if "community" in selfPath: + projPath = selfPath[: selfPath.find("community")] else: - projPath = selfPath[:selfPath.find("tests")] + projPath = selfPath[: selfPath.find("tests")] paths = [] for root, dirs, files in os.walk(projPath): - if ((tool) in files): + if (tool) in files: rootRealPath = os.path.dirname(os.path.realpath(root)) - if ("packaging" not in rootRealPath): + if "packaging" not in rootRealPath: paths.append(os.path.join(root, tool)) break - if (len(paths) == 0): + if len(paths) == 0: tdLog.exit("taosBenchmark not found!") return else: @@ -55,7 +55,7 @@ class TDTestCase: def run(self): binPath = self.getPath() - cmd = "%s -n 100 -t 100 -y" %binPath + cmd = "%s -n 100 -t 100 -y" % binPath tdLog.info("%s" % cmd) os.system("%s" % cmd) tdSql.execute("use test") @@ -77,14 +77,16 @@ class TDTestCase: tdSql.checkData(4, 3, "TAG") tdSql.checkData(5, 0, "location") tdSql.checkData(5, 1, "VARCHAR") - tdSql.checkData(5, 2, 16) + tdSql.checkData(5, 2, 24) tdSql.checkData(5, 3, "TAG") tdSql.query("select count(*) from test.meters where groupid >= 0") tdSql.checkData(0, 0, 10000) - tdSql.query("select count(*) from test.meters where location = 'San Francisco' or location = 'Los Angles' or location = 'San Diego' or location = 'San Jose' or \ - location = 'Palo Alto' or location = 'Campbell' or location = 'Mountain View' or location = 'Sunnyvale' or location = 'Santa Clara' or location = 'Cupertino' ") + tdSql.query( + "select count(*) from test.meters where location = 'California.SanFrancisco' or location = 'California.LosAngles' or location = 'California.SanDiego' or location = 'California.SanJose' or \ + location = 'California.PaloAlto' or location = 'California.Campbell' or location = 'California.MountainView' or location = 'California.Sunnyvale' or location = 'California.SantaClara' or location = 'California.Cupertino' " + ) tdSql.checkData(0, 0, 10000) def stop(self): diff --git a/tests/pytest/tools/taosdumpTest2.py b/tests/pytest/tools/taosdumpTest2.py index 7071adb3a95f1c240d22a7dad6354d1e08b28400..8a85ce10ed53946abe4f8ecd4a022752e07f94c1 100644 --- a/tests/pytest/tools/taosdumpTest2.py +++ b/tests/pytest/tools/taosdumpTest2.py @@ -134,7 +134,6 @@ class TDTestCase: os.system("%s -i ./taosdumptest/tmp -y" % binPath) tdSql.execute("use test") - tdSql.error("show vnodes '' ") tdSql.query("show stables") tdSql.checkRows(1) tdSql.checkData(0, 0, 'stb') diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 46bae734ea72901ef704969045186a10c52a9a72..161c87844058ca6852c6649141e37f6cdf6a202f 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -221,6 +221,7 @@ ./test.sh -f tsim/table/describe.sim ./test.sh -f tsim/table/double.sim ./test.sh -f tsim/table/float.sim +./test.sh -f tsim/table/hash.sim ./test.sh -f tsim/table/int.sim ./test.sh -f tsim/table/limit.sim ./test.sh -f tsim/table/smallint.sim @@ -248,6 +249,12 @@ ./test.sh -f tsim/stream/windowClose.sim ./test.sh -f tsim/stream/ignoreExpiredData.sim ./test.sh -f tsim/stream/sliding.sim +#./test.sh -f tsim/stream/partitionbyColumnInterval.sim +#./test.sh -f tsim/stream/partitionbyColumnSession.sim +#./test.sh -f tsim/stream/partitionbyColumnState.sim +#./test.sh -f tsim/stream/deleteInterval.sim +#./test.sh -f tsim/stream/deleteSession.sim +#./test.sh -f tsim/stream/deleteState.sim # ---- transaction ---- ./test.sh -f tsim/trans/lossdata1.sim diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c deleted file mode 100644 index d623adacf941e26d0de74c0c582beb7ca83c9c13..0000000000000000000000000000000000000000 --- a/tests/script/sh/abs_max.c +++ /dev/null @@ -1,88 +0,0 @@ -#include -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - - -#define TSDB_DATA_INT_NULL 0x80000000LL -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL - -void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - int r = 0; - printf("abs_max input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); - if (itype == 5) { - r=*(long *)dataOutput; - *numOfOutput=0; - - for(i=0;i r) { - r = v; - } - } - - *(long *)dataOutput=r; - - printf("abs_max out, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); - } -} - - - -void abs_max_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) { - int i; - int r = 0; - printf("abs_max_finalize dataoutput:%p:%d, numOfOutput:%d, buf:%p\n", dataOutput, *dataOutput, *numOfOutput, buf); - *numOfOutput=1; - printf("abs_max finalize, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); -} - -void abs_max_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int r = 0; - - if (numOfRows > 0) { - r = *((long *)data); - } - printf("abs_max_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); - for (int i = 1; i < numOfRows; ++i) { - printf("abs_max_merge %d - %ld\n", i, *((long *)data + i)); - if (*((long*)data + i) > r) { - r= *((long*)data + i); - } - } - - *(long*)dataOutput=r; - if (numOfRows > 0) { - *numOfOutput=1; - } else { - *numOfOutput=0; - } - - printf("abs_max_merge, dataoutput:%ld, numOfOutput:%d\n", *(long *)dataOutput, *numOfOutput); -} - - -int abs_max_init(SUdfInit* buf) { - printf("abs_max init\n"); - return 0; -} - - -void abs_max_destroy(SUdfInit* buf) { - printf("abs_max destroy\n"); -} - diff --git a/tests/script/sh/add_one.c b/tests/script/sh/add_one.c deleted file mode 100644 index e12cf8f26f6ddad67f9f7b091c033de46a3f6f50..0000000000000000000000000000000000000000 --- a/tests/script/sh/add_one.c +++ /dev/null @@ -1,33 +0,0 @@ -#include -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - -void add_one(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBUf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - int r = 0; - printf("add_one input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); - if (itype == 4) { - for(i=0;i -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - -typedef struct SDemo{ - double sum; - int num; - short otype; -}SDemo; - -#define FLOAT_NULL 0x7FF00000 // it is an NAN -#define DOUBLE_NULL 0x7FFFFF0000000000LL // it is an NAN - - -void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - double r = 0; - SDemo *p = (SDemo *)interBuf; - SDemo *q = (SDemo *)dataOutput; - printf("demo input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, interBUf:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, interBuf, tsOutput, numOfOutput, buf); - - for(i=0;isum += r*r; - } - - p->otype = otype; - p->num += numOfRows; - - q->sum = p->sum; - q->num = p->num; - q->otype = p->otype; - - *numOfOutput=1; - - printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput); -} - - -void demo_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int i; - SDemo *p = (SDemo *)data; - SDemo res = {0}; - printf("demo_merge input data:%p, rows:%d, dataoutput:%p, numOfOutput:%p, buf:%p\n", data, numOfRows, dataOutput, numOfOutput, buf); - - for(i=0;isum * p->sum; - res.num += p->num; - p++; - } - - p->sum = res.sum; - p->num = res.num; - - *numOfOutput=1; - - printf("demo out, sum:%f, num:%d, numOfOutput:%d\n", p->sum, p->num, *numOfOutput); -} - - - -void demo_finalize(char* dataOutput, char* interBuf, int* numOfOutput, SUdfInit* buf) { - SDemo *p = (SDemo *)interBuf; - printf("demo_finalize interbuf:%p, numOfOutput:%p, buf:%p, sum:%f, num:%d\n", interBuf, numOfOutput, buf, p->sum, p->num); - if (p->otype == 6) { - if (p->num != 30000) { - *(unsigned int *)dataOutput = FLOAT_NULL; - } else { - *(float *)dataOutput = (float)(p->sum / p->num); - } - printf("finalize values:%f\n", *(float *)dataOutput); - } else if (p->otype == 7) { - if (p->num != 30000) { - *(unsigned long long *)dataOutput = DOUBLE_NULL; - } else { - *(double *)dataOutput = (double)(p->sum / p->num); - } - printf("finalize values:%f\n", *(double *)dataOutput); - } - - *numOfOutput=1; - - printf("demo finalize, numOfOutput:%d\n", *numOfOutput); -} - - -int demo_init(SUdfInit* buf) { - printf("demo init\n"); - return 0; -} - - -void demo_destroy(SUdfInit* buf) { - printf("demo destroy\n"); -} - diff --git a/tests/script/sh/demo.lua b/tests/script/sh/demo.lua deleted file mode 100644 index c5e5582fc30b58db30a5b18faa4ccfd0a5f656d0..0000000000000000000000000000000000000000 --- a/tests/script/sh/demo.lua +++ /dev/null @@ -1,43 +0,0 @@ -funcName = "test" - -global = {} - -function test_init() - return global -end - -function test_add(rows, ans, key) - t = {} - t["sum"] = 0.0 - t["num"] = 0 - for i=1, #rows do - t["sum"] = t["sum"] + rows[i] * rows[i] - end - t["num"] = #rows - - - if (ans[key] ~= nil) - then - ans[key]["sum"] = ans[key]["sum"] + t["sum"] - ans[key]["num"] = ans[key]["num"] + t["num"] - else - ans[key] = t - end - - return ans; -end - -function test_finalize(ans, key) - local ret = 0.0 - - if (ans[key] ~= nil and ans[key]["num"] == 30000) - then - ret = ans[key]["sum"]/ans[key]["num"] - ans[key]["sum"] = 0.0 - ans[key]["num"] = 0 - else - ret = inf - end - - return ret, ans -end diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c deleted file mode 100644 index d6eea5d291c4add6bd40ae36492a1366c67f2cfd..0000000000000000000000000000000000000000 --- a/tests/script/sh/sum_double.c +++ /dev/null @@ -1,84 +0,0 @@ -#include -#include -#include - -typedef struct SUdfInit{ - int maybe_null; /* 1 if function can return NULL */ - int decimals; /* for real functions */ - long long length; /* For string functions */ - char *ptr; /* free pointer for function data */ - int const_item; /* 0 if result is independent of arguments */ -} SUdfInit; - -#define TSDB_DATA_INT_NULL 0x80000000LL - - -void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, - int* numOfOutput, short otype, short obytes, SUdfInit* buf) { - int i; - int r = 0; - printf("sum_double input data:%p, type:%d, rows:%d, ts:%p,%lld, dataoutput:%p, tsOutput:%p, numOfOutput:%p, buf:%p\n", data, itype, numOfRows, ts, *ts, dataOutput, tsOutput, numOfOutput, buf); - if (itype == 4) { - r=*(int *)dataOutput; - *numOfOutput=0; - - for(i=0;iptr)=*(int*)dataOutput*2; - *(int*)dataOutput=*(int*)(buf->ptr); - printf("sum_double finalize, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput); -} - -void sum_double_merge(char* data, int32_t numOfRows, char* dataOutput, int32_t* numOfOutput, SUdfInit* buf) { - int r = 0; - int sum = 0; - - printf("sum_double_merge numOfRows:%d, dataoutput:%p, buf:%p\n", numOfRows, dataOutput, buf); - for (int i = 0; i < numOfRows; ++i) { - printf("sum_double_merge %d - %d\n", i, *((int*)data + i)); - sum +=*((int*)data + i); - } - - *(int*)dataOutput+=sum; - if (numOfRows > 0) { - *numOfOutput=1; - } else { - *numOfOutput=0; - } - - printf("sum_double_merge, dataoutput:%d, numOfOutput:%d\n", *(int *)dataOutput, *numOfOutput); -} - - -int sum_double_init(SUdfInit* buf) { - buf->maybe_null=1; - buf->ptr = taosMemoryMalloc(sizeof(int)); - printf("sum_double init\n"); - return 0; -} - - -void sum_double_destroy(SUdfInit* buf) { - taosMemoryFree(buf->ptr); - printf("sum_double destroy\n"); -} - diff --git a/tests/script/tsim/column/table.sim b/tests/script/tsim/column/table.sim index 4f1d32c373e3712275deec14a54e7efa0e77de61..03c4799681015a63e1e343168037f6cd4d601b8b 100644 --- a/tests/script/tsim/column/table.sim +++ b/tests/script/tsim/column/table.sim @@ -159,6 +159,7 @@ if $data01 != 10 then return -1 endi if $data02 != 4.500000000 then + print expect 4.500000000, actual: $data02 return -1 endi if $data03 != 4.500000000 then diff --git a/tests/script/tsim/db/basic1.sim b/tests/script/tsim/db/basic1.sim index 69eeb9347b22b154c7609b2ff89e36aa43f63a82..5d7d0bd9e9a682a761ffb33f486545f7028f5bfc 100644 --- a/tests/script/tsim/db/basic1.sim +++ b/tests/script/tsim/db/basic1.sim @@ -107,6 +107,37 @@ if $data30 != 12 then return -1 endi +print =============== show vnodes +sql show vnodes 1 +if $rows != 9 then + return -1 +endi + +if $data(4)[1] != 1 then + return -1 +endi + +if $data(4)[2] != leader then + return -1 +endi + +if $data(4)[3] != d2 then + return -1 +endi + +if $data(4)[4] != 1 then + return -1 +endi + +if $data(4)[5] != localhost:7100 then + return -1 +endi + +sql show vnodes 'localhost:7100' +if $rows != 9 then + return -1 +endi + print =============== drop database sql drop database d2 sql drop database d3 diff --git a/tests/script/tsim/parser/commit.sim b/tests/script/tsim/parser/commit.sim index ae19a4803bc6bd0f75c2a307696d7b0fc6f1ecb6..a9bf8b26ebda178ed789f3119eac7203a6889f9b 100644 --- a/tests/script/tsim/parser/commit.sim +++ b/tests/script/tsim/parser/commit.sim @@ -97,6 +97,7 @@ while $loop <= $loops endw sql select count(*) from $stb if $data00 != $totalNum then + print expect $totalNum , actual: $data00 return -1 endi $loop = $loop + 1 diff --git a/tests/script/tsim/parser/slimit_alter_tags.sim b/tests/script/tsim/parser/slimit_alter_tags.sim index 3827b14b453132687c254250538d53ae51ab7279..b5afbfa56eea5fb6ba440e08818e861d5a96da3d 100644 --- a/tests/script/tsim/parser/slimit_alter_tags.sim +++ b/tests/script/tsim/parser/slimit_alter_tags.sim @@ -128,6 +128,7 @@ if $rows != 5 then return -1 endi if $data00 != $rowNum then + print expect $rowNum , actual: $data00 return -1 endi if $data10 != $rowNum then diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index 5392979c0a218884a50d0ebe9ddb39558e82304f..8942f7f702787c9e026e0c37e47ce56765f554bf 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -5,7 +5,7 @@ sleep 50 sql connect print =============== create database -sql create database test vgroups 1 +sql create database test vgroups 1; sql select * from information_schema.ins_databases if $rows != 3 then return -1 @@ -13,7 +13,7 @@ endi print $data00 $data01 $data02 -sql use test +sql use test; sql create table t1(ts timestamp, a int, b int , c int, d double); @@ -462,10 +462,10 @@ if $data25 != 3 then return -1 endi -sql create database test2 vgroups 1 -sql select * from information_schema.ins_databases +sql create database test2 vgroups 1; +sql select * from information_schema.ins_databases; -sql use test2 +sql use test2; sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); sql create table t1 using st tags(1,1,1); sql create table t2 using st tags(2,2,2); @@ -588,4 +588,38 @@ if $data00 != 5 then goto loop3 endi +#max,min selectivity +sql create database test3 vgroups 1; +sql use test3; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from ts1 interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts1 values(1648791222001,2,2,3); +sleep 50 + +$loop_count = 0 +loop3: +sql select * from streamtST3; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data02 != 1 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data12 != 2 then + print =====data12=$data12 + goto loop3 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/deleteInterval.sim b/tests/script/tsim/stream/deleteInterval.sim new file mode 100644 index 0000000000000000000000000000000000000000..00d10afad99a258d3c931add7e7ceedc653e85df --- /dev/null +++ b/tests/script/tsim/stream/deleteInterval.sim @@ -0,0 +1,495 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,2,2,2,2.0); +sql insert into t1 values(1648791213002,3,3,3,3.0); +sql insert into t1 values(1648791213003,4,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,2,2,2,2.0); +sql insert into t1 values(1648791213006,3,3,3,3.0); +sql insert into t1 values(1648791213007,4,4,4,4.0); + +sql insert into t1 values(1648791223000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,3,3,3,3.0); +sql insert into t1 values(1648791223003,4,4,4,4.0); + +sql insert into t1 values(1648791233000,1,1,1,1.0); +sql insert into t1 values(1648791233001,2,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,4,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql drop database if exists test; +sql create database test2 vgroups 4; +sql create database test vgroups 1; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +sleep 200 + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop8: +sleep 200 + +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001; + +$loop_count = 0 + +loop11: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop11 +endi + +sleep 200 + +sql delete from st where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop12: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop12 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop12 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop12 +endi + +sql insert into t1 values(1648791213004,3,2,3,1.0); +sql insert into t1 values(1648791213005,3,2,3,1.0); +sql insert into t1 values(1648791213006,3,2,3,1.0); +sql insert into t1 values(1648791223004,1,2,3,1.0); +sql insert into t2 values(1648791213004,3,2,3,1.0); +sql insert into t2 values(1648791213005,3,2,3,1.0); +sql insert into t2 values(1648791213006,3,2,3,1.0); +sql insert into t2 values(1648791223004,1,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006; + +$loop_count = 0 + +loop13: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 3 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t1 values(1648791223005,1,2,3,1.0); +sql insert into t1 values(1648791223006,1,2,3,1.0); +sql insert into t2 values(1648791223005,1,2,3,1.0); +sql insert into t2 values(1648791223006,1,2,3,1.0); + +sql insert into t1 values(1648791233005,4,2,3,1.0); +sql insert into t1 values(1648791233006,2,2,3,1.0); +sql insert into t2 values(1648791233005,5,2,3,1.0); +sql insert into t2 values(1648791233006,3,2,3,1.0); + +sleep 200 + +sql delete from st where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop14: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop14 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop14 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop14 +endi + +return 1 + +sql drop stream if exists streams3; +sql drop database if exists test3; +sql drop database if exists test; +sql create database test3 vgroups 4; +sql create database test vgroups 1; +sql use test3; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +sql delete from t1; + +loop15: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop15 +endi + +$loop_count = 0 + +sql delete from t1 where ts > 100; + +loop16: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop16 +endi + +$loop_count = 0 + +sql delete from st; + +loop17: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop17 +endi + + + + + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/deleteSession.sim b/tests/script/tsim/stream/deleteSession.sim new file mode 100644 index 0000000000000000000000000000000000000000..541609633b023611815252cde0109cdc01094198 --- /dev/null +++ b/tests/script/tsim/stream/deleteSession.sim @@ -0,0 +1,532 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3 from t1 session(ts, 5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,2,2,2,2.0); +sql insert into t1 values(1648791213002,3,3,3,3.0); +sql insert into t1 values(1648791213003,4,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,2,2,2,2.0); +sql insert into t1 values(1648791213006,3,3,3,3.0); +sql insert into t1 values(1648791213007,4,4,4,4.0); + +sql insert into t1 values(1648791223000,1,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,3,3,3,3.0); +sql insert into t1 values(1648791223003,4,4,4,4.0); + +sql insert into t1 values(1648791233000,1,1,1,1.0); +sql insert into t1 values(1648791233001,2,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,4,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop7 +endi + +sleep 200 + +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop8: +sleep 200 + +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791223000 and ts <= 1648791223001; + +$loop_count = 0 + +loop11: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 6 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop11 +endi + +sleep 200 + +sql delete from st where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop12: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 1 then + print =====rows=$rows + goto loop12 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop12 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop12 +endi + +sql insert into t1 values(1648791213004,3,2,3,1.0); +sql insert into t1 values(1648791213005,3,2,3,1.0); +sql insert into t1 values(1648791213006,3,2,3,1.0); +sql insert into t1 values(1648791223004,1,2,3,1.0); +sql insert into t2 values(1648791213004,3,2,3,1.0); +sql insert into t2 values(1648791213005,3,2,3,1.0); +sql insert into t2 values(1648791213006,3,2,3,1.0); +sql insert into t2 values(1648791223004,1,2,3,1.0); + +sleep 200 + +sql delete from t2 where ts >= 1648791213004 and ts <= 1648791213006; + +$loop_count = 0 + +loop13: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 3 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t1 values(1648791223005,1,2,3,1.0); +sql insert into t1 values(1648791223006,1,2,3,1.0); +sql insert into t2 values(1648791223005,1,2,3,1.0); +sql insert into t2 values(1648791223006,1,2,3,1.0); + +sql insert into t1 values(1648791233005,4,2,3,1.0); +sql insert into t1 values(1648791233006,2,2,3,1.0); +sql insert into t2 values(1648791233005,5,2,3,1.0); +sql insert into t2 values(1648791233006,3,2,3,1.0); + +sleep 200 + +sql delete from st where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop14: +sleep 200 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop14 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop14 +endi + +if $data12 != 3 then + print =====data12=$data12 + goto loop14 +endi + +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop database if exists test3; +sql drop database if exists test; +sql create database test3 vgroups 4; +sql create database test vgroups 1; +sql use test3; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams3 trigger at_once into test.streamt3 as select _wstart c1, count(*) c2, max(a) c3 from st session(ts,5s); + +sql insert into t1 values(1648791210000,1,1,1,NULL); +sql insert into t1 values(1648791210001,2,2,2,NULL); +sql insert into t2 values(1648791213001,3,3,3,NULL); +sql insert into t2 values(1648791213003,4,4,4,NULL); +sql insert into t1 values(1648791216000,5,5,5,NULL); +sql insert into t1 values(1648791216002,6,6,6,NULL); +sql insert into t1 values(1648791216004,7,7,7,NULL); +sql insert into t2 values(1648791218001,8,8,8,NULL); +sql insert into t2 values(1648791218003,9,9,9,NULL); +sql insert into t1 values(1648791222000,10,10,10,NULL); +sql insert into t1 values(1648791222003,11,11,11,NULL); +sql insert into t1 values(1648791222005,12,12,12,NULL); + +sql insert into t1 values(1648791232005,13,13,13,NULL); +sql insert into t2 values(1648791242005,14,14,14,NULL); + +$loop_count = 0 + +loop19: +sleep 200 +sql select * from test.streamt3 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop19 +endi + +sql delete from t2 where ts >= 1648791213001 and ts <= 1648791218003; + +$loop_count = 0 + +loop20: +sleep 200 +sql select * from test.streamt3 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 5 then + print =====rows=$rows + goto loop20 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop20 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop20 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop20 +endi + +if $data12 != 7 then + print =====data12=$data12 + goto loop20 +endi + +if $data21 != 3 then + print =====data21=$data21 + goto loop20 +endi + +if $data22 != 12 then + print =====data22=$data22 + goto loop20 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop20 +endi + +if $data32 != 13 then + print =====data32=$data32 + goto loop20 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop20 +endi + +if $data42 != 14 then + print =====data42=$data42 + goto loop20 +endi + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/deleteState.sim b/tests/script/tsim/stream/deleteState.sim new file mode 100644 index 0000000000000000000000000000000000000000..ecd9f55340edbc79265255848f5240f0c02fd737 --- /dev/null +++ b/tests/script/tsim/stream/deleteState.sim @@ -0,0 +1,198 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 200 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(b) c3 from t1 state_window(a); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sleep 200 +sql delete from t1 where ts = 1648791213000; + +$loop_count = 0 + +loop0: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 0 then + print =====rows=$rows + goto loop0 +endi + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop1: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213001,1,2,2,2.0); +sql insert into t1 values(1648791213002,1,3,3,3.0); +sql insert into t1 values(1648791213003,1,4,4,4.0); + +sleep 200 +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791213002; + +$loop_count = 0 + +loop3: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop3 +endi + +sql insert into t1 values(1648791223000,2,2,3,1.0); +sql insert into t1 values(1648791223001,2,2,3,1.0); +sql insert into t1 values(1648791223002,2,2,3,1.0); +sql insert into t1 values(1648791223003,2,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop4 +endi + +sleep 200 + +sql delete from t1 where ts >= 1648791223000 and ts <= 1648791223003; + +$loop_count = 0 + +loop5: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +if $data02 != 4 then + print =====data02=$data02 + goto loop5 +endi + +sql insert into t1 values(1648791213000,1,1,1,1.0); +sql insert into t1 values(1648791213005,1,2,2,2.0); +sql insert into t1 values(1648791213006,1,3,3,3.0); +sql insert into t1 values(1648791213007,1,4,4,4.0); + +sql insert into t1 values(1648791223000,2,1,1,1.0); +sql insert into t1 values(1648791223001,2,2,2,2.0); +sql insert into t1 values(1648791223002,2,3,3,3.0); +sql insert into t1 values(1648791223003,2,4,4,4.0); + +sql insert into t1 values(1648791233000,3,1,1,1.0); +sql insert into t1 values(1648791233001,3,2,2,2.0); +sql insert into t1 values(1648791233008,3,3,3,3.0); +sql insert into t1 values(1648791233009,3,4,4,4.0); + +sql delete from t1 where ts >= 1648791213001 and ts <= 1648791233005; + +$loop_count = 0 + +loop6: +sleep 200 +sql select * from streamt order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop6 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 4 then + print =====data12=$data12 + goto loop6 +endi + + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/distributeInterval0.sim b/tests/script/tsim/stream/distributeInterval0.sim index b6b427343ed8c5c03367aef6b7edc2cd5495d469..9b2e94055672d47ce4d9f0bd24663f7975d824c0 100644 --- a/tests/script/tsim/stream/distributeInterval0.sim +++ b/tests/script/tsim/stream/distributeInterval0.sim @@ -198,7 +198,7 @@ endi sql select _wstart, count(*) c1, count(d) c2 , sum(a) c3 , max(b) c4, min(c) c5, avg(d) from st interval(10s); -sql create database test1 vgroups 1; +sql create database test1 vgroups 4; sql use test1; sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); sql create table ts1 using st tags(1,1,1); @@ -232,4 +232,43 @@ if $data11 != 2 then goto loop2 endi +#max,min selectivity +sql create database test3 vgroups 4; +sql use test3; +sql create stable st(ts timestamp, a int, b int , c int) tags(ta int,tb int,tc int); +sql create table ts1 using st tags(1,1,1); +sql create table ts2 using st tags(2,2,2); +sql create stream stream_t3 trigger at_once into streamtST3 as select ts, min(a) c6, a, b, c, ta, tb, tc from st interval(10s) ; + +sql insert into ts1 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts1 values(1648791222001,2,2,3); +sleep 50 +sql insert into ts2 values(1648791211000,1,2,3); +sleep 50 +sql insert into ts2 values(1648791222001,2,2,3); +sleep 50 + +$loop_count = 0 +loop3: +sql select * from streamtST3; + +sleep 300 +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +# row 0 +if $data02 != 1 then + print =====data02=$data02 + goto loop3 +endi + +# row 1 +if $data12 != 2 then + print =====data12=$data12 + goto loop3 +endi + system sh/stop_dnodes.sh diff --git a/tests/script/tsim/stream/partitionbyColumnInterval.sim b/tests/script/tsim/stream/partitionbyColumnInterval.sim new file mode 100644 index 0000000000000000000000000000000000000000..24fdb9c99445864b01e95b21aa2db4c103054223 --- /dev/null +++ b/tests/script/tsim/stream/partitionbyColumnInterval.sim @@ -0,0 +1,570 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop0: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop0 +endi + + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop1: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 + +loop2: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop2 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.0); +sql insert into t1 values(1648791213002,2,2,3,1.0); +sql insert into t1 values(1648791213002,1,2,3,1.0); + +$loop_count = 0 + +loop3: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop3 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop4 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop4 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop4 +endi + +if $data22 != 1 then + print =====data22=$data22 + goto loop4 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop4 +endi + +if $data32 != 2 then + print =====data32=$data32 + goto loop4 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop4 +endi + +if $data42 != 3 then + print =====data42=$data42 + goto loop4 +endi + +sql drop stream if exists streams1; +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,1,2,1,1.0); +sql insert into t1 values(1648791213001,2,1,2,2.0); +sql insert into t1 values(1648791213001,1,2,3,2.0); + +$loop_count = 0 + +loop5: +sleep 50 +sql select * from streamt1 order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +sql insert into t1 values(1648791223000,1,2,4,2.0); +sql insert into t1 values(1648791223001,1,2,5,2.0); +sql insert into t1 values(1648791223002,1,2,5,2.0); +sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0); + +$loop_count = 0 + +loop6: +sleep 50 +sql select * from streamt1 order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 6 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop6 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop6 +endi + +if $data22 != 7 then + print =====data22=$data22 + goto loop6 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop6 +endi + +if $data32 != 5 then + print =====data32=$data32 + goto loop6 +endi + +sql drop stream if exists streams2; +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop7 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop7 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t2 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop8: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 + +loop9: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop9 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop9 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.0); +sql insert into t1 values(1648791213002,2,2,3,1.0); +sql insert into t1 values(1648791213002,1,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213001,2,2,3,1.0); +sql insert into t2 values(1648791213002,2,2,3,1.0); +sql insert into t2 values(1648791213002,1,2,3,1.0); + +$loop_count = 0 + +loop10: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop10 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop10 +endi + +if $data11 != 4 thenloop4 + print =====data11=$data11 + goto loop10 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop10 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); + +$loop_count = 0 + +loop11: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 4 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop11 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop11 +endi + +if $data22 != 2 then + print =====data22=$data22 + goto loop11 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop11 +endi + +if $data32 != 3 then + print =====data32=$data32 + goto loop11 +endi + +if $data41 != 4 then + print =====data41=$data41 + goto loop11 +endi + +if $data42 != 1 then + print =====data42=$data42 + goto loop11 +endi + +sql drop stream if exists streams4; +sql drop database if exists test4; +sql create database test4 vgroups 4; +sql use test4; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(2,2,2); +sql create table t4 using st tags(2,2,2); +sql create stream streams4 trigger at_once into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a interval(10s); + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); +sql insert into t3 values(1648791213000,2,2,3,1.0); +sql insert into t4 values(1648791213000,2,2,3,1.0); +sql insert into t4 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop13: +sleep 50 +sql select * from test.streamt4 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t4 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791233000,2,2,3,1.0); + + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop14: +sleep 50 +sql select * from test.streamt4 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop14 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop14 +endi + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +system sh/stop_dnodes.sh + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionbyColumnSession.sim b/tests/script/tsim/stream/partitionbyColumnSession.sim new file mode 100644 index 0000000000000000000000000000000000000000..1742d52cf03f9a6d2ec159295495cebfbb39aefc --- /dev/null +++ b/tests/script/tsim/stream/partitionbyColumnSession.sim @@ -0,0 +1,567 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql drop stream if exists streams0; +sql drop stream if exists streams1; +sql drop stream if exists streams2; +sql drop stream if exists streams3; +sql drop stream if exists streams4; +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a session(ts, 5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop0: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop0 +endi + + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop1: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 + +loop2: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop2 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.0); +sql insert into t1 values(1648791213002,2,2,3,1.0); +sql insert into t1 values(1648791213002,1,2,3,1.0); + +$loop_count = 0 + +loop3: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop3 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 50 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop4 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop4 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop4 +endi + +if $data22 != 1 then + print =====data22=$data22 + goto loop4 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop4 +endi + +if $data32 != 2 then + print =====data32=$data32 + goto loop4 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop4 +endi + +if $data42 != 3 then + print =====data42=$data42 + goto loop4 +endi + +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(c) c3, _group_key(a+b) c4 from t1 partition by a+b session(ts, 5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,1,2,1,1.0); +sql insert into t1 values(1648791213001,2,1,2,2.0); +sql insert into t1 values(1648791213001,1,2,3,2.0); + +$loop_count = 0 + +loop5: +sleep 50 +sql select * from streamt1 order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +sql insert into t1 values(1648791223000,1,2,4,2.0); +sql insert into t1 values(1648791223001,1,2,5,2.0); +sql insert into t1 values(1648791223002,1,2,5,2.0); +sql insert into t1 values(1648791213001,1,1,6,2.0) (1648791223002,1,1,7,2.0); + +$loop_count = 0 + +loop6: +sleep 50 +sql select * from streamt1 order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 6 then + print =====data12=$data12 + goto loop6 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop6 +endi + +if $data22 != 5 then + print =====data22=$data22 + goto loop6 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop6 +endi + +if $data32 != 7 then + print =====data32=$data32 + goto loop6 +endi + +sql drop database if exists test2; +sql create database test2 vgroups 4; +sql use test2; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create stream streams2 trigger at_once into test.streamt2 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t2 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop7: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop7 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop7 +endi + +sql insert into t1 values(1648791213000,1,2,3,1.0); +sql insert into t2 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop8: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop8 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop8 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); + +$loop_count = 0 + +loop9: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop9 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop9 +endi + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791213001,2,2,3,1.0); +sql insert into t1 values(1648791213002,2,2,3,1.0); +sql insert into t1 values(1648791213002,1,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213001,2,2,3,1.0); +sql insert into t2 values(1648791213002,2,2,3,1.0); +sql insert into t2 values(1648791213002,1,2,3,1.0); + +$loop_count = 0 + +loop10: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 4 then + print =====data01=$data01 + goto loop10 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop10 +endi + +if $data11 != 2 thenloop4 + print =====data11=$data11 + goto loop10 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop10 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t1 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); +sql insert into t2 values(1648791223000,1,2,3,1.0); +sql insert into t2 values(1648791223001,1,2,3,1.0); +sql insert into t2 values(1648791223002,3,2,3,1.0); +sql insert into t2 values(1648791223003,3,2,3,1.0); +sql insert into t2 values(1648791213001,1,2,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); + +$loop_count = 0 + +loop11: +sleep 50 +sql select * from test.streamt2 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop11 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop11 +endi + +if $data11 != 4 then + print =====data11=$data11 + goto loop11 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop11 +endi + +if $data21 != 4 then + print =====data21=$data21 + goto loop11 +endi + +if $data22 != 1 then + print =====data22=$data22 + goto loop11 +endi + +if $data31 != 2 then + print =====data31=$data31 + goto loop11 +endi + +if $data32 != 2 then + print =====data32=$data32 + goto loop11 +endi + +if $data41 != 2 then + print =====data41=$data41 + goto loop11 +endi + +if $data42 != 3 then + print =====data42=$data42 + goto loop11 +endi + +sql drop database if exists test4; +sql create database test4 vgroups 4; +sql use test4; +sql create stable st(ts timestamp, a int, b int, c int, d double) tags(ta int,tb int,tc int); +sql create table t1 using st tags(1,1,1); +sql create table t2 using st tags(2,2,2); +sql create table t3 using st tags(2,2,2); +sql create table t4 using st tags(2,2,2); +sql create stream streams4 trigger at_once into test.streamt4 as select _wstart c1, count(*) c2, max(a) c3 from st partition by a session(ts, 5s); + +sql insert into t1 values(1648791213000,2,2,3,1.0); +sql insert into t2 values(1648791213000,2,2,3,1.0); +sql insert into t3 values(1648791213000,2,2,3,1.0); +sql insert into t4 values(1648791213000,2,2,3,1.0); +sql insert into t4 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop13: +sleep 50 +sql select * from test.streamt4 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 2 then + print =====rows=$rows + goto loop13 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop13 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop13 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop13 +endi + +if $data12 != 2 then + print =====data12=$data12 + goto loop13 +endi + +sql insert into t4 values(1648791213000,2,2,3,1.0); +sql insert into t1 values(1648791233000,2,2,3,1.0); + + +sql insert into t1 values(1648791213000,1,2,3,1.0); + +$loop_count = 0 + +loop14: +sleep 50 +sql select * from test.streamt4 order by c1, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 20 then + return -1 +endi + +if $rows != 3 then + print =====rows=$rows + goto loop14 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop14 +endi + +if $data11 != 3 then + print =====data11=$data11 + goto loop14 +endi + +if $data21 != 1 then + print =====data21=$data21 + goto loop14 +endi + +system sh/stop_dnodes.sh + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +#goto looptest \ No newline at end of file diff --git a/tests/script/tsim/stream/partitionbyColumnState.sim b/tests/script/tsim/stream/partitionbyColumnState.sim new file mode 100644 index 0000000000000000000000000000000000000000..75d01b17ec6495be9644ce7e7db401a0147b3a52 --- /dev/null +++ b/tests/script/tsim/stream/partitionbyColumnState.sim @@ -0,0 +1,279 @@ +$loop_all = 0 +looptest: + +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sleep 50 +sql connect + +sql drop database if exists test; +sql create database test vgroups 1; +sql use test; +sql create table t1(ts timestamp, a int, b int , c int, d double); +sql create stream streams0 trigger at_once into streamt as select _wstart c1, count(*) c2, max(a) c3, _group_key(a) c4 from t1 partition by a state_window(b); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); + +$loop_count = 0 + +loop0: +sleep 300 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop0 +endi + +if $data02 != NULL then + print =====data02=$data02 + goto loop0 +endi + + +sql insert into t1 values(1648791213000,1,1,3,1.0); + +$loop_count = 0 + +loop1: +sleep 300 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop1 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop1 +endi + +sql insert into t1 values(1648791213000,2,1,3,1.0); + +$loop_count = 0 + +loop2: +sleep 300 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop2 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop2 +endi + +sql insert into t1 values(1648791213000,2,1,3,1.0); +sql insert into t1 values(1648791213001,2,1,3,1.0); +sql insert into t1 values(1648791213002,2,1,3,1.0); +sql insert into t1 values(1648791213002,1,1,3,1.0); + +$loop_count = 0 + +loop3: +sleep 300 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop3 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop3 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop3 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop3 +endi + +sql insert into t1 values(1648791223000,1,2,3,1.0); +sql insert into t1 values(1648791223001,1,2,3,1.0); +sql insert into t1 values(1648791223002,3,2,3,1.0); +sql insert into t1 values(1648791223003,3,2,3,1.0); +sql insert into t1 values(1648791213001,1,1,3,1.0) (1648791223001,2,2,3,1.0) (1648791223003,1,2,3,1.0); + +$loop_count = 0 + +loop4: +sleep 300 +sql select * from streamt order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop4 +endi + +if $data02 != 2 then + print =====data02=$data02 + goto loop4 +endi + +if $data11 != 2 then + print =====data11=$data11 + goto loop4 +endi + +if $data12 != 1 then + print =====data12=$data12 + goto loop4 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop4 +endi + +if $data22 != 1 then + print =====data22=$data22 + goto loop4 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop4 +endi + +if $data32 != 2 then + print =====data32=$data32 + goto loop4 +endi + +if $data41 != 1 then + print =====data41=$data41 + goto loop4 +endi + +if $data42 != 3 then + print =====data42=$data42 + goto loop4 +endi + +sql drop database if exists test1; +sql create database test1 vgroups 1; +sql use test1; +sql create table t1(ts timestamp, a int, b int , c int, d int); +sql create stream streams1 trigger at_once into streamt1 as select _wstart c1, count(*) c2, max(d) c3, _group_key(a+b) c4 from t1 partition by a+b state_window(c); + +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,NULL,NULL,NULL,NULL); +sql insert into t1 values(1648791213000,1,2,1,1); +sql insert into t1 values(1648791213001,2,1,1,2); +sql insert into t1 values(1648791213001,1,2,1,3); + +$loop_count = 0 + +loop5: +sleep 300 +sql select * from streamt1 order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 2 then + print =====data01=$data01 + goto loop5 +endi + +sql insert into t1 values(1648791223000,1,2,2,4); +sql insert into t1 values(1648791223001,1,2,2,5); +sql insert into t1 values(1648791223002,1,2,2,6); +sql insert into t1 values(1648791213001,1,1,1,7) (1648791223002,1,1,2,8); + +$loop_count = 0 + +loop6: +sleep 300 +sql select * from streamt1 order by c1, c4, c2, c3; + +$loop_count = $loop_count + 1 +if $loop_count == 10 then + return -1 +endi + +if $data01 != 1 then + print =====data01=$data01 + goto loop6 +endi + +if $data02 != 1 then + print =====data02=$data02 + goto loop6 +endi + +if $data11 != 1 then + print =====data11=$data11 + goto loop6 +endi + +if $data12 != 7 then + print =====data12=$data12 + goto loop6 +endi + +if $data21 != 2 then + print =====data21=$data21 + goto loop6 +endi + +if $data22 != 5 then + print =====data22=$data22 + goto loop6 +endi + +if $data31 != 1 then + print =====data31=$data31 + goto loop6 +endi + +if $data32 != 8 then + print =====data32=$data32 + goto loop6 +endi + +system sh/stop_dnodes.sh + +$loop_all = $loop_all + 1 +print ============loop_all=$loop_all + +#goto looptest diff --git a/tests/script/tsim/table/hash.sim b/tests/script/tsim/table/hash.sim new file mode 100644 index 0000000000000000000000000000000000000000..664f86713720e1ae1969027508e2931d23397f08 --- /dev/null +++ b/tests/script/tsim/table/hash.sim @@ -0,0 +1,84 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +#=========== prepare +#sql create database d1 vgroups 2 +sql create database d1 vgroups 2 table_prefix 3 table_suffix 2 +sql select * from information_schema.ins_databases +print $data(d1)[27] $data(d1)[28] +if $data(d1)[27] != 3 then + return -1 +endi +if $data(d1)[28] != 2 then + return -1 +endi + +sql use d1; +sql create table st (ts timestamp, i int) tags (j int); +sql create table st_ct_1 using st tags(3) st_ct_2 using st tags(4) st_ct_3 using st tags(5) st_ct_4 using st tags(6) st_ct_5 using st tags(7) +sql insert into st_ct_1 values(now+1s, 1) +sql insert into st_ct_1 values(now+2s, 2) +sql insert into st_ct_1 values(now+3s, 3) +sql insert into st_ct_2 values(now+1s, 1) +sql insert into st_ct_2 values(now+2s, 2) +sql insert into st_ct_2 values(now+3s, 3) +sql insert into st_ct_3 values(now+1s, 1) +sql insert into st_ct_3 values(now+2s, 2) +sql insert into st_ct_3 values(now+3s, 2) +sql insert into st_ct_4 values(now+1s, 1) +sql insert into st_ct_4 values(now+2s, 2) +sql insert into st_ct_4 values(now+3s, 2) +sql insert into st_ct_5 values(now+1s, 1) +sql insert into st_ct_5 values(now+2s, 2) +sql insert into st_ct_5 values(now+3s, 2) + +# check query +sql select * from st +if $rows != 15 then + return -1 +endi + +# check table vgroup +sql select * from information_schema.ins_tables where db_name = 'd1' +if $data(st_ct_1)[6] != 2 then + return -1 +endi +if $data(st_ct_2)[6] != 2 then + return -1 +endi +if $data(st_ct_3)[6] != 2 then + return -1 +endi +if $data(st_ct_4)[6] != 2 then + return -1 +endi +if $data(st_ct_5)[6] != 2 then + return -1 +endi + +# check invalid table name +sql create table c1 using st tags(3) +sql create table c12 using st tags(3) +sql create table c123 using st tags(3) +sql create table c1234 using st tags(3) +sql create table c12345 using st tags(3) +sql select * from information_schema.ins_tables where db_name = 'd1' +if $data(c1)[6] != 2 then + return -1 +endi +if $data(c12)[6] != 3 then + return -1 +endi +if $data(c123)[6] != 2 then + return -1 +endi +if $data(c1234)[6] != 3 then + return -1 +endi +if $data(c12345)[6] != 3 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/valgrind/checkError6.sim b/tests/script/tsim/valgrind/checkError6.sim index d85a1bebc898ca79a20e4c495081077a2b1a4249..11a387ed4dffd7258120154adb9d7874ee0b21b5 100644 --- a/tests/script/tsim/valgrind/checkError6.sim +++ b/tests/script/tsim/valgrind/checkError6.sim @@ -158,6 +158,8 @@ print =============== restart system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start -v +sleep 1000 + sql select avg(tbcol) as c from stb sql select avg(tbcol) as c from stb where ts <= 1601481840000 sql select avg(tbcol) as c from stb where tgcol < 5 and ts <= 1601481840000 diff --git a/tests/system-test/0-others/taosShellNetChk.py b/tests/system-test/0-others/taosShellNetChk.py index dd44852d4902e7d444e9aebc52fa3b4bb186fc08..80f6a6bc3007bdb2f5a2dfaca2b0fe71d9ea2042 100644 --- a/tests/system-test/0-others/taosShellNetChk.py +++ b/tests/system-test/0-others/taosShellNetChk.py @@ -231,7 +231,7 @@ class TDTestCase: if platform.system().lower() == 'windows': os.system('ps -a | grep taos | awk \'{print $2}\' | xargs kill -9') else: - os.system('pkill taos') + os.system('pkill -9 taos') def stop(self): tdSql.close() diff --git a/tests/system-test/1-insert/delete_data.py b/tests/system-test/1-insert/delete_data.py index 068d212ac4e49d00fe7ff938118352dbe462a988..f620a4b18a0d7ddf5a0d65c902d0411c270e6463 100644 --- a/tests/system-test/1-insert/delete_data.py +++ b/tests/system-test/1-insert/delete_data.py @@ -81,6 +81,7 @@ class TDTestCase: 'binary':self.binary_val, 'nchar':self.nchar_val } + def insert_base_data(self,col_type,tbname,rows,base_data): for i in range(rows): if col_type.lower() == 'tinyint': @@ -290,6 +291,9 @@ class TDTestCase: self.delete_data_ntb() self.delete_data_ctb() self.delete_data_stb() + tdDnodes.stoptaosd(1) + tdDnodes.starttaosd(1) + self.delete_data_ntb() def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/1-insert/mutil_stage.py b/tests/system-test/1-insert/mutil_stage.py index 764da1f166cb9bf149002aae27fc6d7c078be446..63317e8036b6455984231bf634b23f7238935642 100644 --- a/tests/system-test/1-insert/mutil_stage.py +++ b/tests/system-test/1-insert/mutil_stage.py @@ -1,4 +1,5 @@ from datetime import datetime +from platform import platform import time from typing import List, Any, Tuple @@ -83,6 +84,8 @@ class TDTestCase: def del_old_datadir(self, filename): cmd = f"sed -i '/^dataDir/d' {filename}" + if platform.system().lower() == 'darwin': + cmd = f"sed -i '' '/^dataDir/d' {filename}" if os.system(cmd) != 0: tdLog.exit(cmd) diff --git a/tests/system-test/1-insert/update_data.py b/tests/system-test/1-insert/update_data.py index deff4b42a1784b5bc59466d1c067536e83c10d06..417adc39acc306fbd7b2d74518653d0347fb27cb 100644 --- a/tests/system-test/1-insert/update_data.py +++ b/tests/system-test/1-insert/update_data.py @@ -154,10 +154,12 @@ class TDTestCase: up_bool = random.randint(0,100)%2 up_float = random.uniform(constant.FLOAT_MIN,constant.FLOAT_MAX) up_double = random.uniform(constant.DOUBLE_MIN*(1E-300),constant.DOUBLE_MAX*(1E-300)) - binary_length = random.randint(0,self.str_length) - nchar_length = random.randint(0,self.str_length) - up_binary = tdCom.getLongName(binary_length) - up_nchar = tdCom.getLongName(nchar_length) + binary_length = [] + for i in range(self.str_length+1): + binary_length.append(i) + nchar_length = [] + for i in range(self.str_length+1): + nchar_length.append(i) for col_name,col_type in column_dict.items(): if tb_type == 'ntb': tdSql.execute(f'create table {tbname} (ts timestamp,{col_name} {col_type})') @@ -188,9 +190,13 @@ class TDTestCase: elif col_type.lower() == 'double': self.update_and_check_data(tbname,col_name,col_type,up_double,dbname) elif 'binary' in col_type.lower(): - self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) + for i in binary_length: + up_binary = tdCom.getLongName(i) + self.update_and_check_data(tbname,col_name,col_type,up_binary,dbname) elif 'nchar' in col_type.lower(): - self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) + for i in nchar_length: + up_nchar = tdCom.getLongName(i) + self.update_and_check_data(tbname,col_name,col_type,up_nchar,dbname) elif col_type.lower() == 'timestamp': self.update_and_check_data(tbname,col_name,col_type,self.ts+1,dbname) tdSql.execute(f'insert into {tbname} values({self.ts},null)') diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index 169b1c2c387c7158635483f8ce8868891e42e3c2..5cc9a2d9e888425763914c01baeababfeebc6166 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -38,18 +38,7 @@ class TDTestCase: elif i>=9: tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query(f"select max(now()) from {dbname}.stb_1") - tdSql.checkRows(1) - - tdSql.query(f"select last(ts) from {dbname}.stb_1") - lastTs = tdSql.getData(0, 0) - tdSql.query(f"select max(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, lastTs) - - tdSql.query(f"select last(ts) from {dbname}.stb") - lastTs = tdSql.getData(0, 0) - tdSql.query(f"select max(ts) from {dbname}.stb") - tdSql.checkData(0, 0, lastTs) + tdSql.error(f"select max(now()) from {dbname}.stb_1") tdSql.query(f"select max(col1) from {dbname}.stb_1 where col2<=5") tdSql.checkData(0,0,5) @@ -78,13 +67,7 @@ class TDTestCase: elif i>=9: tdSql.checkData(0, 0, np.max(floatData)) - tdSql.query(f"select max(now()) from {dbname}.ntb") - tdSql.checkRows(1) - - tdSql.query(f"select last(ts) from {dbname}.ntb") - lastTs = tdSql.getData(0, 0) - tdSql.query(f"select max(ts) from {dbname}.ntb") - tdSql.checkData(0, 0, lastTs) + tdSql.error(f"select max(now()) from {dbname}.ntb") tdSql.query(f"select max(col1) from {dbname}.ntb where col2<=5") tdSql.checkData(0,0,5) diff --git a/tests/system-test/2-query/max_partition.py b/tests/system-test/2-query/max_partition.py index 01c267724210591e639753c3566c4826a5218813..08bb7675add128f94f6ab5be0b9a7a8c90e49afe 100644 --- a/tests/system-test/2-query/max_partition.py +++ b/tests/system-test/2-query/max_partition.py @@ -181,7 +181,7 @@ class TDTestCase: # bug need fix tdSql.checkData(0,1,None) - tdSql.query(f"select c1 , twa(c1) from {dbname}.stb partition by c1 order by c1") + tdSql.query(f"select c1 , twa(c1) from {dbname}.sub_stb_1 partition by c1 order by c1") tdSql.checkRows(11) tdSql.checkData(0,1,None) diff --git a/tests/system-test/2-query/min.py b/tests/system-test/2-query/min.py index 3d46b7b2224f834360c17cdc311dbf1e0d5a4535..d97c4340f4022af6372134892dba25c371af1b51 100644 --- a/tests/system-test/2-query/min.py +++ b/tests/system-test/2-query/min.py @@ -37,6 +37,8 @@ class TDTestCase: floatData.append(i + 0.1) # max verifacation + tdSql.error(f"select min(now()) from {dbname}.stb_1") + tdSql.error(f"select min(ts) from {dbname}.stb_1") tdSql.error(f"select min(col7) from {dbname}.stb_1") tdSql.error(f"select min(col8) from {dbname}.stb_1") tdSql.error(f"select min(col9) from {dbname}.stb_1") @@ -67,20 +69,9 @@ class TDTestCase: tdSql.query(f"select min(col1) from {dbname}.stb_1 where col2>=5") tdSql.checkData(0,0,5) - tdSql.query(f"select min(now()) from {dbname}.stb_1") - tdSql.checkRows(1) - - tdSql.query(f"select first(ts) from {dbname}.stb_1") - firstTs = tdSql.getData(0, 0) - tdSql.query(f"select min(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, firstTs) - - tdSql.query(f"select first(ts) from {dbname}.stb_1") - firstTs = tdSql.getData(0, 0) - tdSql.query(f"select min(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, firstTs) - + tdSql.error(f"select min(now()) from {dbname}.stb_1") + tdSql.error(f"select min(ts) from {dbname}.stb_1") tdSql.error(f"select min(col7) from {dbname}.stb_1") tdSql.error(f"select min(col8) from {dbname}.stb_1") tdSql.error(f"select min(col9) from {dbname}.stb_1") @@ -111,19 +102,8 @@ class TDTestCase: tdSql.query(f"select min(col1) from {dbname}.stb where col2>=5") tdSql.checkData(0,0,5) - tdSql.query(f"select min(now()) from {dbname}.stb_1") - tdSql.checkRows(1) - - tdSql.query(f"select first(ts) from {dbname}.stb_1") - firstTs = tdSql.getData(0, 0) - tdSql.query(f"select min(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, firstTs) - - tdSql.query(f"select first(ts) from {dbname}.stb_1") - firstTs = tdSql.getData(0, 0) - tdSql.query(f"select min(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, firstTs) - + tdSql.error(f"select min(now()) from {dbname}.stb_1") + tdSql.error(f"select min(ts) from {dbname}.stb_1") tdSql.error(f"select min(col7) from {dbname}.ntb") tdSql.error(f"select min(col8) from {dbname}.ntb") tdSql.error(f"select min(col9) from {dbname}.ntb") @@ -154,19 +134,6 @@ class TDTestCase: tdSql.query(f"select min(col1) from {dbname}.ntb where col2>=5") tdSql.checkData(0,0,5) - tdSql.query(f"select min(now()) from {dbname}.stb_1") - tdSql.checkRows(1) - - tdSql.query(f"select first(ts) from {dbname}.stb_1") - firstTs = tdSql.getData(0, 0) - tdSql.query(f"select min(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, firstTs) - - tdSql.query(f"select first(ts) from {dbname}.stb_1") - firstTs = tdSql.getData(0, 0) - tdSql.query(f"select min(ts) from {dbname}.stb_1") - tdSql.checkData(0, 0, firstTs) - def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index b7e167c8b5a6b7db3421f00dcb5da8fff530c8d6..4dae2ad6c0ec289e034929e6a949eed8b665c899 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -53,7 +53,7 @@ class TDTestCase: tdSql.query(f"select * from {dbname}.t_0799064f5487946e5d22164a822acfc8 order by _ts") tdSql.checkRows(2) tdSql.checkData(0, 3, "kk") - tdSql.checkData(1, 3, None) + tdSql.checkData(1, 3, "") tdSql.query(f"select distinct tbname from {dbname}.`sys.if.bytes.out`") diff --git a/tests/system-test/2-query/ttl_comment.py b/tests/system-test/2-query/ttl_comment.py index c26393158cefe46fb054d7bd3e28a621cab73199..187df2e2d33846a2e61a7519cdec8c13b352f9da 100644 --- a/tests/system-test/2-query/ttl_comment.py +++ b/tests/system-test/2-query/ttl_comment.py @@ -137,7 +137,8 @@ class TDTestCase: tdSql.execute(f"create table {dbname}.child_table2 using {dbname}.super_table1 tags(1) comment ''") tdSql.execute(f"create table {dbname}.child_table3 using {dbname}.super_table1 tags(1) comment 'child'") tdSql.execute(f"insert into {dbname}.child_table4 using {dbname}.super_table1 tags(1) values(now, 1)") - + tdSql.execute(f"insert into {dbname}.child_table5 using {dbname}.super_table1 tags(1) ttl 23 comment '' values(now, 1)") + tdSql.error(f"insert into {dbname}.child_table6 using {dbname}.super_table1 tags(1) ttl -23 comment '' values(now, 1)") tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table1'") tdSql.checkData(0, 0, 'child_table1') @@ -160,6 +161,11 @@ class TDTestCase: tdSql.checkData(0, 7, 0) tdSql.checkData(0, 8, None) + tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table5'") + tdSql.checkData(0, 0, 'child_table5') + tdSql.checkData(0, 7, 23) + tdSql.checkData(0, 8, '') + tdSql.execute(f"alter table {dbname}.child_table1 comment 'nihao'") tdSql.query("select * from information_schema.ins_tables where table_name like 'child_table1'") diff --git a/tests/system-test/2-query/twa.py b/tests/system-test/2-query/twa.py index 62940477cf701d69e8c8e7568ae4b56d68518d81..4c163da485f2790e15fa098b707c67f7ba95360b 100644 --- a/tests/system-test/2-query/twa.py +++ b/tests/system-test/2-query/twa.py @@ -124,7 +124,7 @@ class TDTestCase: tdSql.checkData(0,1,4.500000000) # mixup with other functions - tdSql.query(f"select twa(c1),twa(c2),max(c1),elapsed(ts) from {dbname}.stb1 ") + tdSql.query(f"select twa(c1),twa(c2),max(c1),elapsed(ts) from {dbname}.ct1 ") tdSql.checkData(0,0,1.000000000) tdSql.checkData(0,1,11111.000000000) tdSql.checkData(0,2,1) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index 4305ceff56781fb4e12f725620b17053f6d4b299..0b45b2425826bd6f43942afb635e8fe548bb6a50 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -2,7 +2,7 @@ set -e set -x -python3 ./test.py -f 0-others/taosShell.py +#python3 ./test.py -f 0-others/taosShell.py python3 ./test.py -f 0-others/taosShellError.py python3 ./test.py -f 0-others/taosShellNetChk.py python3 ./test.py -f 0-others/telemetry.py @@ -156,8 +156,8 @@ python3 ./test.py -f 2-query/sin.py python3 ./test.py -f 2-query/sin.py -R python3 ./test.py -f 2-query/smaTest.py python3 ./test.py -f 2-query/smaTest.py -R -python3 ./test.py -f 2-query/sml.py -python3 ./test.py -f 2-query/sml.py -R +#python3 ./test.py -f 2-query/sml.py +#python3 ./test.py -f 2-query/sml.py -R python3 ./test.py -f 2-query/spread.py python3 ./test.py -f 2-query/spread.py -R python3 ./test.py -f 2-query/sqrt.py @@ -333,7 +333,7 @@ python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py python3 ./test.py -f 7-tmq/tmq_taosx.py -# python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py #------------querPolicy 2----------- @@ -512,6 +512,6 @@ python3 ./test.py -f 2-query/count_partition.py -Q 3 python3 ./test.py -f 2-query/max_partition.py -Q 3 python3 ./test.py -f 2-query/last_row.py -Q 3 python3 ./test.py -f 2-query/tsbsQuery.py -Q 3 -python3 ./test.py -f 2-query/sml.py -Q 3 +#python3 ./test.py -f 2-query/sml.py -Q 3 python3 ./test.py -f 2-query/interp.py -Q 3 diff --git a/utils/CMakeLists.txt b/utils/CMakeLists.txt new file mode 100644 index 0000000000000000000000000000000000000000..59cbbb31471acc377dd614610aa715ae72c9ec31 --- /dev/null +++ b/utils/CMakeLists.txt @@ -0,0 +1,4 @@ +#ADD_SUBDIRECTORY(examples/c) +ADD_SUBDIRECTORY(tsim) +ADD_SUBDIRECTORY(test/c) +#ADD_SUBDIRECTORY(comparisonTest/tdengine) diff --git a/tests/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt similarity index 97% rename from tests/test/c/CMakeLists.txt rename to utils/test/c/CMakeLists.txt index 31331b52651fee79c837a077869bc45ec7acfe6c..839c65b633b17872051075be33ce291fbeaefd5a 100644 --- a/tests/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -1,4 +1,5 @@ add_executable(tmq_demo tmqDemo.c) +add_dependencies(tmq_demo taos) add_executable(tmq_sim tmqSim.c) add_executable(create_table createTable.c) add_executable(tmq_taosx_ci tmq_taosx_ci.c) diff --git a/tests/test/c/createTable.c b/utils/test/c/createTable.c similarity index 100% rename from tests/test/c/createTable.c rename to utils/test/c/createTable.c diff --git a/tests/test/c/sdbDump.c b/utils/test/c/sdbDump.c similarity index 100% rename from tests/test/c/sdbDump.c rename to utils/test/c/sdbDump.c diff --git a/tests/test/c/sml_test.c b/utils/test/c/sml_test.c similarity index 96% rename from tests/test/c/sml_test.c rename to utils/test/c/sml_test.c index 18181d2073e154c7f3fe183d164e1f858fddcb4f..ca3d464da769a9da8c87abd2be466ac2269fabcd 100644 --- a/tests/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -63,6 +63,7 @@ int smlProcess_influx_Test() { printf("%s result:%s\n", __FUNCTION__, taos_errstr(pRes)); int code = taos_errno(pRes); taos_free_result(pRes); + return code; } @@ -91,7 +92,7 @@ int smlProcess_telnet_Test() { int smlProcess_json1_Test() { TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db"); taos_free_result(pRes); pRes = taos_query(taos, "use sml_db"); @@ -111,7 +112,7 @@ int smlProcess_json1_Test() { " }," " {" " \"metric\": \"sys.cpu.nice\"," - " \"timestamp\": 1346846400," + " \"timestamp\": 1662344042," " \"value\": 9," " \"tags\": {" " \"host\": \"web02\"," @@ -140,7 +141,7 @@ int smlProcess_json2_Test() { "{" " \"metric\": \"meter_current0\"," " \"timestamp\": {" - " \"value\" : 1346846400," + " \"value\" : 1662344042," " \"type\" : \"s\"" " }," " \"value\": {" @@ -180,7 +181,7 @@ int smlProcess_json3_Test() { "{" " \"metric\": \"meter_current1\"," " \"timestamp\": {" - " \"value\" : 1346846400," + " \"value\" : 1662344042," " \"type\" : \"s\"" " }," " \"value\": {" @@ -248,7 +249,7 @@ int smlProcess_json4_Test() { "{" " \"metric\": \"meter_current2\"," " \"timestamp\": {" - " \"value\" : 1346846500000," + " \"value\" : 1662344042000," " \"type\" : \"ms\"" " }," " \"value\": \"ni\"," @@ -1100,34 +1101,91 @@ int sml_add_tag_col_Test() { return code; } +int smlProcess_18784_Test() { + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + + TAOS_RES *pRes = taos_query(taos, "create database if not exists sml_db schemaless 1"); + taos_free_result(pRes); + + pRes = taos_query(taos, "use sml_db"); + taos_free_result(pRes); + + const char *sql[] = { + "disk,device=sdc inodes_used=176059i,total=1081101176832i 1661943960000000000", + "disk,device=sdc inodes_free=66932805i 1661943960000000000", + }; + pRes = taos_schemaless_insert(taos, (char **)sql, sizeof(sql) / sizeof(sql[0]), TSDB_SML_LINE_PROTOCOL, 0); + printf("%s result:%s, rows:%d\n", __FUNCTION__, taos_errstr(pRes), taos_affected_rows(pRes)); + int code = taos_errno(pRes); + ASSERT(!code); + ASSERT(taos_affected_rows(pRes) == 2); + taos_free_result(pRes); + + pRes = taos_query(taos, "select * from disk"); + ASSERT(pRes); + int fieldNum = taos_field_count(pRes); + ASSERT(fieldNum == 5); + printf("fieldNum:%d\n", fieldNum); + TAOS_ROW row = NULL; + int32_t rowIndex = 0; + while((row = taos_fetch_row(pRes)) != NULL) { + int64_t ts = *(int64_t*)row[0]; + int64_t used = *(int64_t*)row[1]; + int64_t total = *(int64_t*)row[2]; + int64_t freed = *(int64_t*)row[3]; + if(rowIndex == 0){ + ASSERT(ts == 1661943960000); + ASSERT(used == 176059); + ASSERT(total == 1081101176832); + ASSERT(freed == 66932805); +// ASSERT_EQ(latitude, 24.5208); +// ASSERT_EQ(longitude, 28.09377); +// ASSERT_EQ(elevation, 428); +// ASSERT_EQ(velocity, 0); +// ASSERT_EQ(heading, 304); +// ASSERT_EQ(grade, 0); +// ASSERT_EQ(fuel_consumption, 25); + }else{ +// ASSERT(0); + } + rowIndex++; + } + taos_free_result(pRes); + + return code; +} + int main(int argc, char *argv[]) { int ret = 0; ret = smlProcess_influx_Test(); - if(ret) return ret; + ASSERT(!ret); ret = smlProcess_telnet_Test(); - if(ret) return ret; + ASSERT(!ret); ret = smlProcess_json1_Test(); - if(ret) return ret; + ASSERT(!ret); ret = smlProcess_json2_Test(); - if(ret) return ret; + ASSERT(!ret); ret = smlProcess_json3_Test(); - if(ret) return ret; + ASSERT(!ret); ret = smlProcess_json4_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_TD15662_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_TD15742_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_16384_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_oom_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_16368_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_dup_time_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_16960_Test(); - if(ret) return ret; + ASSERT(!ret); ret = sml_add_tag_col_Test(); + ASSERT(!ret); + ret = smlProcess_18784_Test(); + ASSERT(!ret); return ret; } diff --git a/tests/test/c/tmqDemo.c b/utils/test/c/tmqDemo.c similarity index 100% rename from tests/test/c/tmqDemo.c rename to utils/test/c/tmqDemo.c diff --git a/tests/test/c/tmqSim.c b/utils/test/c/tmqSim.c similarity index 100% rename from tests/test/c/tmqSim.c rename to utils/test/c/tmqSim.c diff --git a/tests/test/c/tmq_taosx_ci.c b/utils/test/c/tmq_taosx_ci.c similarity index 100% rename from tests/test/c/tmq_taosx_ci.c rename to utils/test/c/tmq_taosx_ci.c diff --git a/tests/tsim/CMakeLists.txt b/utils/tsim/CMakeLists.txt similarity index 100% rename from tests/tsim/CMakeLists.txt rename to utils/tsim/CMakeLists.txt diff --git a/tests/tsim/inc/simInt.h b/utils/tsim/inc/simInt.h similarity index 100% rename from tests/tsim/inc/simInt.h rename to utils/tsim/inc/simInt.h diff --git a/tests/tsim/inc/simParse.h b/utils/tsim/inc/simParse.h similarity index 100% rename from tests/tsim/inc/simParse.h rename to utils/tsim/inc/simParse.h diff --git a/tests/tsim/src/simExe.c b/utils/tsim/src/simExe.c similarity index 100% rename from tests/tsim/src/simExe.c rename to utils/tsim/src/simExe.c diff --git a/tests/tsim/src/simMain.c b/utils/tsim/src/simMain.c similarity index 100% rename from tests/tsim/src/simMain.c rename to utils/tsim/src/simMain.c diff --git a/tests/tsim/src/simParse.c b/utils/tsim/src/simParse.c similarity index 100% rename from tests/tsim/src/simParse.c rename to utils/tsim/src/simParse.c diff --git a/tests/tsim/src/simSystem.c b/utils/tsim/src/simSystem.c similarity index 100% rename from tests/tsim/src/simSystem.c rename to utils/tsim/src/simSystem.c