diff --git a/cmake/cmake.platform b/cmake/cmake.platform index acf17e9427bc453e1ece67cca5cbfe45f8827337..849d31f93e9eaa0f9c834034ee277392201df0cf 100644 --- a/cmake/cmake.platform +++ b/cmake/cmake.platform @@ -70,5 +70,46 @@ ELSEIF (${CMAKE_SYSTEM_NAME} MATCHES "Windows") ENDIF() +IF ("${CPUTYPE}" STREQUAL "") + MESSAGE(STATUS "The current platform " ${CMAKE_SYSTEM_PROCESSOR} " is detected") + + IF (CMAKE_SYSTEM_PROCESSOR MATCHES "(amd64)|(AMD64)") + MESSAGE(STATUS "The current platform is amd64") + SET(PLATFORM_ARCH_STR "amd64") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "(x86)|(X86)") + MESSAGE(STATUS "The current platform is x86") + SET(PLATFORM_ARCH_STR "i386") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "armv7l") + MESSAGE(STATUS "The current platform is aarch32") + SET(PLATFORM_ARCH_STR "arm") + ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "aarch64") + MESSAGE(STATUS "The current platform is aarch64") + SET(PLATFORM_ARCH_STR "arm64") + ENDIF () +ELSE () + # if generate ARM version: + # cmake -DCPUTYPE=aarch32 .. or cmake -DCPUTYPE=aarch64 + IF (${CPUTYPE} MATCHES "aarch32") + SET(PLATFORM_ARCH_STR "arm") + MESSAGE(STATUS "input cpuType: aarch32") + ELSEIF (${CPUTYPE} MATCHES "aarch64") + SET(PLATFORM_ARCH_STR "arm64") + MESSAGE(STATUS "input cpuType: aarch64") + ELSEIF (${CPUTYPE} MATCHES "mips64") + SET(PLATFORM_ARCH_STR "mips") + MESSAGE(STATUS "input cpuType: mips64") + ELSEIF (${CPUTYPE} MATCHES "x64") + SET(PLATFORM_ARCH_STR "amd64") + MESSAGE(STATUS "input cpuType: x64") + ELSEIF (${CPUTYPE} MATCHES "x86") + SET(PLATFORM_ARCH_STR "i386") + MESSAGE(STATUS "input cpuType: x86") + ELSE () + MESSAGE(STATUS "input cpuType unknown " ${CPUTYPE}) + ENDIF () +ENDIF () + +MESSAGE(STATUS "platform arch:" ${PLATFORM_ARCH_STR}) + MESSAGE("C Compiler ID: ${CMAKE_C_COMPILER_ID}") MESSAGE("CXX Compiler ID: ${CMAKE_CXX_COMPILER_ID}") diff --git a/docs/en/10-cluster/02-cluster-mgmt.md b/docs/en/10-cluster/02-cluster-mgmt.md index 674c92e2766a4eb304079140af19c8efea72d55e..bd3386c41161fc55b4bedcecd6ad3ab5c35be8b6 100644 --- a/docs/en/10-cluster/02-cluster-mgmt.md +++ b/docs/en/10-cluster/02-cluster-mgmt.md @@ -54,14 +54,14 @@ Database changed. taos> show vgroups; vgId | tables | status | onlines | v1_dnode | v1_status | compacting | ========================================================================================== - 14 | 38000 | ready | 1 | 1 | master | 0 | - 15 | 38000 | ready | 1 | 1 | master | 0 | - 16 | 38000 | ready | 1 | 1 | master | 0 | - 17 | 38000 | ready | 1 | 1 | master | 0 | - 18 | 37001 | ready | 1 | 1 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | + 14 | 38000 | ready | 1 | 1 | leader | 0 | + 15 | 38000 | ready | 1 | 1 | leader | 0 | + 16 | 38000 | ready | 1 | 1 | leader | 0 | + 17 | 38000 | ready | 1 | 1 | leader | 0 | + 18 | 37001 | ready | 1 | 1 | leader | 0 | + 19 | 37000 | ready | 1 | 1 | leader | 0 | + 20 | 37000 | ready | 1 | 1 | leader | 0 | + 21 | 37000 | ready | 1 | 1 | leader | 0 | Query OK, 8 row(s) in set (0.001154s) ``` @@ -161,14 +161,14 @@ First `show vgroups` is executed to show the vgroup distribution. taos> show vgroups; vgId | tables | status | onlines | v1_dnode | v1_status | compacting | ========================================================================================== - 14 | 38000 | ready | 1 | 3 | master | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | - 18 | 37001 | ready | 1 | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | + 14 | 38000 | ready | 1 | 3 | leader | 0 | + 15 | 38000 | ready | 1 | 3 | leader | 0 | + 16 | 38000 | ready | 1 | 3 | leader | 0 | + 17 | 38000 | ready | 1 | 3 | leader | 0 | + 18 | 37001 | ready | 1 | 3 | leader | 0 | + 19 | 37000 | ready | 1 | 1 | leader | 0 | + 20 | 37000 | ready | 1 | 1 | leader | 0 | + 21 | 37000 | ready | 1 | 1 | leader | 0 | Query OK, 8 row(s) in set (0.001314s) ``` @@ -191,14 +191,14 @@ Query OK, 0 row(s) in set (0.000575s) taos> show vgroups; vgId | tables | status | onlines | v1_dnode | v1_status | v2_dnode | v2_status | compacting | ================================================================================================================= - 14 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 15 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 16 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 17 | 38000 | ready | 1 | 3 | master | 0 | NULL | 0 | - 18 | 37001 | ready | 2 | 1 | slave | 3 | master | 0 | - 19 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 20 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | - 21 | 37000 | ready | 1 | 1 | master | 0 | NULL | 0 | + 14 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | + 15 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | + 16 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | + 17 | 38000 | ready | 1 | 3 | leader | 0 | NULL | 0 | + 18 | 37001 | ready | 2 | 1 | follower | 3 | leader | 0 | + 19 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | + 20 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | + 21 | 37000 | ready | 1 | 1 | leader | 0 | NULL | 0 | Query OK, 8 row(s) in set (0.001242s) ``` @@ -207,7 +207,7 @@ It can be seen from above output that vgId 18 has been moved from dnode 3 to dno :::note - Manual load balancing can only be performed when the automatic load balancing is disabled, i.e. `balance` is set to 0. -- Only a vnode in normal state, i.e. master or slave, can be moved. vnode can't be moved when its in status offline, unsynced or syncing. +- Only a vnode in normal state, i.e. leader or follower, can be moved. vnode can't be moved when its in status offline, unsynced or syncing. - Before moving a vnode, it's necessary to make sure the target dnode has enough resources: CPU, memory and disk. ::: diff --git a/docs/en/10-cluster/03-ha-and-lb.md b/docs/en/10-cluster/03-ha-and-lb.md index bd718eef9f8dc181628132de831dbca2af59d158..9780e8f6c68904e444d07c6a8c87b095c6b70ead 100644 --- a/docs/en/10-cluster/03-ha-and-lb.md +++ b/docs/en/10-cluster/03-ha-and-lb.md @@ -27,7 +27,7 @@ There may be multiple dnodes in a cluster, but only one mnode can be started in SHOW MNODES; ``` -The end point and role/status (master, slave, unsynced, or offline) of all mnodes can be shown by the above command. When the first dnode is started in a cluster, there must be one mnode in this dnode. Without at least one mnode, the cluster cannot work. If `numOfMNodes` is configured to 2, another mnode will be started when the second dnode is launched. +The end point and role/status (leader, follower, unsynced, or offline) of all mnodes can be shown by the above command. When the first dnode is started in a cluster, there must be one mnode in this dnode. Without at least one mnode, the cluster cannot work. If `numOfMNodes` is configured to 2, another mnode will be started when the second dnode is launched. For the high availability of mnode, `numOfMnodes` needs to be configured to 2 or a higher value. Because the data consistency between mnodes must be guaranteed, the replica confirmation parameter `quorum` is set to 2 automatically if `numOfMNodes` is set to 2 or higher. @@ -58,13 +58,13 @@ When a dnode is offline, it can be detected by the TDengine cluster. There are t - If the dnode has been offline over the threshold configured in `offlineThreshold` in `taos.cfg`, the dnode will be removed from the cluster automatically. A system alert will be generated and automatic load balancing will be triggered if `balance` is set to 1. When the removed dnode is restarted and becomes online, it will not join the cluster automatically. The system administrator has to manually join the dnode to the cluster. :::note -If all the vnodes in a vgroup (or mnodes in mnode group) are in offline or unsynced status, the master node can only be voted on, after all the vnodes or mnodes in the group become online and can exchange status. Following this, the vgroup (or mnode group) is able to provide service. +If all the vnodes in a vgroup (or mnodes in mnode group) are in offline or unsynced status, the leader node can only be voted on, after all the vnodes or mnodes in the group become online and can exchange status. Following this, the vgroup (or mnode group) is able to provide service. ::: ## Arbitrator -The "arbitrator" component is used to address the special case when the number of replicas is set to an even number like 2,4 etc. If half of the vnodes in a vgroup don't work, it is impossible to vote and select a master node. This situation also applies to mnodes if the number of mnodes is set to an even number like 2,4 etc. +The "arbitrator" component is used to address the special case when the number of replicas is set to an even number like 2,4 etc. If half of the vnodes in a vgroup don't work, it is impossible to vote and select a leader node. This situation also applies to mnodes if the number of mnodes is set to an even number like 2,4 etc. To resolve this problem, a new arbitrator component named `tarbitrator`, an abbreviation of TDengine Arbitrator, was introduced. The `tarbitrator` simulates a vnode or mnode but it's only responsible for network communication and doesn't handle any actual data access. As long as more than half of the vnode or mnode, including Arbitrator, are available the vnode group or mnode group can provide data insertion or query services normally. diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index 990af861961e9daf4ac775462e21d6d9852d17c1..fe18349a6dae3ad44772b4a30a2c3d4ad75b0f47 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -21,7 +21,7 @@ The following example is in an Ubuntu environment and uses the `curl` tool to ve The following example lists all databases on the host h1.taosdata.com. To use it in your environment, replace `h1.taosdata.com` and `6041` (the default port) with the actual running TDengine service FQDN and port number. ```html -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' h1.taosdata.com:6041/rest/sql +curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "show databases;" h1.taosdata.com:6041/rest/sql ``` The following return value results indicate that the verification passed. @@ -106,13 +106,13 @@ The HTTP request's BODY is a complete SQL command, and the data table in the SQL Use `curl` to initiate an HTTP request with a custom authentication method, with the following syntax. ```bash -curl -H 'Authorization: Basic ' -d '' :/rest/sql/[db_name] +curl -L -H "Authorization: Basic " -d "" :/rest/sql/[db_name] ``` Or ```bash -curl -u username:password -d '' :/rest/sql/[db_name] +curl -L -u username:password -d "" :/rest/sql/[db_name] ``` where `TOKEN` is the string after Base64 encoding of `{username}:{password}`, e.g. `root:taosdata` is encoded as `cm9vdDp0YW9zZGF0YQ==`. @@ -192,7 +192,7 @@ Response body: - query all records from table d1001 of database demo ```bash - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sql + curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sql ``` Response body: @@ -218,7 +218,7 @@ Response body: - Create database demo: ```bash - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6041/rest/sql + curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "create database demo" 192.168.0.1:6041/rest/sql ``` Response body: @@ -240,7 +240,7 @@ Response body: When the HTTP request URL uses `/rest/sqlt`, the returned result set's timestamp value will be in Unix timestamp format, for example: ```bash -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sqlt +curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sqlt ``` Response body: @@ -268,7 +268,7 @@ Response body: When the HTTP request URL uses `/rest/sqlutc`, the timestamp of the returned result set will be expressed as a UTC format, for example: ```bash - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6041/rest/sqlutc + curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.t1" 192.168.0.1:6041/rest/sqlutc ``` Response body: diff --git a/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json b/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json index f651983528ca824b4e6b14586aac5a5bfb4ecab8..54dc1062d6440cc0fc7b8c69d9e4c6b53e4cd01e 100644 --- a/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json +++ b/docs/en/14-reference/07-tdinsight/assets/15146-tdengine-monitor-dashboard.json @@ -211,7 +211,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Master MNode", + "title": "Leader MNode", "transformations": [ { "id": "filterByValue", @@ -221,7 +221,7 @@ "config": { "id": "regex", "options": { - "value": "master" + "value": "leader" } }, "fieldName": "role" @@ -300,7 +300,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Master MNode Create Time", + "title": "Leader MNode Create Time", "transformations": [ { "id": "filterByValue", @@ -310,7 +310,7 @@ "config": { "id": "regex", "options": { - "value": "master" + "value": "leader" } }, "fieldName": "role" diff --git a/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json b/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json index b4254c428b28a0084e54b5e3c509dd2e0ec651b9..1add8522a712aa2cfef6187e577c42d205432b66 100644 --- a/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json +++ b/docs/en/14-reference/07-tdinsight/assets/tdengine-grafana-7.x.json @@ -153,7 +153,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Master MNode", + "title": "Leader MNode", "transformations": [ { "id": "filterByValue", @@ -163,7 +163,7 @@ "config": { "id": "regex", "options": { - "value": "master" + "value": "leader" } }, "fieldName": "role" @@ -246,7 +246,7 @@ ], "timeFrom": null, "timeShift": null, - "title": "Master MNode Create Time", + "title": "Leader MNode Create Time", "transformations": [ { "id": "filterByValue", @@ -256,7 +256,7 @@ "config": { "id": "regex", "options": { - "value": "master" + "value": "leader" } }, "fieldName": "role" diff --git a/docs/en/14-reference/07-tdinsight/index.md b/docs/en/14-reference/07-tdinsight/index.md index cebfafa225e6e8de75ff84bb51fa664784177910..e74c9de7b2aa71278a99d45f250e0dcaf86d4704 100644 --- a/docs/en/14-reference/07-tdinsight/index.md +++ b/docs/en/14-reference/07-tdinsight/index.md @@ -274,8 +274,8 @@ Details of the metrics are as follows. This section contains the current information and status of the cluster, the alert information is also here (from left to right, top to bottom). - **First EP**: the `firstEp` setting in the current TDengine cluster. -- **Version**: TDengine server version (master mnode). -- **Master Uptime**: The time elapsed since the current Master MNode was elected as Master. +- **Version**: TDengine server version (leader mnode). +- **Leader Uptime**: The time elapsed since the current Leader MNode was elected as Leader. - **Expire Time** - Enterprise version expiration time. - **Used Measuring Points** - The number of measuring points used by the Enterprise Edition. - **Databases** - The number of databases. @@ -333,7 +333,7 @@ Data node resource usage display with repeated multiple rows for the variable `$ 2. **Has MNodes?**: whether the current dnode is a mnode. 3. **CPU Cores**: the number of CPU cores. 4. **VNodes Number**: the number of VNodes in the current dnode. -5. **VNodes Masters**: the number of vnodes in the master role. +5. **VNodes Masters**: the number of vnodes in the leader role. 6. **Current CPU Usage of taosd**: CPU usage rate of taosd processes. 7. **Current Memory Usage of taosd**: memory usage of taosd processes. 8. **Disk Used**: The total disk usage percentage of the taosd data directory. diff --git a/docs/en/14-reference/12-directory.md b/docs/en/14-reference/12-directory.md index 304e3bcb434ee9a6ba338577a4d1ba546b548e3f..d6cffd22e054a759e67d34dd3e8fbb1a8585569c 100644 --- a/docs/en/14-reference/12-directory.md +++ b/docs/en/14-reference/12-directory.md @@ -26,7 +26,6 @@ All executable files of TDengine are in the _/usr/local/taos/bin_ directory by d - _remove.sh_: script to uninstall TDengine, please execute it carefully, link to the **rmtaos** command in the /usr/bin directory. Will remove the TDengine installation directory `/usr/local/taos`, but will keep `/etc/taos`, `/var/lib/taos`, `/var/log/taos` - _taosadapter_: server-side executable that provides RESTful services and accepts writing requests from a variety of other softwares - _tarbitrator_: provides arbitration for two-node cluster deployments -- _run_taosd_and_taosadapter.sh_: script to start both taosd and taosAdapter - _TDinsight.sh_: script to download TDinsight and install it - _set_core.sh_: script for setting up the system to generate core dump files for easy debugging - _taosd-dump-cfg.gdb_: script to facilitate debugging of taosd's gdb execution. diff --git a/docs/en/21-tdinternal/01-arch.md b/docs/en/21-tdinternal/01-arch.md index 4d8bed4d2d6b3a0404e10213aeab599767325cc2..d7d472eb98a22325e850f4f040dccaa34d02bbff 100644 --- a/docs/en/21-tdinternal/01-arch.md +++ b/docs/en/21-tdinternal/01-arch.md @@ -22,9 +22,9 @@ A complete TDengine system runs on one or more physical nodes. Logically, it inc **Virtual node (vnode)**: To better support data sharding, load balancing and prevent data from overheating or skewing, data nodes are virtualized into multiple virtual nodes (vnode, V2, V3, V4, etc. in the figure). Each vnode is a relatively independent work unit, which is the basic unit of time-series data storage and has independent running threads, memory space and persistent storage path. A vnode contains a certain number of tables (data collection points). When a new table is created, the system checks whether a new vnode needs to be created. The number of vnodes that can be created on a data node depends on the capacity of the hardware of the physical node where the data node is located. A vnode belongs to only one DB, but a DB can have multiple vnodes. In addition to the stored time-series data, a vnode also stores the schema and tag values of the included tables. A virtual node is uniquely identified in the system by the EP of the data node and the VGroup ID to which it belongs and is created and managed by the management node. -**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 5) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The master/slave mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the master. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. +**Management node (mnode)**: A virtual logical unit responsible for monitoring and maintaining the running status of all data nodes and load balancing among nodes (M in the figure). At the same time, the management node is also responsible for the storage and management of metadata (including users, databases, tables, static tags, etc.), so it is also called Meta Node. Multiple (up to 5) mnodes can be configured in a TDengine cluster, and they are automatically constructed into a virtual management node group (M0, M1, M2 in the figure). The leader/follower mechanism is adopted for the mnode group and the data synchronization is carried out in a strongly consistent way. Any data update operation can only be executed on the leader. The creation of mnode cluster is completed automatically by the system without manual intervention. There is at most one mnode on each dnode, which is uniquely identified by the EP of the data node to which it belongs. Each dnode automatically obtains the EP of the dnode where all mnodes in the whole cluster are located, through internal messaging interaction. -**Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed in a master/slave mechanism. Write operations can only be performed on the master vnode, and then replicated to slave vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `“replica”` when creating a DB, and the default is 1. Using the multi-replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID. If two virtual nodes have the same vnode group ID, it means that they belong to the same group and the data is backed up to each other. The number of virtual nodes in a virtual node group can be dynamically changed, allowing only one, that is, no data replication. VGroup ID is never changed. Even if a virtual node group is deleted, its ID will not be reused. +**Virtual node group (VGroup)**: Vnodes on different data nodes can form a virtual node group to ensure the high availability of the system. The virtual node group is managed in a leader/follower mechanism. Write operations can only be performed on the leader vnode, and then replicated to follower vnodes, thus ensuring that one single replica of data is copied on multiple physical nodes. The number of virtual nodes in a vgroup equals the number of data replicas. If the number of replicas of a DB is N, the system must have at least N data nodes. The number of replicas can be specified by the parameter `“replica”` when creating a DB, and the default is 1. Using the multi-replication feature of TDengine, the same high data reliability can be achieved without the need for expensive storage devices such as disk arrays. Virtual node groups are created and managed by the management node, and the management node assigns a system unique ID, aka VGroup ID. If two virtual nodes have the same vnode group ID, it means that they belong to the same group and the data is backed up to each other. The number of virtual nodes in a virtual node group can be dynamically changed, allowing only one, that is, no data replication. VGroup ID is never changed. Even if a virtual node group is deleted, its ID will not be reused. **TAOSC**: TAOSC is the driver provided by TDengine to applications. It is responsible for dealing with the interaction between application and cluster, and provides the native interface for the C/C++ language. It is also embedded in the JDBC, C #, Python, Go, Node.js language connection libraries. Applications interact with the whole cluster through TAOSC instead of directly connecting to data nodes in the cluster. This module is responsible for obtaining and caching metadata; forwarding requests for insertion, query, etc. to the correct data node; when returning the results to the application, TAOSC also needs to be responsible for the final level of aggregation, sorting, filtering and other operations. For JDBC, C/C++/C#/Python/Go/Node.js interfaces, this module runs on the physical node where the application is located. At the same time, in order to support the fully distributed RESTful interface, TAOSC has a running instance on each dnode of TDengine cluster. @@ -62,13 +62,13 @@ To explain the relationship between vnode, mnode, TAOSC and application and thei 1. Application initiates a request to insert data through JDBC, ODBC, or other APIs. 2. TAOSC checks the cache to see if meta data exists for the table. If it does, it goes straight to Step 4. If not, TAOSC sends a get meta-data request to mnode. 3. Mnode returns the meta-data of the table to TAOSC. Meta-data contains the schema of the table, and also the vgroup information to which the table belongs (the vnode ID and the End Point of the dnode where the table belongs. If the number of replicas is N, there will be N groups of End Points). If TAOSC does not receive a response from the mnode for a long time, and there are multiple mnodes, TAOSC will send a request to the next mnode. -4. TAOSC initiates an insert request to master vnode. +4. TAOSC initiates an insert request to leader vnode. 5. After vnode inserts the data, it gives a reply to TAOSC, indicating that the insertion is successful. If TAOSC doesn't get a response from vnode for a long time, TAOSC will treat this node as offline. In this case, if there are multiple replicas of the inserted database, TAOSC will issue an insert request to the next vnode in vgroup. 6. TAOSC notifies APP that writing is successful. For Step 2 and 3, when TAOSC starts, it does not know the End Point of mnode, so it will directly initiate a request to the configured serving End Point of the cluster. If the dnode that receives the request does not have a mnode configured, it will reply with the mnode EP list, so that TAOSC will re-issue a request to obtain meta-data to the EP of another mnode. -For Step 4 and 5, without caching, TAOSC can't recognize the master in the virtual node group, so assumes that the first vnode is the master and sends a request to it. If this vnode is not the master, it will reply to the actual master as a new target to which TAOSC shall send a request. Once a response of successful insertion is obtained, TAOSC will cache the information of master node. +For Step 4 and 5, without caching, TAOSC can't recognize the leader in the virtual node group, so assumes that the first vnode is the leader and sends a request to it. If this vnode is not the leader, it will reply to the actual leader as a new target to which TAOSC shall send a request. Once a response of successful insertion is obtained, TAOSC will cache the information of leader node. The above describes the process of inserting data. The processes of querying and computing are the same. TAOSC encapsulates and hides all these complicated processes, and it is transparent to applications. @@ -119,65 +119,65 @@ The load balancing process does not require any manual intervention, and it is t ## Data Writing and Replication Process -If a database has N replicas, a virtual node group has N virtual nodes. But only one is the Master and all others are slaves. When the application writes a new record to system, only the Master vnode can accept the writing request. If a slave vnode receives a writing request, the system will notifies TAOSC to redirect. +If a database has N replicas, a virtual node group has N virtual nodes. But only one is the Leader and all others are slaves. When the application writes a new record to system, only the Leader vnode can accept the writing request. If a follower vnode receives a writing request, the system will notifies TAOSC to redirect. -### Master vnode Writing Process +### Leader vnode Writing Process -Master Vnode uses a writing process as follows: +Leader Vnode uses a writing process as follows: -![TDengine Database Master Writing Process](write_master.webp) -
Figure 3: TDengine Master writing process
+![TDengine Database Leader Writing Process](write_master.webp) +
Figure 3: TDengine Leader writing process
-1. Master vnode receives the application data insertion request, verifies, and moves to next step; +1. Leader vnode receives the application data insertion request, verifies, and moves to next step; 2. If the system configuration parameter `“walLevel”` is greater than 0, vnode will write the original request packet into database log file WAL. If walLevel is set to 2 and fsync is set to 0, TDengine will make WAL data written immediately to ensure that even system goes down, all data can be recovered from database log file; -3. If there are multiple replicas, vnode will forward data packet to slave vnodes in the same virtual node group, and the forwarded packet has a version number with data; +3. If there are multiple replicas, vnode will forward data packet to follower vnodes in the same virtual node group, and the forwarded packet has a version number with data; 4. Write into memory and add the record to “skip list”; -5. Master vnode returns a confirmation message to the application, indicating a successful write. +5. Leader vnode returns a confirmation message to the application, indicating a successful write. 6. If any of Step 2, 3 or 4 fails, the error will directly return to the application. -### Slave vnode Writing Process +### Follower vnode Writing Process -For a slave vnode, the write process as follows: +For a follower vnode, the write process as follows: -![TDengine Database Slave Writing Process](write_slave.webp) -
Figure 4: TDengine Slave Writing Process
+![TDengine Database Follower Writing Process](write_slave.webp) +
Figure 4: TDengine Follower Writing Process
-1. Slave vnode receives a data insertion request forwarded by Master vnode; +1. Follower vnode receives a data insertion request forwarded by Leader vnode; 2. If the system configuration parameter `“walLevel”` is greater than 0, vnode will write the original request packet into database log file WAL. If walLevel is set to 2 and fsync is set to 0, TDengine will make WAL data written immediately to ensure that even system goes down, all data can be recovered from database log file; 3. Write into memory and add the record to “skip list”. -Compared with Master vnode, slave vnode has no forwarding or reply confirmation step, means two steps less. But writing into memory and WAL is exactly the same. +Compared with Leader vnode, follower vnode has no forwarding or reply confirmation step, means two steps less. But writing into memory and WAL is exactly the same. ### Remote Disaster Recovery and IDC (Internet Data Center) Migration -As discussed above, TDengine writes using Master and Slave processes. TDengine adopts asynchronous replication for data synchronization. This method can greatly improve write performance, with no obvious impact from network delay. By configuring IDC and rack number for each physical node, it can be ensured that for a virtual node group, virtual nodes are composed of physical nodes from different IDC and different racks, thus implementing remote disaster recovery without other tools. +As discussed above, TDengine writes using Leader and Follower processes. TDengine adopts asynchronous replication for data synchronization. This method can greatly improve write performance, with no obvious impact from network delay. By configuring IDC and rack number for each physical node, it can be ensured that for a virtual node group, virtual nodes are composed of physical nodes from different IDC and different racks, thus implementing remote disaster recovery without other tools. -On the other hand, TDengine supports dynamic modification of the replica number. Once the number of replicas increases, the newly added virtual nodes will immediately enter the data synchronization process. After synchronization is complete, added virtual nodes can provide services. In the synchronization process, master and other synchronized virtual nodes keep serving. With this feature, TDengine can provide IDC migration without service interruption. It is only necessary to add new physical nodes to the existing IDC cluster, and then remove old physical nodes after the data synchronization is completed. +On the other hand, TDengine supports dynamic modification of the replica number. Once the number of replicas increases, the newly added virtual nodes will immediately enter the data synchronization process. After synchronization is complete, added virtual nodes can provide services. In the synchronization process, leader and other synchronized virtual nodes keep serving. With this feature, TDengine can provide IDC migration without service interruption. It is only necessary to add new physical nodes to the existing IDC cluster, and then remove old physical nodes after the data synchronization is completed. However, the asynchronous replication has a very low probability scenario where data may be lost. The specific scenario is as follows: -1. Master vnode has finished its 5-step operations, confirmed the success of writing to APP, and then goes down; -2. Slave vnode receives the write request, then processing fails before writing to the log in Step 2; -3. Slave vnode will become the new master, thus losing one record. +1. Leader vnode has finished its 5-step operations, confirmed the success of writing to APP, and then goes down; +2. Follower vnode receives the write request, then processing fails before writing to the log in Step 2; +3. Follower vnode will become the new leader, thus losing one record. In theory, for asynchronous replication, there is no guarantee to prevent data loss. However, this is an extremely low probability scenario as described above. Note: Remote disaster recovery and no-downtime IDC migration are only supported by Enterprise Edition. **Hint: This function is not available yet** -### Master/slave Selection +### Leader/follower Selection Vnode maintains a version number. When memory data is persisted, the version number will also be persisted. For each data update operation, whether it is time-series data or metadata, this version number will be increased by one. -When a vnode starts, the roles (master, slave) are uncertain, and the data is in an unsynchronized state. It’s necessary to establish TCP connections with other nodes in the virtual node group and exchange status, including version and its own roles. Through the exchange, the system implements a master-selection process. The rules are as follows: +When a vnode starts, the roles (leader, follower) are uncertain, and the data is in an unsynchronized state. It’s necessary to establish TCP connections with other nodes in the virtual node group and exchange status, including version and its own roles. Through the exchange, the system implements a leader-selection process. The rules are as follows: -1. If there’s only one replica, it’s always master -2. When all replicas are online, the one with latest version is master -3. Over half of online nodes are virtual nodes, and some virtual node is slave, it will automatically become master -4. For 2 and 3, if multiple virtual nodes meet the requirement, the first vnode in virtual node group list will be selected as master. +1. If there’s only one replica, it’s always leader +2. When all replicas are online, the one with latest version is leader +3. Over half of online nodes are virtual nodes, and some virtual node is follower, it will automatically become leader +4. For 2 and 3, if multiple virtual nodes meet the requirement, the first vnode in virtual node group list will be selected as leader. ### Synchronous Replication -For scenarios with strong data consistency requirements, asynchronous data replication is not applicable, because there is a small probability of data loss. So, TDengine provides a synchronous replication mechanism for users. When creating a database, in addition to specifying the number of replicas, user also needs to specify a new parameter “quorum”. If quorum is greater than one, it means that every time the Master forwards a message to the replica, it needs to wait for “quorum-1” reply confirms before informing the application that data has been successfully written in slave. If “quorum-1” reply confirms are not received within a certain period of time, the master vnode will return an error to the application. +For scenarios with strong data consistency requirements, asynchronous data replication is not applicable, because there is a small probability of data loss. So, TDengine provides a synchronous replication mechanism for users. When creating a database, in addition to specifying the number of replicas, user also needs to specify a new parameter “quorum”. If quorum is greater than one, it means that every time the Leader forwards a message to the replica, it needs to wait for “quorum-1” reply confirms before informing the application that data has been successfully written in follower. If “quorum-1” reply confirms are not received within a certain period of time, the leader vnode will return an error to the application. With synchronous replication, performance of system will decrease and latency will increase. Because metadata needs strong consistency, the default for data synchronization between mnodes is synchronous replication. diff --git a/docs/en/27-train-faq/03-docker.md b/docs/en/27-train-faq/03-docker.md index afee13c1377b0b4331d6f7ec20251d1aa2db81a1..0378fffb8bdbc4cae8d4d2176ec3d745a548c2fe 100644 --- a/docs/en/27-train-faq/03-docker.md +++ b/docs/en/27-train-faq/03-docker.md @@ -109,7 +109,7 @@ taos> It's also able to access the REST interface provided by TDengine in container from the host. ``` -curl -u root:taosdata -d 'show databases' 127.0.0.1:6041/rest/sql +curl -L -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql ``` Output is like below: @@ -147,7 +147,7 @@ docker run -d --name tdengine-taosd -p 6030-6042:6030-6042 -p 6030-6042:6030-604 - Verify the REST interface: ```bash -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' 127.0.0.1:6041/rest/sql +curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "show databases;" 127.0.0.1:6041/rest/sql ``` Below is an example output: diff --git a/docs/zh/14-reference/02-rest-api/02-rest-api.mdx b/docs/zh/14-reference/02-rest-api/02-rest-api.mdx index 43099319b9c5bb1420c199cfa9f7def0b2c44d3d..a8a92606e4aadf7298359023e739d568788094fd 100644 --- a/docs/zh/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/zh/14-reference/02-rest-api/02-rest-api.mdx @@ -21,7 +21,7 @@ RESTful 接口不依赖于任何 TDengine 的库，因此客户端不需要安 下面示例是列出所有的数据库，请把 h1.taosdata.com 和 6041（缺省值）替换为实际运行的 TDengine 服务 FQDN 和端口号： ```html -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' h1.taosdata.com:6041/rest/sql +curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "show databases;" h1.taosdata.com:6041/rest/sql ``` 返回值结果如下表示验证通过： @@ -106,13 +106,13 @@ HTTP 请求的 BODY 里就是一个完整的 SQL 语句，SQL 语句中的数据 使用 `curl` 通过自定义身份认证方式来发起一个 HTTP Request，语法如下： ```bash -curl -H 'Authorization: Basic ' -d '' :/rest/sql/[db_name] +curl -L -H "Authorization: Basic " -d "" :/rest/sql/[db_name] ``` 或者 ```bash -curl -u username:password -d '' :/rest/sql/[db_name] +curl -L -u username:password -d "" :/rest/sql/[db_name] ``` 其中，`TOKEN` 为 `{username}:{password}` 经过 Base64 编码之后的字符串，例如 `root:taosdata` 编码后为 `cm9vdDp0YW9zZGF0YQ==` @@ -192,7 +192,7 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata - 在 demo 库里查询表 d1001 的所有记录： ```bash - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sql + curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sql ``` 返回值： @@ -218,7 +218,7 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata - 创建库 demo： ```bash - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'create database demo' 192.168.0.1:6041/rest/sql + curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "create database demo" 192.168.0.1:6041/rest/sql ``` 返回值： @@ -240,7 +240,7 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata HTTP 请求 URL 采用 `/rest/sqlt` 时，返回结果集的时间戳将采用 Unix 时间戳格式表示，例如 ```bash -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001' 192.168.0.1:6041/rest/sqlt +curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.d1001" 192.168.0.1:6041/rest/sqlt ``` 返回结果： @@ -268,7 +268,7 @@ curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.d1001 HTTP 请求 URL 采用 `/rest/sqlutc` 时，返回结果集的时间戳将采用 UTC 时间字符串表示，例如 ```bash - curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'select * from demo.t1' 192.168.0.1:6041/rest/sqlutc + curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "select * from demo.t1" 192.168.0.1:6041/rest/sqlutc ``` 返回值： diff --git a/docs/zh/14-reference/12-directory.md b/docs/zh/14-reference/12-directory.md index f8c8cb4a082f691cf75db9bed3b42d0d6e1bc8a3..0caf7e03c32b475e82b6f0bcf58ba2d9225aa6bc 100644 --- a/docs/zh/14-reference/12-directory.md +++ b/docs/zh/14-reference/12-directory.md @@ -26,7 +26,6 @@ TDengine 的所有可执行文件默认存放在 _/usr/local/taos/bin_ 目录下 - _remove.sh_：卸载 TDengine 的脚本，请谨慎执行，链接到/usr/bin 目录下的**rmtaos**命令。会删除 TDengine 的安装目录/usr/local/taos，但会保留/etc/taos、/var/lib/taos、/var/log/taos - _taosadapter_: 提供 RESTful 服务和接受其他多种软件写入请求的服务端可执行文件 - _tarbitrator_: 提供双节点集群部署的仲裁功能 -- _run_taosd_and_taosadapter.sh_：同时启动 taosd 和 taosAdapter 的脚本 - _TDinsight.sh_：用于下载 TDinsight 并安装的脚本 - _set_core.sh_：用于方便调试设置系统生成 core dump 文件的脚本 - _taosd-dump-cfg.gdb_：用于方便调试 taosd 的 gdb 执行脚本。 diff --git a/docs/zh/27-train-faq/03-docker.md b/docs/zh/27-train-faq/03-docker.md index 7791569b25e102b4634f0fb899fc0973cacc0aa1..72b4603dda2a078d04427290c560c9818f19915e 100644 --- a/docs/zh/27-train-faq/03-docker.md +++ b/docs/zh/27-train-faq/03-docker.md @@ -108,7 +108,7 @@ taos> 也可以在宿主机使用 curl 通过 RESTful 端口访问 Docker 容器内的 TDengine server。 ``` -curl -u root:taosdata -d 'show databases' 127.0.0.1:6041/rest/sql +curl -L -u root:taosdata -d "show databases" 127.0.0.1:6041/rest/sql ``` 输出示例如下： @@ -148,7 +148,7 @@ docker run -d --name tdengine-taosd -p 6030-6042:6030-6042 -p 6030-6042:6030-604 使用 curl 命令验证 RESTful 接口可以正常工作： ```bash -curl -H 'Authorization: Basic cm9vdDp0YW9zZGF0YQ==' -d 'show databases;' 127.0.0.1:6041/rest/sql +curl -L -H "Authorization: Basic cm9vdDp0YW9zZGF0YQ==" -d "show databases;" 127.0.0.1:6041/rest/sql ``` 输出示例如下： diff --git a/examples/c/stream_demo.c b/examples/c/stream_demo.c index 5f6e3b2aeb6c27bbcc5f9dd7fcd89af3431e2f25..961eb6c93ad51ba785a9075c2e026fa20f7a8446 100644 --- a/examples/c/stream_demo.c +++ b/examples/c/stream_demo.c @@ -90,9 +90,10 @@ int32_t create_stream() { /*const char* sql = "select min(k), max(k), sum(k) as sum_of_k from st1";*/ /*const char* sql = "select sum(k) from tu1 interval(10m)";*/ /*pRes = tmq_create_stream(pConn, "stream1", "out1", sql);*/ - pRes = taos_query(pConn, - "create stream stream1 trigger window_close into outstb as select _wstartts, sum(k) from st1 " - "interval(10s) "); + pRes = taos_query( + pConn, + "create stream stream1 trigger window_close watermark 10s into outstb as select _wstartts, sum(k) from st1 " + "interval(10s) "); if (taos_errno(pRes) != 0) { printf("failed to create stream stream1, reason:%s\n", taos_errstr(pRes)); return -1; diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 7870d7d9a188030806fc113fbadcb2f76172ef66..6fb7e7a1fce8677870abd3f1489dce2bd8b678b2 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -26,6 +26,14 @@ static void msg_process(TAOS_RES* msg) { printf("topic: %s\n", tmq_get_topic_name(msg)); printf("db: %s\n", tmq_get_db_name(msg)); printf("vg: %d\n", tmq_get_vgroup_id(msg)); + if (tmq_get_res_type(msg) == TMQ_RES_TABLE_META) { + void* meta; + int32_t metaLen; + tmq_get_raw_meta(msg, &meta, &metaLen); + + printf("meta, len is %d\n", metaLen); + return; + } while (1) { TAOS_ROW row = taos_fetch_row(msg); if (row == NULL) break; @@ -76,19 +84,41 @@ int32_t init_env() { } taos_free_result(pRes); + pRes = taos_query(pConn, "insert into ct0 values(now, 1, 2, 'a')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct0, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); + pRes = taos_query(pConn, "create table if not exists ct1 using st1 tags(2000)"); if (taos_errno(pRes) != 0) { - printf("failed to create child table tu2, reason:%s\n", taos_errstr(pRes)); + printf("failed to create child table ct1, reason:%s\n", taos_errstr(pRes)); return -1; } + taos_free_result(pRes); + + pRes = taos_query(pConn, "insert into ct1 values(now, 3, 4, 'b')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct1, reason:%s\n", taos_errstr(pRes)); + return -1; + } + taos_free_result(pRes); pRes = taos_query(pConn, "create table if not exists ct3 using st1 tags(3000)"); if (taos_errno(pRes) != 0) { - printf("failed to create child table tu3, reason:%s\n", taos_errstr(pRes)); + printf("failed to create child table ct3, reason:%s\n", taos_errstr(pRes)); return -1; } + taos_free_result(pRes); + pRes = taos_query(pConn, "insert into ct3 values(now, 5, 6, 'c')"); + if (taos_errno(pRes) != 0) { + printf("failed to insert into ct3, reason:%s\n", taos_errstr(pRes)); + return -1; + } taos_free_result(pRes); + return 0; } @@ -107,8 +137,8 @@ int32_t create_topic() { } taos_free_result(pRes); - /*pRes = taos_query(pConn, "create topic topic_ctb_column as database abc1");*/ - pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1"); + pRes = taos_query(pConn, "create topic topic_ctb_column with meta as database abc1"); + /*pRes = taos_query(pConn, "create topic topic_ctb_column as select ts, c1, c2, c3 from st1");*/ if (taos_errno(pRes) != 0) { printf("failed to create topic topic_ctb_column, reason:%s\n", taos_errstr(pRes)); return -1; @@ -168,6 +198,9 @@ tmq_t* build_consumer() { tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set(conf, "enable.auto.commit", "true"); + + tmq_conf_set(conf, "experiment.use.snapshot", "false"); + tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); tmq_t* tmq = tmq_consumer_new(conf, NULL, 0); assert(tmq); diff --git a/include/client/taos.h b/include/client/taos.h index 61538e392a0daa14b3f2def6ed74b0ee51d8d7ef..d31d5c582cd128db3b5217698a27dd92d8a3d108 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -187,8 +187,8 @@ DLL_EXPORT TAOS_ROW *taos_result_block(TAOS_RES *res); DLL_EXPORT const char *taos_get_server_info(TAOS *taos); DLL_EXPORT const char *taos_get_client_info(); -DLL_EXPORT const char *taos_errstr(TAOS_RES *tres); -DLL_EXPORT int taos_errno(TAOS_RES *tres); +DLL_EXPORT const char *taos_errstr(TAOS_RES *res); +DLL_EXPORT int taos_errno(TAOS_RES *res); DLL_EXPORT void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param); DLL_EXPORT void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param); @@ -252,6 +252,16 @@ DLL_EXPORT void tmq_conf_set_auto_commit_cb(tmq_conf_t *conf, tmq_comm /* -------------------------TMQ MSG HANDLE INTERFACE---------------------- */ +enum tmq_res_t { + TMQ_RES_INVALID = -1, + TMQ_RES_DATA = 1, + TMQ_RES_TABLE_META = 2, +}; + +typedef enum tmq_res_t tmq_res_t; + +DLL_EXPORT tmq_res_t tmq_get_res_type(TAOS_RES *res); +DLL_EXPORT int32_t tmq_get_raw_meta(TAOS_RES *res, void **raw_meta, int32_t *raw_meta_len); DLL_EXPORT const char *tmq_get_topic_name(TAOS_RES *res); DLL_EXPORT const char *tmq_get_db_name(TAOS_RES *res); DLL_EXPORT int32_t tmq_get_vgroup_id(TAOS_RES *res); diff --git a/include/common/systable.h b/include/common/systable.h index d2c28941c77ae92c7a52c6e9df89de1650ad4360..faea4dd3f80f7287f9a45db6f4ad71d8b0fcf18e 100644 --- a/include/common/systable.h +++ b/include/common/systable.h @@ -41,6 +41,7 @@ extern "C" { #define TSDB_INS_TABLE_VGROUPS "vgroups" #define TSDB_INS_TABLE_VNODES "vnodes" #define TSDB_INS_TABLE_CONFIGS "configs" +#define TSDB_INS_TABLE_DNODE_VARIABLES "dnode_variables" #define TSDB_PERFORMANCE_SCHEMA_DB "performance_schema" #define TSDB_PERFS_TABLE_SMAS "smas" diff --git a/include/common/tcommon.h b/include/common/tcommon.h index a05287761ef5dbf8d28c075bbc6cb9cac9910651..09f24cc55565380e5a01ee2dc82f2b919b7136dc 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -34,6 +34,7 @@ enum { enum { TMQ_MSG_TYPE__DUMMY = 0, TMQ_MSG_TYPE__POLL_RSP, + TMQ_MSG_TYPE__POLL_META_RSP, TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__END_RSP, }; @@ -44,11 +45,16 @@ typedef enum EStreamType { STREAM_REPROCESS, STREAM_INVALID, STREAM_GET_ALL, + STREAM_DELETE, + STREAM_RETRIEVE, } EStreamType; typedef struct { SArray* pTableList; SHashObj* map; // speedup acquire the tableQueryInfo by table uid + void* pTagCond; + void* pTagIndexCond; + uint64_t suid; } STableListInfo; typedef struct SColumnDataAgg { @@ -63,14 +69,13 @@ typedef struct SColumnDataAgg { typedef struct SDataBlockInfo { STimeWindow window; - int32_t rows; + int32_t rows; // todo hide this attribute int32_t rowSize; - int64_t uid; // the uid of table, from which current data block comes - int64_t blockId; // block id, generated by physical planner + uint64_t uid; // the uid of table, from which current data block comes + uint16_t blockId; // block id, generated by physical planner uint64_t groupId; // no need to serialize - int16_t numOfCols; int16_t hasVarCol; - int32_t capacity; + uint32_t capacity; // TODO: optimize and remove following int32_t childId; // used for stream, do not serialize EStreamType type; // used for stream, do not serialize @@ -91,7 +96,7 @@ typedef struct SVarColAttr { // pBlockAgg->numOfNull == info.rows, all data are null // pBlockAgg->numOfNull == 0, no data are null. typedef struct SColumnInfoData { - SColumnInfo info; // TODO filter info needs to be removed + SColumnInfo info; // column info bool hasNull; // if current column data has null value. char* pData; // the corresponding block data in memory union { @@ -110,8 +115,8 @@ typedef struct SQueryTableDataCond { int32_t type; // data block load type: int32_t numOfTWindows; STimeWindow* twindows; - int32_t startVersion; - int32_t endVersion; + int64_t startVersion; + int64_t endVersion; } SQueryTableDataCond; void* blockDataDestroy(SSDataBlock* pBlock); diff --git a/include/common/tdatablock.h b/include/common/tdatablock.h index 709462a7449dade291fc06166b7fc08e80f96cb4..6204b9dcfca0be850d036f0eeb1687f7ba6148bb 100644 --- a/include/common/tdatablock.h +++ b/include/common/tdatablock.h @@ -71,7 +71,8 @@ SEpSet getEpSet_s(SCorEpSet* pEpSet); #define colDataGetData(p1_, r_) \ ((IS_VAR_DATA_TYPE((p1_)->info.type)) ? colDataGetVarData(p1_, r_) : colDataGetNumData(p1_, r_)) -#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && *(data) == TSDB_DATA_TYPE_NULL) +#define IS_JSON_NULL(type, data) ((type) == TSDB_DATA_TYPE_JSON && \ + (*(data) == TSDB_DATA_TYPE_NULL || tTagIsJsonNull(data))) static FORCE_INLINE bool colDataIsNull_s(const SColumnInfoData* pColumnInfoData, uint32_t row) { if (!pColumnInfoData->hasNull) { @@ -183,9 +184,9 @@ static FORCE_INLINE void colDataAppendDouble(SColumnInfoData* pColumnInfoData, u int32_t getJsonValueLen(const char* data); int32_t colDataAppend(SColumnInfoData* pColumnInfoData, uint32_t currentRow, const char* pData, bool isNull); -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2); -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows); +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo); int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex); int32_t colDataGetLength(const SColumnInfoData* pColumnInfoData, int32_t numOfRows); @@ -211,7 +212,7 @@ size_t blockDataGetSerialMetaSize(uint32_t numOfCols); int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo); int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullFirst); -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows); +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows); int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows); void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows); @@ -222,6 +223,11 @@ size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize); int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n); SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData); +SSDataBlock* createDataBlock(); +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData); + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId); +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index); void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen, int32_t numOfCols, int8_t needCompress); @@ -235,7 +241,7 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks char* buildCtbNameByGroupId(const char* stbName, uint64_t groupId); static FORCE_INLINE int32_t blockGetEncodeSize(const SSDataBlock* pBlock) { - return blockDataGetSerialMetaSize(pBlock->info.numOfCols) + blockDataGetSize(pBlock); + return blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)) + blockDataGetSize(pBlock); } static FORCE_INLINE int32_t blockCompressColData(SColumnInfoData* pColRes, int32_t numOfRows, char* data, diff --git a/include/common/tdataformat.h b/include/common/tdataformat.h index 2e69640a065f9049f2bcb2a2c3f190f0178fca3f..f9ede63f7f6898f571e6462a4cf3b813be96a405 100644 --- a/include/common/tdataformat.h +++ b/include/common/tdataformat.h @@ -70,6 +70,8 @@ int32_t tGetTSRow(uint8_t *p, STSRow2 *pRow); // STag int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag); void tTagFree(STag *pTag); +bool tTagIsJson(const void *pTag); +bool tTagIsJsonNull(void *tagVal); bool tTagGet(const STag *pTag, STagVal *pTagVal); char *tTagValToData(const STagVal *pTagVal, bool isJson); int32_t tEncodeTag(SEncoder *pEncoder, const STag *pTag); diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 8c03d3ff42b5b0b28f8f1958acfcd7dc8520d464..e083ebcf7867148ae36d76ca4388d0a48bda58ab 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -137,6 +137,8 @@ extern bool tsSmlDataFormat; // internal extern int32_t tsTransPullupInterval; extern int32_t tsMqRebalanceInterval; +extern int32_t tsTtlUnit; +extern int32_t tsTtlPushInterval; #define NEEDTO_COMPRESSS_MSG(size) (tsCompressMsgSize != -1 && (size) > tsCompressMsgSize) diff --git a/include/common/tmsg.h b/include/common/tmsg.h index fd9ff3eb6535008c88037d76bbf5ce4aff6c0a43..d289ff13aedb55db408d3020ba356a5160dd7ead 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -168,7 +168,7 @@ typedef struct { int32_t vgId; char* dbFName; char* tbName; -} SBuildTableMetaInput; +} SBuildTableInput; typedef struct { char db[TSDB_DB_FNAME_LEN]; @@ -444,6 +444,7 @@ typedef struct { char* comment; char* pAst1; char* pAst2; + SArray* pFuncs; } SMCreateStbReq; int32_t tSerializeSMCreateStbReq(void* buf, int32_t bufLen, SMCreateStbReq* pReq); @@ -628,11 +629,6 @@ typedef struct { uint8_t scale; } SColumnInfo; -typedef struct { - int64_t uid; - TSKEY key; // last accessed ts, for subscription -} STableIdInfo; - typedef struct STimeWindow { TSKEY skey; TSKEY ekey; @@ -667,6 +663,41 @@ int32_t tSerializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp int32_t tDeserializeSQueryTableRsp(void* buf, int32_t bufLen, SQueryTableRsp* pRsp); +typedef struct { + SMsgHead header; + char dbFName[TSDB_DB_FNAME_LEN]; + char tbName[TSDB_TABLE_NAME_LEN]; +} STableCfgReq; + +typedef struct { + char tbName[TSDB_TABLE_NAME_LEN]; + char stbName[TSDB_TABLE_NAME_LEN]; + char dbFName[TSDB_DB_FNAME_LEN]; + int32_t numOfTags; + int32_t numOfColumns; + int8_t tableType; + int64_t delay1; + int64_t delay2; + int64_t watermark1; + int64_t watermark2; + int32_t ttl; + SArray* pFuncs; + int32_t commentLen; + char* pComment; + SSchema* pSchemas; + int32_t tagsLen; + char* pTags; +} STableCfg; + +typedef STableCfg STableCfgRsp; + +int32_t tSerializeSTableCfgReq(void* buf, int32_t bufLen, STableCfgReq* pReq); +int32_t tDeserializeSTableCfgReq(void* buf, int32_t bufLen, STableCfgReq* pReq); + +int32_t tSerializeSTableCfgRsp(void* buf, int32_t bufLen, STableCfgRsp* pRsp); +int32_t tDeserializeSTableCfgRsp(void* buf, int32_t bufLen, STableCfgRsp* pRsp); +void tFreeSTableCfgRsp(STableCfgRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; int32_t numOfVgroups; @@ -798,6 +829,14 @@ typedef struct { int32_t tSerializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); int32_t tDeserializeSQnodeListReq(void* buf, int32_t bufLen, SQnodeListReq* pReq); +typedef struct { + int32_t rowNum; +} SDnodeListReq; + +int32_t tSerializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); +int32_t tDeserializeSDnodeListReq(void* buf, int32_t bufLen, SDnodeListReq* pReq); + + typedef struct SQueryNodeAddr { int32_t nodeId; // vgId or qnodeId SEpSet epSet; @@ -816,6 +855,15 @@ int32_t tSerializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); int32_t tDeserializeSQnodeListRsp(void* buf, int32_t bufLen, SQnodeListRsp* pRsp); void tFreeSQnodeListRsp(SQnodeListRsp* pRsp); +typedef struct { + SArray* dnodeList; // SArray +} SDnodeListRsp; + +int32_t tSerializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp); +int32_t tDeserializeSDnodeListRsp(void* buf, int32_t bufLen, SDnodeListRsp* pRsp); +void tFreeSDnodeListRsp(SDnodeListRsp* pRsp); + + typedef struct { SArray* pArray; // Array of SUseDbRsp } SUseDbBatchRsp; @@ -1174,6 +1222,13 @@ typedef struct { char* data; } STagData; +typedef struct { + int32_t useless; // useless +} SShowVariablesReq; + +int32_t tSerializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); +int32_t tDeserializeSShowVariablesReq(void* buf, int32_t bufLen, SShowVariablesReq* pReq); + /* * sql: show tables like '%a_%' * payload is the query condition, e.g., '%a_%' @@ -1286,11 +1341,19 @@ typedef struct { int32_t dnodeId; char config[TSDB_DNODE_CONFIG_LEN]; char value[TSDB_DNODE_VALUE_LEN]; -} SMCfgDnodeReq, SDCfgDnodeReq; +} SMCfgDnodeReq; int32_t tSerializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); int32_t tDeserializeSMCfgDnodeReq(void* buf, int32_t bufLen, SMCfgDnodeReq* pReq); +typedef struct { + char config[TSDB_DNODE_CONFIG_LEN]; + char value[TSDB_DNODE_VALUE_LEN]; +} SDCfgDnodeReq; + +int32_t tSerializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); +int32_t tDeserializeSDCfgDnodeReq(void* buf, int32_t bufLen, SDCfgDnodeReq* pReq); + typedef struct { int32_t dnodeId; } SMCreateMnodeReq, SMDropMnodeReq, SDDropMnodeReq, SMCreateQnodeReq, SMDropQnodeReq, SDCreateQnodeReq, SDDropQnodeReq, @@ -1317,7 +1380,7 @@ int32_t tSerializeSSetStandbyReq(void* buf, int32_t bufLen, SSetStandbyReq* pReq int32_t tDeserializeSSetStandbyReq(void* buf, int32_t bufLen, SSetStandbyReq* pReq); typedef struct { - char queryStrId[TSDB_QUERY_ID_LEN]; + char queryStrId[TSDB_QUERY_ID_LEN]; } SKillQueryReq; int32_t tSerializeSKillQueryReq(void* buf, int32_t bufLen, SKillQueryReq* pReq); @@ -1560,6 +1623,7 @@ typedef struct { char name[TSDB_TOPIC_FNAME_LEN]; // accout.topic int8_t igExists; int8_t subType; + int8_t withMeta; char* sql; char subDbName[TSDB_DB_FNAME_LEN]; union { @@ -2282,6 +2346,9 @@ typedef struct { int8_t igNotExists; } SMDropStreamReq; +int32_t tSerializeSMDropStreamReq(void* buf, int32_t bufLen, const SMDropStreamReq* pReq); +int32_t tDeserializeSMDropStreamReq(void* buf, int32_t bufLen, SMDropStreamReq* pReq); + typedef struct { int8_t reserved; } SMDropStreamRsp; @@ -2303,6 +2370,7 @@ typedef struct { int64_t newConsumerId; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int8_t subType; + int8_t withMeta; char* qmsg; int64_t suid; } SMqRebVgReq; @@ -2315,6 +2383,7 @@ static FORCE_INLINE int32_t tEncodeSMqRebVgReq(void** buf, const SMqRebVgReq* pR tlen += taosEncodeFixedI64(buf, pReq->newConsumerId); tlen += taosEncodeString(buf, pReq->subKey); tlen += taosEncodeFixedI8(buf, pReq->subType); + tlen += taosEncodeFixedI8(buf, pReq->withMeta); if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { tlen += taosEncodeString(buf, pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { @@ -2330,6 +2399,7 @@ static FORCE_INLINE void* tDecodeSMqRebVgReq(const void* buf, SMqRebVgReq* pReq) buf = taosDecodeFixedI64(buf, &pReq->newConsumerId); buf = taosDecodeStringTo(buf, pReq->subKey); buf = taosDecodeFixedI8(buf, &pReq->subType); + buf = taosDecodeFixedI8(buf, &pReq->withMeta); if (pReq->subType == TOPIC_SUB_TYPE__COLUMN) { buf = taosDecodeString(buf, &pReq->qmsg); } else if (pReq->subType == TOPIC_SUB_TYPE__TABLE) { @@ -2602,6 +2672,7 @@ typedef struct { SMsgHead head; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int8_t withTbName; + int8_t useSnapshot; int32_t epoch; uint64_t reqId; int64_t consumerId; @@ -2674,6 +2745,34 @@ static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { taosArrayDestroy(pSubTopicEp->vgs); } +typedef struct { + SMqRspHead head; + int64_t reqOffset; + int64_t rspOffset; + int16_t resMsgType; + int32_t metaRspLen; + void* metaRsp; +} SMqMetaRsp; + +static FORCE_INLINE int32_t tEncodeSMqMetaRsp(void** buf, const SMqMetaRsp* pRsp) { + int32_t tlen = 0; + tlen += taosEncodeFixedI64(buf, pRsp->reqOffset); + tlen += taosEncodeFixedI64(buf, pRsp->rspOffset); + tlen += taosEncodeFixedI16(buf, pRsp->resMsgType); + tlen += taosEncodeFixedI32(buf, pRsp->metaRspLen); + tlen += taosEncodeBinary(buf, pRsp->metaRsp, pRsp->metaRspLen); + return tlen; +} + +static FORCE_INLINE void* tDecodeSMqMetaRsp(const void* buf, SMqMetaRsp* pRsp) { + buf = taosDecodeFixedI64(buf, &pRsp->reqOffset); + buf = taosDecodeFixedI64(buf, &pRsp->rspOffset); + buf = taosDecodeFixedI16(buf, &pRsp->resMsgType); + buf = taosDecodeFixedI32(buf, &pRsp->metaRspLen); + buf = taosDecodeBinary(buf, &pRsp->metaRsp, pRsp->metaRspLen); + return (void*)buf; +} + typedef struct { SMqRspHead head; int64_t reqOffset; diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index eeac619105c1ff9bc909833c02d6d4ba2e41352e..1babb450038e692ed4d2241e9ad4a096a38c6ba7 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -81,6 +81,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_SERVER_STATUS, "server-status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_NET_TEST, "net-test", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL) TD_NEW_MSG_SEG(TDMT_MND_MSG) TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) @@ -101,6 +102,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_ALTER_QNODE, "alter-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_QNODE, "drop-qnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_QNODE_LIST, "qnode-list", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_DNODE_LIST, "dnode-list", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_SNODE, "create-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_SNODE, "alter-snode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_SNODE, "drop-snode", NULL, NULL) @@ -131,6 +133,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_DROP_INDEX, "drop-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_INDEX, "get-index", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GET_TABLE_INDEX, "get-table-index", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TABLE_CFG, "table-cfg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_CREATE_TOPIC, "create-topic", SMCreateTopicReq, SMCreateTopicRsp) TD_DEF_MSG_TYPE(TDMT_MND_ALTER_TOPIC, "alter-topic", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_DROP_TOPIC, "drop-topic", NULL, NULL) @@ -144,13 +147,14 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MQ_TIMER, "mq-tmr", SMTimerReq, NULL) TD_DEF_MSG_TYPE(TDMT_MND_TELEM_TIMER, "telem-tmr", SMTimerReq, SMTimerReq) TD_DEF_MSG_TYPE(TDMT_MND_TRANS_TIMER, "trans-tmr", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_TTL_TIMER, "ttl-tmr", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_TRANS, "kill-trans", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_QUERY, "kill-query", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_KILL_CONN, "kill-conn", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_HEARTBEAT, "heartbeat", SClientHbBatchReq, SClientHbBatchRsp) TD_DEF_MSG_TYPE(TDMT_MND_STATUS, "status", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SHOW, "show", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "retrieve", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SYSTABLE_RETRIEVE, "mnd-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_GRANT, "grant", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_AUTH, "auth", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_APPLY_MSG, "mnode-apply", NULL, NULL) @@ -158,6 +162,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_MND_MERGE_VGROUP, "merge-vgroup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_REDISTRIBUTE_VGROUP, "redistribute-vgroup", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_MND_SPLIT_VGROUP, "split-vgroup", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_MND_SHOW_VARIABLES, "show-variables", NULL, NULL) TD_NEW_MSG_SEG(TDMT_VND_MSG) TD_DEF_MSG_TYPE(TDMT_VND_SUBMIT, "submit", SSubmitReq, SSubmitRsp) @@ -171,6 +176,7 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_UPDATE_TAG_VAL, "update-tag-val", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLE_META, "vnode-table-meta", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TABLES_META, "vnode-tables-meta", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_VND_TABLE_CFG, "vnode-table-cfg", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_STB, "vnode-create-stb", SVCreateStbReq, NULL) TD_DEF_MSG_TYPE(TDMT_VND_ALTER_STB, "vnode-alter-stb", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_STB, "vnode-drop-stb", SVDropStbReq, NULL) @@ -187,7 +193,6 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_CONSUME, "vnode-consume", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TRIGGER, "vnode-stream-trigger", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_STREAM_DISPATCH_WRITE, "vnode-stream-task-dispatch-write", NULL, NULL) - TD_DEF_MSG_TYPE(TDMT_VND_STREAM_TASK_DROP, "vnode-stream-task-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CREATE_SMA, "vnode-create-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_CANCEL_SMA, "vnode-cancel-sma", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_SMA, "vnode-drop-sma", NULL, NULL) @@ -199,14 +204,14 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_ALTER_HASHRANGE, "alter-hashrange", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_COMPACT, "compact", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_DROP_TTL_TABLE, "drop-ttl-stb", NULL, NULL) - TD_NEW_MSG_SEG(TDMT_QND_MSG) - //shared by snode and vnode TD_NEW_MSG_SEG(TDMT_STREAM_MSG) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DEPLOY, "stream-task-deploy", SStreamTaskDeployReq, SStreamTaskDeployRsp) + TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DROP, "stream-task-drop", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RUN, "stream-task-run", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_DISPATCH, "stream-task-dispatch", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_STREAM_TASK_RECOVER, "stream-task-recover", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_STREAM_RETRIEVE, "stream-retrieve", NULL, NULL) TD_NEW_MSG_SEG(TDMT_SCH_MSG) TD_DEF_MSG_TYPE(TDMT_SCH_LINK_BROKEN, "link-broken", NULL, NULL) diff --git a/include/common/tname.h b/include/common/tname.h index 28f97d10285d0f620ef919db9e691c98e3a28197..c691c2f7b217811b60fcd5bc4249d9ad5441b79a 100644 --- a/include/common/tname.h +++ b/include/common/tname.h @@ -63,6 +63,8 @@ int32_t tNameSetAcctId(SName* dst, int32_t acctId); bool tNameDBNameEqual(SName* left, SName* right); +bool tNameTbNameEqual(SName* left, SName* right); + typedef struct { // input SArray* tags; // element is SSmlKv diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index 35b4a5ce7de5301941e9013e55138d65d964616f..8e00119876a31ce3f1314c4fb898a8d8f44c8649 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -49,214 +49,216 @@ #define TK_CONNS 31 #define TK_STATE 32 #define TK_USER 33 -#define TK_PRIVILEGE 34 -#define TK_DROP 35 -#define TK_GRANT 36 -#define TK_ON 37 -#define TK_TO 38 -#define TK_REVOKE 39 -#define TK_FROM 40 -#define TK_NK_COMMA 41 -#define TK_READ 42 -#define TK_WRITE 43 -#define TK_NK_DOT 44 -#define TK_DNODE 45 -#define TK_PORT 46 -#define TK_NK_INTEGER 47 -#define TK_DNODES 48 -#define TK_NK_IPTOKEN 49 -#define TK_LOCAL 50 -#define TK_QNODE 51 -#define TK_BNODE 52 -#define TK_SNODE 53 -#define TK_MNODE 54 -#define TK_DATABASE 55 -#define TK_USE 56 -#define TK_IF 57 -#define TK_NOT 58 -#define TK_EXISTS 59 -#define TK_BUFFER 60 -#define TK_CACHELAST 61 -#define TK_COMP 62 -#define TK_DURATION 63 -#define TK_NK_VARIABLE 64 -#define TK_FSYNC 65 -#define TK_MAXROWS 66 -#define TK_MINROWS 67 -#define TK_KEEP 68 -#define TK_PAGES 69 -#define TK_PAGESIZE 70 -#define TK_PRECISION 71 -#define TK_REPLICA 72 -#define TK_STRICT 73 -#define TK_WAL 74 -#define TK_VGROUPS 75 -#define TK_SINGLE_STABLE 76 -#define TK_RETENTIONS 77 -#define TK_SCHEMALESS 78 -#define TK_NK_COLON 79 -#define TK_TABLE 80 -#define TK_NK_LP 81 -#define TK_NK_RP 82 -#define TK_STABLE 83 -#define TK_ADD 84 -#define TK_COLUMN 85 -#define TK_MODIFY 86 -#define TK_RENAME 87 -#define TK_TAG 88 -#define TK_SET 89 -#define TK_NK_EQ 90 -#define TK_USING 91 -#define TK_TAGS 92 -#define TK_COMMENT 93 -#define TK_BOOL 94 -#define TK_TINYINT 95 -#define TK_SMALLINT 96 -#define TK_INT 97 -#define TK_INTEGER 98 -#define TK_BIGINT 99 -#define TK_FLOAT 100 -#define TK_DOUBLE 101 -#define TK_BINARY 102 -#define TK_TIMESTAMP 103 -#define TK_NCHAR 104 -#define TK_UNSIGNED 105 -#define TK_JSON 106 -#define TK_VARCHAR 107 -#define TK_MEDIUMBLOB 108 -#define TK_BLOB 109 -#define TK_VARBINARY 110 -#define TK_DECIMAL 111 -#define TK_MAX_DELAY 112 -#define TK_WATERMARK 113 -#define TK_ROLLUP 114 -#define TK_TTL 115 -#define TK_SMA 116 -#define TK_FIRST 117 -#define TK_LAST 118 -#define TK_SHOW 119 -#define TK_DATABASES 120 -#define TK_TABLES 121 -#define TK_STABLES 122 -#define TK_MNODES 123 -#define TK_MODULES 124 -#define TK_QNODES 125 -#define TK_FUNCTIONS 126 -#define TK_INDEXES 127 -#define TK_ACCOUNTS 128 -#define TK_APPS 129 -#define TK_CONNECTIONS 130 -#define TK_LICENCE 131 -#define TK_GRANTS 132 -#define TK_QUERIES 133 -#define TK_SCORES 134 -#define TK_TOPICS 135 -#define TK_VARIABLES 136 -#define TK_BNODES 137 -#define TK_SNODES 138 -#define TK_CLUSTER 139 -#define TK_TRANSACTIONS 140 -#define TK_DISTRIBUTED 141 -#define TK_LIKE 142 -#define TK_INDEX 143 -#define TK_FULLTEXT 144 +#define TK_ENABLE 34 +#define TK_NK_INTEGER 35 +#define TK_SYSINFO 36 +#define TK_DROP 37 +#define TK_GRANT 38 +#define TK_ON 39 +#define TK_TO 40 +#define TK_REVOKE 41 +#define TK_FROM 42 +#define TK_NK_COMMA 43 +#define TK_READ 44 +#define TK_WRITE 45 +#define TK_NK_DOT 46 +#define TK_DNODE 47 +#define TK_PORT 48 +#define TK_DNODES 49 +#define TK_NK_IPTOKEN 50 +#define TK_LOCAL 51 +#define TK_QNODE 52 +#define TK_BNODE 53 +#define TK_SNODE 54 +#define TK_MNODE 55 +#define TK_DATABASE 56 +#define TK_USE 57 +#define TK_IF 58 +#define TK_NOT 59 +#define TK_EXISTS 60 +#define TK_BUFFER 61 +#define TK_CACHELAST 62 +#define TK_COMP 63 +#define TK_DURATION 64 +#define TK_NK_VARIABLE 65 +#define TK_FSYNC 66 +#define TK_MAXROWS 67 +#define TK_MINROWS 68 +#define TK_KEEP 69 +#define TK_PAGES 70 +#define TK_PAGESIZE 71 +#define TK_PRECISION 72 +#define TK_REPLICA 73 +#define TK_STRICT 74 +#define TK_WAL 75 +#define TK_VGROUPS 76 +#define TK_SINGLE_STABLE 77 +#define TK_RETENTIONS 78 +#define TK_SCHEMALESS 79 +#define TK_NK_COLON 80 +#define TK_TABLE 81 +#define TK_NK_LP 82 +#define TK_NK_RP 83 +#define TK_STABLE 84 +#define TK_ADD 85 +#define TK_COLUMN 86 +#define TK_MODIFY 87 +#define TK_RENAME 88 +#define TK_TAG 89 +#define TK_SET 90 +#define TK_NK_EQ 91 +#define TK_USING 92 +#define TK_TAGS 93 +#define TK_COMMENT 94 +#define TK_BOOL 95 +#define TK_TINYINT 96 +#define TK_SMALLINT 97 +#define TK_INT 98 +#define TK_INTEGER 99 +#define TK_BIGINT 100 +#define TK_FLOAT 101 +#define TK_DOUBLE 102 +#define TK_BINARY 103 +#define TK_TIMESTAMP 104 +#define TK_NCHAR 105 +#define TK_UNSIGNED 106 +#define TK_JSON 107 +#define TK_VARCHAR 108 +#define TK_MEDIUMBLOB 109 +#define TK_BLOB 110 +#define TK_VARBINARY 111 +#define TK_DECIMAL 112 +#define TK_MAX_DELAY 113 +#define TK_WATERMARK 114 +#define TK_ROLLUP 115 +#define TK_TTL 116 +#define TK_SMA 117 +#define TK_FIRST 118 +#define TK_LAST 119 +#define TK_SHOW 120 +#define TK_DATABASES 121 +#define TK_TABLES 122 +#define TK_STABLES 123 +#define TK_MNODES 124 +#define TK_MODULES 125 +#define TK_QNODES 126 +#define TK_FUNCTIONS 127 +#define TK_INDEXES 128 +#define TK_ACCOUNTS 129 +#define TK_APPS 130 +#define TK_CONNECTIONS 131 +#define TK_LICENCE 132 +#define TK_GRANTS 133 +#define TK_QUERIES 134 +#define TK_SCORES 135 +#define TK_TOPICS 136 +#define TK_VARIABLES 137 +#define TK_BNODES 138 +#define TK_SNODES 139 +#define TK_CLUSTER 140 +#define TK_TRANSACTIONS 141 +#define TK_DISTRIBUTED 142 +#define TK_LIKE 143 +#define TK_INDEX 144 #define TK_FUNCTION 145 #define TK_INTERVAL 146 #define TK_TOPIC 147 #define TK_AS 148 -#define TK_CONSUMER 149 -#define TK_GROUP 150 -#define TK_DESC 151 -#define TK_DESCRIBE 152 -#define TK_RESET 153 -#define TK_QUERY 154 -#define TK_CACHE 155 -#define TK_EXPLAIN 156 -#define TK_ANALYZE 157 -#define TK_VERBOSE 158 -#define TK_NK_BOOL 159 -#define TK_RATIO 160 -#define TK_NK_FLOAT 161 -#define TK_COMPACT 162 -#define TK_VNODES 163 -#define TK_IN 164 -#define TK_OUTPUTTYPE 165 -#define TK_AGGREGATE 166 -#define TK_BUFSIZE 167 -#define TK_STREAM 168 -#define TK_INTO 169 -#define TK_TRIGGER 170 -#define TK_AT_ONCE 171 -#define TK_WINDOW_CLOSE 172 -#define TK_KILL 173 -#define TK_CONNECTION 174 -#define TK_TRANSACTION 175 -#define TK_BALANCE 176 -#define TK_VGROUP 177 -#define TK_MERGE 178 -#define TK_REDISTRIBUTE 179 -#define TK_SPLIT 180 -#define TK_SYNCDB 181 -#define TK_DELETE 182 -#define TK_NULL 183 -#define TK_NK_QUESTION 184 -#define TK_NK_ARROW 185 -#define TK_ROWTS 186 -#define TK_TBNAME 187 -#define TK_QSTARTTS 188 -#define TK_QENDTS 189 -#define TK_WSTARTTS 190 -#define TK_WENDTS 191 -#define TK_WDURATION 192 -#define TK_CAST 193 -#define TK_NOW 194 -#define TK_TODAY 195 -#define TK_TIMEZONE 196 -#define TK_COUNT 197 -#define TK_LAST_ROW 198 -#define TK_BETWEEN 199 -#define TK_IS 200 -#define TK_NK_LT 201 -#define TK_NK_GT 202 -#define TK_NK_LE 203 -#define TK_NK_GE 204 -#define TK_NK_NE 205 -#define TK_MATCH 206 -#define TK_NMATCH 207 -#define TK_CONTAINS 208 -#define TK_JOIN 209 -#define TK_INNER 210 -#define TK_SELECT 211 -#define TK_DISTINCT 212 -#define TK_WHERE 213 -#define TK_PARTITION 214 -#define TK_BY 215 -#define TK_SESSION 216 -#define TK_STATE_WINDOW 217 -#define TK_SLIDING 218 -#define TK_FILL 219 -#define TK_VALUE 220 -#define TK_NONE 221 -#define TK_PREV 222 -#define TK_LINEAR 223 -#define TK_NEXT 224 -#define TK_HAVING 225 -#define TK_RANGE 226 -#define TK_EVERY 227 -#define TK_ORDER 228 -#define TK_SLIMIT 229 -#define TK_SOFFSET 230 -#define TK_LIMIT 231 -#define TK_OFFSET 232 -#define TK_ASC 233 -#define TK_NULLS 234 -#define TK_ID 235 -#define TK_NK_BITNOT 236 -#define TK_INSERT 237 -#define TK_VALUES 238 -#define TK_IMPORT 239 -#define TK_NK_SEMI 240 -#define TK_FILE 241 +#define TK_WITH 149 +#define TK_META 150 +#define TK_CONSUMER 151 +#define TK_GROUP 152 +#define TK_DESC 153 +#define TK_DESCRIBE 154 +#define TK_RESET 155 +#define TK_QUERY 156 +#define TK_CACHE 157 +#define TK_EXPLAIN 158 +#define TK_ANALYZE 159 +#define TK_VERBOSE 160 +#define TK_NK_BOOL 161 +#define TK_RATIO 162 +#define TK_NK_FLOAT 163 +#define TK_COMPACT 164 +#define TK_VNODES 165 +#define TK_IN 166 +#define TK_OUTPUTTYPE 167 +#define TK_AGGREGATE 168 +#define TK_BUFSIZE 169 +#define TK_STREAM 170 +#define TK_INTO 171 +#define TK_TRIGGER 172 +#define TK_AT_ONCE 173 +#define TK_WINDOW_CLOSE 174 +#define TK_KILL 175 +#define TK_CONNECTION 176 +#define TK_TRANSACTION 177 +#define TK_BALANCE 178 +#define TK_VGROUP 179 +#define TK_MERGE 180 +#define TK_REDISTRIBUTE 181 +#define TK_SPLIT 182 +#define TK_SYNCDB 183 +#define TK_DELETE 184 +#define TK_NULL 185 +#define TK_NK_QUESTION 186 +#define TK_NK_ARROW 187 +#define TK_ROWTS 188 +#define TK_TBNAME 189 +#define TK_QSTARTTS 190 +#define TK_QENDTS 191 +#define TK_WSTARTTS 192 +#define TK_WENDTS 193 +#define TK_WDURATION 194 +#define TK_CAST 195 +#define TK_NOW 196 +#define TK_TODAY 197 +#define TK_TIMEZONE 198 +#define TK_COUNT 199 +#define TK_LAST_ROW 200 +#define TK_BETWEEN 201 +#define TK_IS 202 +#define TK_NK_LT 203 +#define TK_NK_GT 204 +#define TK_NK_LE 205 +#define TK_NK_GE 206 +#define TK_NK_NE 207 +#define TK_MATCH 208 +#define TK_NMATCH 209 +#define TK_CONTAINS 210 +#define TK_JOIN 211 +#define TK_INNER 212 +#define TK_SELECT 213 +#define TK_DISTINCT 214 +#define TK_WHERE 215 +#define TK_PARTITION 216 +#define TK_BY 217 +#define TK_SESSION 218 +#define TK_STATE_WINDOW 219 +#define TK_SLIDING 220 +#define TK_FILL 221 +#define TK_VALUE 222 +#define TK_NONE 223 +#define TK_PREV 224 +#define TK_LINEAR 225 +#define TK_NEXT 226 +#define TK_HAVING 227 +#define TK_RANGE 228 +#define TK_EVERY 229 +#define TK_ORDER 230 +#define TK_SLIMIT 231 +#define TK_SOFFSET 232 +#define TK_LIMIT 233 +#define TK_OFFSET 234 +#define TK_ASC 235 +#define TK_NULLS 236 +#define TK_ID 237 +#define TK_NK_BITNOT 238 +#define TK_INSERT 239 +#define TK_VALUES 240 +#define TK_IMPORT 241 +#define TK_NK_SEMI 242 +#define TK_FILE 243 #define TK_NK_SPACE 300 #define TK_NK_COMMENT 301 diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index f1cfa58f62f8b80dc7f3725fcba6e0d3395e3d39..2509c106015682ff38c5de3d1965fa7b97d0d7a7 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -95,8 +95,8 @@ int32_t mndGetLoad(SMnode *pMnode, SMnodeLoad *pLoad); */ int32_t mndProcessRpcMsg(SRpcMsg *pMsg); int32_t mndProcessSyncMsg(SRpcMsg *pMsg); -int32_t mndPreProcessMsg(SRpcMsg *pMsg); -void mndAbortPreprocessMsg(SRpcMsg *pMsg); +int32_t mndPreProcessQueryMsg(SRpcMsg *pMsg); +void mndPostProcessQueryMsg(SRpcMsg *pMsg); /** * @brief Generate machine code diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index a7ac767883a74548755b110c4ab6261075d3a443..20f4765190bb434c363c8cd2d2e35a7c0716bdfe 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -68,7 +68,9 @@ typedef struct SCatalogReq { SArray* pIndex; // element is index name SArray* pUser; // element is SUserAuthInfo SArray* pTableIndex; // element is SNAME + SArray* pTableCfg; // element is SNAME bool qNodeRequired; // valid qnode + bool dNodeRequired; // valid dnode bool forceUpdate; } SCatalogReq; @@ -88,6 +90,8 @@ typedef struct SMetaData { SArray* pIndex; // pRes = SIndexInfo* SArray* pUser; // pRes = bool* SArray* pQnodeList; // pRes = SArray* + SArray* pTableCfg; // pRes = STableCfg* + SArray* pDnodeList; // pRes = SArray* } SMetaData; typedef struct SCatalogCfg { @@ -268,6 +272,8 @@ int32_t catalogAsyncGetAllMeta(SCatalog* pCtg, SRequestConnInfo* pConn, uint64_t int32_t catalogGetQnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray* pQnodeList); +int32_t catalogGetDnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray** pDnodeList); + int32_t catalogGetExpiredSTables(SCatalog* pCatalog, SSTableVersion **stables, uint32_t *num); int32_t catalogGetExpiredDBs(SCatalog* pCatalog, SDbVgVersion** dbs, uint32_t* num); @@ -280,6 +286,8 @@ int32_t catalogGetIndexMeta(SCatalog* pCtg, SRequestConnInfo* pConn, const char* int32_t catalogGetTableIndex(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SArray** pRes); +int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg** pCfg); + int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp *pRsp); int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index c23cf162aa6e1ca35e3750aa54e91659c45d6b08..957c40f21e455a91cb916229565760782bd15d31 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -33,7 +33,7 @@ struct SDataSink; struct SSDataBlock; typedef struct SDeleterRes { - uint64_t uid; + uint64_t suid; SArray* uidList; int64_t skey; int64_t ekey; @@ -41,7 +41,8 @@ typedef struct SDeleterRes { } SDeleterRes; typedef struct SDeleterParam { - SArray* pUidList; + uint64_t suid; + SArray* pUidList; } SDeleterParam; typedef struct SDataSinkStat { diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 083f6ae1b07d4db9e7f4d339b63b8b89c0a8becc..6738fc23bc474270119a78fa67c4c777c8c696cc 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -36,11 +36,13 @@ typedef struct SReadHandle { void* vnode; void* mnd; SMsgCb* pMsgCb; + int8_t initTsdbReader; } SReadHandle; enum { STREAM_DATA_TYPE_SUBMIT_BLOCK = 1, STREAM_DATA_TYPE_SSDATA_BLOCK = 2, + STREAM_DATA_TYPE_FROM_SNAPSHOT = 3, }; typedef enum { @@ -56,6 +58,13 @@ typedef enum { */ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle); +/** + * Switch the stream scan to snapshot mode + * @param tinfo + * @return + */ +int32_t qStreamScanSnapshot(qTaskInfo_t tinfo); + /** * Set the input data block for the stream scan. * @param tinfo diff --git a/include/libs/function/function.h b/include/libs/function/function.h index 0d8ae289cc4def348556adc5549e948cff7fcda6..a569c8de5407418ec2da037a889f87132a722751 100644 --- a/include/libs/function/function.h +++ b/include/libs/function/function.h @@ -67,7 +67,7 @@ typedef struct SResultRowEntryInfo { bool initialized:1; // output buffer has been initialized bool complete:1; // query has completed uint8_t isNullRes:6; // the result is null - uint8_t numOfRes; // num of output result in current buffer + uint16_t numOfRes; // num of output result in current buffer } SResultRowEntryInfo; // determine the real data need to calculated the result @@ -172,23 +172,6 @@ typedef struct tExprNode { void tExprTreeDestroy(tExprNode *pNode, void (*fp)(void *)); -typedef struct SAggFunctionInfo { - char name[FUNCTIONS_NAME_MAX_LENGTH]; - int8_t type; // Scalar function or aggregation function - uint32_t functionId; // Function Id - int8_t sFunctionId; // Transfer function for super table query - uint16_t status; - - bool (*init)(SqlFunctionCtx *pCtx, struct SResultRowEntryInfo* pResultCellInfo); // setup the execute environment - void (*addInput)(SqlFunctionCtx *pCtx); - - // finalizer must be called after all exec has been executed to generated final result. - void (*finalize)(SqlFunctionCtx *pCtx); - void (*combine)(SqlFunctionCtx *pCtx); - - int32_t (*dataReqFunc)(SqlFunctionCtx *pCtx, STimeWindow* w, int32_t colId); -} SAggFunctionInfo; - struct SScalarParam { SColumnInfoData *columnData; SHashObj *pHashFilter; diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index cbaff29cb2de5ab4e93d39caa1cd89810fbdddde..1f314d19e7da4863f8ec7ab3e2d2f45169b9babb 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -190,6 +190,7 @@ bool fmIsForbidWindowFunc(int32_t funcId); bool fmIsForbidGroupByFunc(int32_t funcId); bool fmIsIntervalInterpoFunc(int32_t funcId); bool fmIsInterpFunc(int32_t funcId); +bool fmIsLastRowFunc(int32_t funcId); int32_t fmGetDistMethod(const SFunctionNode* pFunc, SFunctionNode** pPartialFunc, SFunctionNode** pMergeFunc); diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 13a8dbd7e3e4dc86602aeaff3cb2ee6f0924be9a..4f18cb19c2802f2811354ca15af227878c29ab00 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -28,6 +28,19 @@ extern "C" { #define DESCRIBE_RESULT_TYPE_LEN (20 + VARSTR_HEADER_SIZE) #define DESCRIBE_RESULT_NOTE_LEN (8 + VARSTR_HEADER_SIZE) +#define SHOW_CREATE_DB_RESULT_COLS 2 +#define SHOW_CREATE_DB_RESULT_FIELD1_LEN (TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE) +#define SHOW_CREATE_DB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN + VARSTR_HEADER_SIZE) + +#define SHOW_CREATE_TB_RESULT_COLS 2 +#define SHOW_CREATE_TB_RESULT_FIELD1_LEN (TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE) +#define SHOW_CREATE_TB_RESULT_FIELD2_LEN (TSDB_MAX_BINARY_LEN + VARSTR_HEADER_SIZE) + +#define SHOW_LOCAL_VARIABLES_RESULT_COLS 2 +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN (TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE) +#define SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN (TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE) + + #define PRIVILEGE_TYPE_MASK(n) (1 << n) #define PRIVILEGE_TYPE_ALL PRIVILEGE_TYPE_MASK(0) @@ -119,14 +132,14 @@ typedef struct SCreateTableStmt { } SCreateTableStmt; typedef struct SCreateSubTableClause { - ENodeType type; - char dbName[TSDB_DB_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; - char useDbName[TSDB_DB_NAME_LEN]; - char useTableName[TSDB_TABLE_NAME_LEN]; - bool ignoreExists; - SNodeList* pSpecificTags; - SNodeList* pValsOfTags; + ENodeType type; + char dbName[TSDB_DB_NAME_LEN]; + char tableName[TSDB_TABLE_NAME_LEN]; + char useDbName[TSDB_DB_NAME_LEN]; + char useTableName[TSDB_TABLE_NAME_LEN]; + bool ignoreExists; + SNodeList* pSpecificTags; + SNodeList* pValsOfTags; STableOptions* pOptions; } SCreateSubTableClause; @@ -170,13 +183,16 @@ typedef struct SCreateUserStmt { ENodeType type; char useName[TSDB_USER_LEN]; char password[TSDB_USET_PASSWORD_LEN]; + int8_t sysinfo; } SCreateUserStmt; typedef struct SAlterUserStmt { ENodeType type; char useName[TSDB_USER_LEN]; - char password[TSDB_USET_PASSWORD_LEN]; int8_t alterType; + char password[TSDB_USET_PASSWORD_LEN]; + int8_t enable; + int8_t sysinfo; } SAlterUserStmt; typedef struct SDropUserStmt { @@ -221,7 +237,7 @@ typedef struct SShowCreateTableStmt { ENodeType type; char dbName[TSDB_DB_NAME_LEN]; char tableName[TSDB_TABLE_NAME_LEN]; - STableMeta* pMeta; + void* pCfg; // STableCfg } SShowCreateTableStmt; typedef struct SShowTableDistributedStmt { @@ -230,6 +246,11 @@ typedef struct SShowTableDistributedStmt { char tableName[TSDB_TABLE_NAME_LEN]; } SShowTableDistributedStmt; +typedef struct SShowDnodeVariablesStmt { + ENodeType type; + SNode* pDnodeId; +} SShowDnodeVariablesStmt; + typedef enum EIndexType { INDEX_TYPE_SMA = 1, INDEX_TYPE_FULLTEXT } EIndexType; typedef struct SIndexOptions { @@ -254,7 +275,6 @@ typedef struct SDropIndexStmt { ENodeType type; bool ignoreNotExists; char indexName[TSDB_INDEX_NAME_LEN]; - char tableName[TSDB_TABLE_NAME_LEN]; } SDropIndexStmt; typedef struct SCreateComponentNodeStmt { @@ -273,6 +293,7 @@ typedef struct SCreateTopicStmt { char subDbName[TSDB_DB_NAME_LEN]; char subSTbName[TSDB_TABLE_NAME_LEN]; bool ignoreExists; + bool withMeta; SNode* pQuery; } SCreateTopicStmt; diff --git a/include/libs/nodes/nodes.h b/include/libs/nodes/nodes.h index 975321cd5be179f411694430109a8da933addc67..5f89d1f6650a2b8618565bc54dea329f535f8c89 100644 --- a/include/libs/nodes/nodes.h +++ b/include/libs/nodes/nodes.h @@ -59,10 +59,10 @@ extern "C" { for (SListCell* cell = (NULL != (list) ? (list)->pHead : NULL); \ (NULL != cell ? (node = &(cell->pNode), true) : (node = NULL, false)); cell = cell->pNext) -#define DESTORY_LIST(list) \ - do { \ - nodesDestroyList((list)); \ - (list) = NULL; \ +#define NODES_DESTORY_LIST(list) \ + do { \ + nodesDestroyList((list)); \ + (list) = NULL; \ } while (0) #define NODES_CLEAR_LIST(list) \ @@ -180,7 +180,9 @@ typedef enum ENodeType { QUERY_NODE_SHOW_VNODES_STMT, QUERY_NODE_SHOW_APPS_STMT, QUERY_NODE_SHOW_SCORES_STMT, - QUERY_NODE_SHOW_VARIABLE_STMT, + QUERY_NODE_SHOW_VARIABLES_STMT, + QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT, + QUERY_NODE_SHOW_DNODE_VARIABLES_STMT, QUERY_NODE_SHOW_CREATE_DATABASE_STMT, QUERY_NODE_SHOW_CREATE_TABLE_STMT, QUERY_NODE_SHOW_CREATE_STABLE_STMT, @@ -217,14 +219,17 @@ typedef enum ENodeType { QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN, QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN, + QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, QUERY_NODE_PHYSICAL_PLAN_PROJECT, QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN, QUERY_NODE_PHYSICAL_PLAN_HASH_AGG, QUERY_NODE_PHYSICAL_PLAN_EXCHANGE, QUERY_NODE_PHYSICAL_PLAN_MERGE, QUERY_NODE_PHYSICAL_PLAN_SORT, + QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT, QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL, + QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL, QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL, diff --git a/include/libs/nodes/plannodes.h b/include/libs/nodes/plannodes.h index b07e8f39d5bcae86ebd32208613f98c7e2bff497..5559cb80607b6daf741b409525039e394018aaae 100644 --- a/include/libs/nodes/plannodes.h +++ b/include/libs/nodes/plannodes.h @@ -40,7 +40,8 @@ typedef enum EScanType { SCAN_TYPE_SYSTEM_TABLE, SCAN_TYPE_STREAM, SCAN_TYPE_TABLE_MERGE, - SCAN_TYPE_BLOCK_INFO + SCAN_TYPE_BLOCK_INFO, + SCAN_TYPE_LAST_ROW } EScanType; typedef struct SScanLogicNode { @@ -65,6 +66,7 @@ typedef struct SScanLogicNode { int8_t intervalUnit; int8_t slidingUnit; SNode* pTagCond; + SNode* pTagIndexCond; int8_t triggerType; int64_t watermark; int16_t tsColId; @@ -260,6 +262,7 @@ typedef struct SScanPhysiNode { typedef SScanPhysiNode STagScanPhysiNode; typedef SScanPhysiNode SBlockDistScanPhysiNode; +typedef SScanPhysiNode SLastRowScanPhysiNode; typedef struct SSystemTableScanPhysiNode { SScanPhysiNode scan; @@ -375,7 +378,8 @@ typedef struct SIntervalPhysiNode { int8_t slidingUnit; } SIntervalPhysiNode; -typedef SIntervalPhysiNode SMergeIntervalPhysiNode; +typedef SIntervalPhysiNode SMergeIntervalPhysiNode; +typedef SIntervalPhysiNode SMergeAlignedIntervalPhysiNode; typedef SIntervalPhysiNode SStreamIntervalPhysiNode; typedef SIntervalPhysiNode SStreamFinalIntervalPhysiNode; typedef SIntervalPhysiNode SStreamSemiIntervalPhysiNode; @@ -417,6 +421,8 @@ typedef struct SSortPhysiNode { SNodeList* pTargets; } SSortPhysiNode; +typedef SSortPhysiNode SGroupSortPhysiNode; + typedef struct SPartitionPhysiNode { SPhysiNode node; SNodeList* pExprs; // these are expression list of partition_by_clause @@ -463,6 +469,7 @@ typedef struct SSubplan { SPhysiNode* pNode; // physical plan of current subplan SDataSinkNode* pDataSink; // data of the subplan flow into the datasink SNode* pTagCond; + SNode* pTagIndexCond; } SSubplan; typedef enum EExplainMode { EXPLAIN_MODE_DISABLE = 1, EXPLAIN_MODE_STATIC, EXPLAIN_MODE_ANALYZE } EExplainMode; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 73e487f073ac44d5ddeee0086da203f329b3238d..a6e466e73e506ec347ecb6487f82ff6b468ede45 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -52,7 +52,7 @@ typedef struct SExprNode { SArray* pAssociation; } SExprNode; -typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG } EColumnType; +typedef enum EColumnType { COLUMN_TYPE_COLUMN = 1, COLUMN_TYPE_TAG, COLUMN_TYPE_TBNAME } EColumnType; typedef struct SColumnNode { SExprNode node; // QUERY_NODE_COLUMN @@ -258,6 +258,7 @@ typedef struct SSelectStmt { bool hasUniqueFunc; bool hasTailFunc; bool hasInterpFunc; + bool hasLastRowFunc; } SSelectStmt; typedef enum ESetOperatorType { SET_OP_TYPE_UNION_ALL = 1, SET_OP_TYPE_UNION } ESetOperatorType; @@ -287,11 +288,11 @@ typedef enum ESqlClause { } ESqlClause; typedef struct SDeleteStmt { - ENodeType type; // QUERY_NODE_DELETE_STMT - SNode* pFromTable; // FROM clause - SNode* pWhere; // WHERE clause - SNode* pCountFunc; // count the number of rows affected - SNode* pTagIndexCond; // pWhere divided into pTagIndexCond and timeRange + ENodeType type; // QUERY_NODE_DELETE_STMT + SNode* pFromTable; // FROM clause + SNode* pWhere; // WHERE clause + SNode* pCountFunc; // count the number of rows affected + SNode* pTagCond; // pWhere divided into pTagCond and timeRange STimeWindow timeRange; uint8_t precision; bool deleteZeroRows; @@ -384,6 +385,7 @@ bool nodesIsArithmeticOp(const SOperatorNode* pOp); bool nodesIsComparisonOp(const SOperatorNode* pOp); bool nodesIsJsonOp(const SOperatorNode* pOp); bool nodesIsRegularOp(const SOperatorNode* pOp); +bool nodesIsBitwiseOp(const SOperatorNode* pOp); bool nodesExprHasColumn(SNode* pNode); bool nodesExprsHasColumn(SNodeList* pList); @@ -395,7 +397,8 @@ void nodesValueNodeToVariant(const SValueNode* pNode, SVariant* pVal); char* nodesGetFillModeString(EFillMode mode); int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc); -int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond); +int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond, + SNode** pOtherCond); #ifdef __cplusplus } diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index 0780343c64f7f1c967113e684c5bd87ab1a33938..9e8ce3ffb6d5b2149ff859ee288d57ee7f6c1907 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -207,7 +207,12 @@ int32_t queryCreateTableMetaFromMsg(STableMetaRsp* msg, bool isSuperTable, STabl char* jobTaskStatusStr(int32_t status); SSchema createSchema(int8_t type, int32_t bytes, col_id_t colId, const char* name); + void destroyQueryExecRes(SQueryExecRes* pRes); +int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len); +char* parseTagDatatoJson(void* p); +int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst); +int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst); extern int32_t (*queryBuildMsg[TDMT_MAX])(void* input, char** msg, int32_t msgSize, int32_t* msgLen, void* (*mallocFp)(int32_t)); diff --git a/include/libs/qworker/qworker.h b/include/libs/qworker/qworker.h index 3b8a37f4208386eb06ff4f3153fab271e3ddb011..36e9b3309c6dd1c2827b9e03ddbbeb80c721f797 100644 --- a/include/libs/qworker/qworker.h +++ b/include/libs/qworker/qworker.h @@ -32,7 +32,7 @@ enum { }; typedef struct SDeleteRes { - uint64_t uid; + uint64_t suid; SArray* uidList; int64_t skey; int64_t ekey; diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index 937ac2b408ed0e720d24ae648f920352c7eb5712..7673e730356fcb75cfa38f41521b5c41d596d6e6 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -30,9 +30,14 @@ extern "C" { typedef struct SStreamTask SStreamTask; enum { - TASK_STATUS__IDLE = 1, - TASK_STATUS__EXECUTING, - TASK_STATUS__CLOSING, + TASK_STATUS__NORMAL = 0, + TASK_STATUS__DROPPING, +}; + +enum { + TASK_EXEC_STATUS__IDLE = 1, + TASK_EXEC_STATUS__EXECUTING, + TASK_EXEC_STATUS__CLOSING, }; enum { @@ -50,16 +55,12 @@ enum { TASK_OUTPUT_STATUS__BLOCKED, }; -enum { - STREAM_CREATED_BY__USER = 1, - STREAM_CREATED_BY__SMA, -}; - enum { STREAM_INPUT__DATA_SUBMIT = 1, STREAM_INPUT__DATA_BLOCK, STREAM_INPUT__TRIGGER, STREAM_INPUT__CHECKPOINT, + STREAM_INPUT__DROP, }; typedef struct { @@ -76,7 +77,7 @@ typedef struct { typedef struct { int8_t type; - int32_t sourceVg; + int32_t srcVgId; int64_t sourceVer; SArray* blocks; // SArray @@ -144,11 +145,6 @@ void streamDataSubmitRefDec(SStreamDataSubmit* pDataSubmit); SStreamDataSubmit* streamSubmitRefClone(SStreamDataSubmit* pSubmit); -#if 0 -int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput); -void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput); -#endif - typedef struct { char* qmsg; // followings are not applicable to encoder and decoder @@ -233,24 +229,33 @@ enum { TASK_TRIGGER_STATUS__ACTIVE, }; +typedef struct { + int32_t nodeId; + int32_t childId; + int32_t taskId; + SEpSet epSet; +} SStreamChildEpInfo; + struct SStreamTask { int64_t streamId; int32_t taskId; - int8_t inputType; - int8_t status; - + int8_t isDataScan; int8_t execType; int8_t sinkType; int8_t dispatchType; int16_t dispatchMsgType; - int8_t dataScan; + int8_t taskStatus; + int8_t execStatus; // node info - int32_t childId; + int32_t selfChildId; int32_t nodeId; SEpSet epSet; + // children info + SArray* childEpInfo; // SArray + // exec STaskExec exec; @@ -288,6 +293,9 @@ struct SStreamTask { SMsgCb* pMsgCb; }; +int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); +int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); + SStreamTask* tNewSStreamTask(int64_t streamId); int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); @@ -366,9 +374,9 @@ typedef struct { typedef struct { int64_t streamId; int32_t taskId; - int32_t sourceTaskId; - int32_t sourceVg; - int32_t sourceChildId; + int32_t dataSrcVgId; + int32_t upstreamTaskId; + int32_t upstreamChildId; int32_t upstreamNodeId; #if 0 int64_t sourceVer; @@ -384,6 +392,23 @@ typedef struct { int8_t inputStatus; } SStreamDispatchRsp; +typedef struct { + int64_t streamId; + int32_t srcTaskId; + int32_t srcNodeId; + int32_t dstTaskId; + int32_t dstNodeId; + int32_t retrieveLen; + SRetrieveTableRsp* pRetrieve; +} SStreamRetrieveReq; + +typedef struct { + int64_t streamId; + int32_t childId; + int32_t rspFromTaskId; + int32_t rspToTaskId; +} SStreamRetrieveRsp; + typedef struct { int64_t streamId; int32_t taskId; @@ -398,6 +423,7 @@ typedef struct { } SStreamTaskRecoverRsp; int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq); +int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq); int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId); int32_t streamSetupTrigger(SStreamTask* pTask); @@ -408,6 +434,9 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp); int32_t streamProcessRecoverReq(SStreamTask* pTask, SStreamTaskRecoverReq* pReq, SRpcMsg* pMsg); int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp); +int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); +int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); + #ifdef __cplusplus } #endif diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index e963f25616cc92ebb64a75c9de6daecca50c8116..a0c07f0c09fde9236fd7d5cf65b2596727261414 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -26,8 +26,9 @@ extern "C" { extern bool gRaftDetailLog; -#define SYNC_INDEX_BEGIN 0 +#define SYNC_INDEX_BEGIN 0 #define SYNC_INDEX_INVALID -1 +#define SYNC_TERM_INVALID 0xFFFFFFFFFFFFFFFF typedef uint64_t SyncNodeId; typedef int32_t SyncGroupId; @@ -162,7 +163,7 @@ typedef struct SSyncLogStore { SyncIndex (*syncLogEndIndex)(struct SSyncLogStore* pLogStore); bool (*syncLogIsEmpty)(struct SSyncLogStore* pLogStore); int32_t (*syncLogEntryCount)(struct SSyncLogStore* pLogStore); - bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); + // bool (*syncLogInRange)(struct SSyncLogStore* pLogStore, SyncIndex index); SyncIndex (*syncLogWriteIndex)(struct SSyncLogStore* pLogStore); SyncIndex (*syncLogLastIndex)(struct SSyncLogStore* pLogStore); @@ -199,7 +200,7 @@ const char* syncGetMyRoleStr(int64_t rid); SyncTerm syncGetMyTerm(int64_t rid); SyncGroupId syncGetVgId(int64_t rid); void syncGetEpSet(int64_t rid, SEpSet* pEpSet); -int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak); +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak); bool syncEnvIsStart(); const char* syncStr(ESyncState state); bool syncIsRestoreFinish(int64_t rid); diff --git a/include/libs/sync/syncTools.h b/include/libs/sync/syncTools.h index 37b465e56ea18420cdb4880099c4b2f8bdf4c322..e51b20a40435e67152f11b19a566bc298e11911d 100644 --- a/include/libs/sync/syncTools.h +++ b/include/libs/sync/syncTools.h @@ -43,7 +43,7 @@ void setElectTimerMS(int64_t rid, int32_t electTimerMS); void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS); // for compatibility, the same as syncPropose -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak); +int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak); // utils const char* syncUtilState2String(ESyncState state); @@ -468,7 +468,7 @@ typedef struct SyncLeaderTransfer { SRaftId destId; */ SNodeInfo newNodeInfo; - SRaftId newLeaderId; + SRaftId newLeaderId; } SyncLeaderTransfer; SyncLeaderTransfer* syncLeaderTransferBuild(int32_t vgId); @@ -489,17 +489,16 @@ void syncLeaderTransferPrint2(char* s, const SyncLeaderTransfer* pMsg); void syncLeaderTransferLog(const SyncLeaderTransfer* pMsg); void syncLeaderTransferLog2(char* s, const SyncLeaderTransfer* pMsg); - // --------------------------------------------- typedef struct SyncReconfigFinish { - uint32_t bytes; - int32_t vgId; - uint32_t msgType; - SSyncCfg oldCfg; - SSyncCfg newCfg; + uint32_t bytes; + int32_t vgId; + uint32_t msgType; + SSyncCfg oldCfg; + SSyncCfg newCfg; SyncIndex newCfgIndex; - SyncTerm newCfgTerm; - uint64_t newCfgSeqNum; + SyncTerm newCfgTerm; + uint64_t newCfgSeqNum; } SyncReconfigFinish; @@ -521,8 +520,6 @@ void syncReconfigFinishPrint2(char* s, const SyncReconfigFinish* pMsg); void syncReconfigFinishLog(const SyncReconfigFinish* pMsg); void syncReconfigFinishLog2(char* s, const SyncReconfigFinish* pMsg); - - // on message ---------------------- int32_t syncNodeOnPingCb(SSyncNode* ths, SyncPing* pMsg); int32_t syncNodeOnPingReplyCb(SSyncNode* ths, SyncPingReply* pMsg); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index c7d1ccd3de8f6e2e921cf605c96d6fa2b1fc231f..92701db2adc5b9dc81d908a7dfcb03cac5706b0c 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -195,7 +195,6 @@ void walCloseReadHandle(SWalReadHandle *); int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver); // only for tq usage -// int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **ppHead); void walSetReaderCapacity(SWalReadHandle *pRead, int32_t capacity); int32_t walFetchHead(SWalReadHandle *pRead, int64_t ver, SWalHead *pHead); int32_t walFetchBody(SWalReadHandle *pRead, SWalHead **ppHead); @@ -211,13 +210,8 @@ void walCloseRef(SWalRef *); int32_t walRefVer(SWalRef *, int64_t ver); int32_t walUnrefVer(SWal *); -// deprecated -#if 0 -int32_t walRead(SWal *, SWalHead **, int64_t ver); -int32_t walReadWithFp(SWal *, FWalWrite writeFp, int64_t verStart, int32_t readNum); -#endif - // lifecycle check +bool walIsEmpty(SWal *); int64_t walGetFirstVer(SWal *); int64_t walGetSnapshotVer(SWal *); int64_t walGetLastVer(SWal *); diff --git a/include/os/os.h b/include/os/os.h index 41180ba49edf6ecee2482ab70cc419983bd11d8d..254c16efbe598a68c549a6c09e36b077929281bf 100644 --- a/include/os/os.h +++ b/include/os/os.h @@ -104,8 +104,6 @@ extern "C" { #include "osTimezone.h" #include "osEnv.h" -void osDefaultInit(); - #ifdef __cplusplus } #endif diff --git a/include/os/osTime.h b/include/os/osTime.h index 949c15ed0d7c7a4e4c317edaace21f29a9eaab36..965ec61671925dd4d9e22e07a58826d47ecd9ed3 100644 --- a/include/os/osTime.h +++ b/include/os/osTime.h @@ -38,7 +38,7 @@ extern "C" { #define MILLISECOND_PER_SECOND (1000i64) #else -#define MILLISECOND_PER_SECOND ((int64_t)1000L) +#define MILLISECOND_PER_SECOND ((int64_t)1000LL) #endif #define MILLISECOND_PER_MINUTE (MILLISECOND_PER_SECOND * 60) @@ -46,9 +46,9 @@ extern "C" { #define MILLISECOND_PER_DAY (MILLISECOND_PER_HOUR * 24) #define MILLISECOND_PER_WEEK (MILLISECOND_PER_DAY * 7) -#define NANOSECOND_PER_USEC (1000L) -#define NANOSECOND_PER_MSEC (1000000L) -#define NANOSECOND_PER_SEC (1000000000L) +#define NANOSECOND_PER_USEC (1000LL) +#define NANOSECOND_PER_MSEC (1000000LL) +#define NANOSECOND_PER_SEC (1000000000LL) #define NANOSECOND_PER_MINUTE (NANOSECOND_PER_SEC * 60) #define NANOSECOND_PER_HOUR (NANOSECOND_PER_MINUTE * 60) #define NANOSECOND_PER_DAY (NANOSECOND_PER_HOUR * 24) @@ -65,21 +65,21 @@ int32_t taosGetTimestampSec(); static FORCE_INLINE int64_t taosGetTimestampMs() { struct timeval systemTime; taosGetTimeOfDay(&systemTime); - return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec / 1000; + return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec / 1000; } //@return timestamp in microsecond static FORCE_INLINE int64_t taosGetTimestampUs() { struct timeval systemTime; taosGetTimeOfDay(&systemTime); - return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; + return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec; } //@return timestamp in nanosecond static FORCE_INLINE int64_t taosGetTimestampNs() { struct timespec systemTime = {0}; taosClockGetTime(CLOCK_REALTIME, &systemTime); - return (int64_t)systemTime.tv_sec * 1000000000L + (int64_t)systemTime.tv_nsec; + return (int64_t)systemTime.tv_sec * 1000000000LL + (int64_t)systemTime.tv_nsec; } char * taosStrpTime(const char *buf, const char *fmt, struct tm *tm); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index f4d8c2a663f1f4214434af7e38facd767399e6a1..8af594530058500dcb9e51bc7146ce96af55bab6 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -71,6 +71,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE TAOS_DEF_ERROR_CODE(0, 0x0029) #define TSDB_CODE_INVALID_TIMESTAMP TAOS_DEF_ERROR_CODE(0, 0x0030) #define TSDB_CODE_MSG_DECODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x0031) +#define TSDB_CODE_NO_AVAIL_DISK TAOS_DEF_ERROR_CODE(0, 0x0032) #define TSDB_CODE_REF_NO_MEMORY TAOS_DEF_ERROR_CODE(0, 0x0040) #define TSDB_CODE_REF_FULL TAOS_DEF_ERROR_CODE(0, 0x0041) @@ -129,6 +130,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TSC_STMT_CLAUSE_ERROR TAOS_DEF_ERROR_CODE(0, 0X0227) #define TSDB_CODE_TSC_QUERY_KILLED TAOS_DEF_ERROR_CODE(0, 0X0228) #define TSDB_CODE_TSC_NO_EXEC_NODE TAOS_DEF_ERROR_CODE(0, 0X0229) +#define TSDB_CODE_TSC_NOT_STABLE_ERROR TAOS_DEF_ERROR_CODE(0, 0X022a) // mnode-common #define TSDB_CODE_MND_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x0300) @@ -419,6 +421,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SYN_NEW_CONFIG_ERROR TAOS_DEF_ERROR_CODE(0, 0x090F) #define TSDB_CODE_SYN_RECONFIG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0910) #define TSDB_CODE_SYN_PROPOSE_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0911) +#define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq diff --git a/include/util/tconfig.h b/include/util/tconfig.h index 06fa9fd9aaf351f59b761a57cc96a4f3529ecf91..fd6360aaf682f6549bd02de6ce9489c3efdd9831 100644 --- a/include/util/tconfig.h +++ b/include/util/tconfig.h @@ -104,6 +104,8 @@ int32_t cfgAddTimezone(SConfig *pCfg, const char *name, const char *defaultVal); const char *cfgStypeStr(ECfgSrcType type); const char *cfgDtypeStr(ECfgDataType type); +void cfgDumpItemValue(SConfigItem *pItem, char* buf, int32_t bufSize, int32_t* pLen); + void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump); int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl); diff --git a/include/util/tdef.h b/include/util/tdef.h index c5a8b95a08cc37e4d665e01f5de01eca97b16087..5befa6a67fc1b6b29b3ad74eaff6466b0672bd2e 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -49,12 +49,12 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_DATA_BOOL_NULL 0x02 #define TSDB_DATA_TINYINT_NULL 0x80 #define TSDB_DATA_SMALLINT_NULL 0x8000 -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_INT_NULL 0x80000000LL +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL #define TSDB_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define TSDB_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define TSDB_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN #define TSDB_DATA_NCHAR_NULL 0xFFFFFFFF #define TSDB_DATA_BINARY_NULL 0xFF @@ -108,8 +108,8 @@ extern const int32_t TYPE_BYTES[16]; #define TSDB_INS_USER_STABLES_DBNAME_COLID 2 #define TSDB_TICK_PER_SECOND(precision) \ - ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1e3L \ - : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1e6L : 1e9L))) + ((int64_t)((precision) == TSDB_TIME_PRECISION_MILLI ? 1000LL \ + : ((precision) == TSDB_TIME_PRECISION_MICRO ? 1000000LL : 1000000000LL))) #define T_MEMBER_SIZE(type, member) sizeof(((type *)0)->member) #define T_APPEND_MEMBER(dst, ptr, type, member) \ @@ -134,7 +134,7 @@ typedef enum EOperatorType { OP_TYPE_MINUS, OP_TYPE_ASSIGN, - // bit operator + // bitwise operator OP_TYPE_BIT_AND, OP_TYPE_BIT_OR, @@ -443,8 +443,8 @@ enum { #define VNODE_HANDLE -3 #define BNODE_HANDLE -4 -#define TSDB_CONFIG_OPTION_LEN 16 -#define TSDB_CONIIG_VALUE_LEN 48 +#define TSDB_CONFIG_OPTION_LEN 32 +#define TSDB_CONFIG_VALUE_LEN 64 #define TSDB_CONFIG_NUMBER 8 #define QUERY_ID_SIZE 20 diff --git a/include/util/tpagedbuf.h b/include/util/tpagedbuf.h index af82e29ec500aec1d5519d59cb5f184bf736d0e4..ef266068cbaff046ec6ebcf0bf02d0b44ee9d3a2 100644 --- a/include/util/tpagedbuf.h +++ b/include/util/tpagedbuf.h @@ -28,7 +28,7 @@ typedef struct SArray* SIDList; typedef struct SPageInfo SPageInfo; typedef struct SDiskbasedBuf SDiskbasedBuf; -#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024L) // in bytes +#define DEFAULT_INTERN_BUF_PAGE_SIZE (1024LL) // in bytes typedef struct SFilePage { int32_t num; diff --git a/include/util/types.h b/include/util/types.h index d48995418e530646e8ef540bfb0630fc90cd9636..ded9dc37d7163f833b5f6959656617c7533b195c 100644 --- a/include/util/types.h +++ b/include/util/types.h @@ -32,7 +32,7 @@ extern "C" { #define GET_UINT64_VAL(x) (*(uint64_t *)(x)) static FORCE_INLINE float taos_align_get_float(const char *pBuf) { -#if __STDC_VERSION__ >= 201112L +#if __STDC_VERSION__ >= 201112LL static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); #else assert(sizeof(float) == sizeof(uint32_t)); @@ -43,7 +43,7 @@ static FORCE_INLINE float taos_align_get_float(const char *pBuf) { } static FORCE_INLINE double taos_align_get_double(const char *pBuf) { -#if __STDC_VERSION__ >= 201112L +#if __STDC_VERSION__ >= 201112LL static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)"); #else assert(sizeof(double) == sizeof(uint64_t)); diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index f07705ff442b65f1295431e59b48ef50a76cadc0..d730cc0d8af8e3db0c16e4609138c0f63ac5329b 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -196,7 +196,6 @@ function install_bin() { ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}rm -f ${bin_link_dir}/tarbitrator || : ${csudo}rm -f ${bin_link_dir}/set_core || : - ${csudo}rm -f ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || : ${csudo}rm -f ${bin_link_dir}/TDinsight.sh || : ${csudo}cp -r ${script_dir}/bin/* ${install_main_dir}/bin && ${csudo}chmod 0555 ${install_main_dir}/bin/* @@ -211,7 +210,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/TDinsight.sh ] && ${csudo}ln -s ${install_main_dir}/bin/TDinsight.sh ${bin_link_dir}/TDinsight.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : [ -x ${install_main_dir}/bin/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ] && ${csudo}ln -s ${install_main_dir}/bin/run_${serverName}_and_${adapterName}.sh ${bin_link_dir}/run_${serverName}_and_${adapterName}.sh || : [ -x ${install_main_dir}/bin/tarbitrator ] && ${csudo}ln -s ${install_main_dir}/bin/tarbitrator ${bin_link_dir}/tarbitrator || : if [ "$verMode" == "cluster" ]; then diff --git a/packaging/tools/make_install.sh b/packaging/tools/make_install.sh index 31cb5e87b99a5ac61fef740bb8c9393c032ef75a..6726b3fe2da1bee3d7f35e2550fa4e23352cfc78 100755 --- a/packaging/tools/make_install.sh +++ b/packaging/tools/make_install.sh @@ -176,7 +176,6 @@ function install_bin() { if [ "$osType" != "Darwin" ]; then ${csudo}rm -f ${bin_link_dir}/perfMonitor || : ${csudo}rm -f ${bin_link_dir}/set_core || : - ${csudo}rm -f ${bin_link_dir}/run_taosd_and_taosadapter.sh || : ${csudo}rm -f ${bin_link_dir}/${uninstallScript} || : ${csudo}cp -r ${binary_dir}/build/bin/${clientName} ${install_main_dir}/bin || : @@ -191,7 +190,6 @@ function install_bin() { ${csudo}cp -r ${script_dir}/taosd-dump-cfg.gdb ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/remove.sh ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/set_core.sh ${install_main_dir}/bin || : - ${csudo}cp -r ${script_dir}/run_taosd_and_taosadapter.sh ${install_main_dir}/bin || : ${csudo}cp -r ${script_dir}/startPre.sh ${install_main_dir}/bin || : ${csudo}chmod 0555 ${install_main_dir}/bin/* @@ -204,7 +202,6 @@ function install_bin() { [ -x ${install_main_dir}/bin/taosdemo ] && ${csudo}ln -s ${install_main_dir}/bin/taosdemo ${bin_link_dir}/taosdemo || : [ -x ${install_main_dir}/bin/perfMonitor ] && ${csudo}ln -s ${install_main_dir}/bin/perfMonitor ${bin_link_dir}/perfMonitor || : [ -x ${install_main_dir}/set_core.sh ] && ${csudo}ln -s ${install_main_dir}/bin/set_core.sh ${bin_link_dir}/set_core || : - [ -x ${install_main_dir}/run_taosd_and_taosadapter.sh ] && ${csudo}ln -s ${install_main_dir}/bin/run_taosd_and_taosadapter.sh ${bin_link_dir}/run_taosd_and_taosadapter.sh || : [ -x ${install_main_dir}/bin/remove.sh ] && ${csudo}ln -s ${install_main_dir}/bin/remove.sh ${bin_link_dir}/${uninstallScript} || : else diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index ea8ebba4509f14f59be2a711c540c06e3a00702c..d0fbf0c4d790e75d4d73bb958da3d7152551f611 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -88,7 +88,6 @@ else ${build_dir}/bin/tarbitrator\ ${script_dir}/remove.sh \ ${script_dir}/set_core.sh \ - ${script_dir}/run_taosd_and_taosadapter.sh \ ${script_dir}/startPre.sh \ ${script_dir}/taosd-dump-cfg.gdb" fi @@ -152,7 +151,6 @@ if [ $adapterName != "taosadapter" ]; then sed -i "s/taosadapter/${adapterName}/g" ${install_dir}/cfg/$adapterName.service mv ${install_dir}/bin/taosadapter ${install_dir}/bin/${adapterName} - mv ${install_dir}/bin/run_taosd_and_taosadapter.sh ${install_dir}/bin/run_${serverName}_and_${adapterName}.sh mv ${install_dir}/bin/taosd-dump-cfg.gdb ${install_dir}/bin/${serverName}-dump-cfg.gdb fi diff --git a/packaging/tools/run_taosd_and_taosadapter.sh b/packaging/tools/run_taosd_and_taosadapter.sh deleted file mode 100755 index 9ab9eb484a4a5bbc4e3d3994d97b61e0f4bd328d..0000000000000000000000000000000000000000 --- a/packaging/tools/run_taosd_and_taosadapter.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -[[ -x /usr/bin/taosadapter ]] && /usr/bin/taosadapter & -taosd diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index c39b492620c4c9bb0922438c1e895b8b4661326d..cfe507e5053572952feb8b9ef3d6dc6bfba6e733 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -20,9 +20,9 @@ extern "C" { #endif +#include "catalog.h" #include "parser.h" #include "planner.h" -#include "catalog.h" #include "query.h" #include "taos.h" #include "tcommon.h" @@ -51,10 +51,12 @@ extern "C" { enum { RES_TYPE__QUERY = 1, RES_TYPE__TMQ, + RES_TYPE__TMQ_META, }; -#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) -#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) +#define TD_RES_QUERY(res) (*(int8_t*)res == RES_TYPE__QUERY) +#define TD_RES_TMQ(res) (*(int8_t*)res == RES_TYPE__TMQ) +#define TD_RES_TMQ_META(res) (*(int8_t*)res == RES_TYPE__TMQ_META) typedef struct SAppInstInfo SAppInstInfo; @@ -66,9 +68,9 @@ typedef struct { int64_t reportBytes; // not implemented int64_t startTime; // ctl - SRWLatch lock; // lock is used in serialization + SRWLatch lock; // lock is used in serialization SAppInstInfo* pAppInstInfo; - SHashObj* activeInfo; // hash + SHashObj* activeInfo; // hash } SAppHbMgr; typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); @@ -76,13 +78,13 @@ typedef int32_t (*FHbRspHandle)(SAppHbMgr* pAppHbMgr, SClientHbRsp* pRsp); typedef int32_t (*FHbReqHandle)(SClientHbKey* connKey, void* param, SClientHbReq* req); typedef struct { - int8_t inited; - int64_t appId; + int8_t inited; + int64_t appId; // ctl int8_t threadStop; TdThread thread; - TdThreadMutex lock; // used when app init and cleanup - SHashObj *appSummary; + TdThreadMutex lock; // used when app init and cleanup + SHashObj* appSummary; SArray* appHbMgrs; // SArray one for each cluster FHbReqHandle reqHandle[CONN_TYPE__MAX]; FHbRspHandle rspHandle[CONN_TYPE__MAX]; @@ -129,7 +131,7 @@ typedef struct STscObj { int8_t connType; int32_t acctId; uint32_t connId; - TAOS *id; // ref ID returned by taosAddRef + TAOS* id; // ref ID returned by taosAddRef TdThreadMutex mutex; // used to protect the operation on db int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; @@ -188,6 +190,14 @@ typedef struct { SReqResultInfo resInfo; } SMqRspObj; +typedef struct { + int8_t resType; + char topic[TSDB_TOPIC_FNAME_LEN]; + char db[TSDB_DB_FNAME_LEN]; + int32_t vgId; + SMqMetaRsp metaRsp; +} SMqMetaRspObj; + typedef struct SRequestObj { int8_t resType; // query or tmq uint64_t requestId; @@ -206,9 +216,9 @@ typedef struct SRequestObj { SRequestSendRecvBody body; bool stableQuery; - bool killed; - uint32_t prevCode; //previous error code: todo refactor, add update flag for catalog - uint32_t retry; + bool killed; + uint32_t prevCode; // previous error code: todo refactor, add update flag for catalog + uint32_t retry; } SRequestObj; typedef struct SSyncQueryParam { @@ -216,15 +226,15 @@ typedef struct SSyncQueryParam { SRequestObj* pRequest; } SSyncQueryParam; -void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); +void* doAsyncFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); +void* doFetchRows(SRequestObj* pRequest, bool setupOneRowPtr, bool convertUcs4); -void doSetOneRowPtr(SReqResultInfo* pResultInfo); -void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); -int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, - bool freeAfterUse); -void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); -void doFreeReqResultInfo(SReqResultInfo* pResInfo); +void doSetOneRowPtr(SReqResultInfo* pResultInfo); +void setResPrecision(SReqResultInfo* pResInfo, int32_t precision); +int32_t setQueryResultFromRsp(SReqResultInfo* pResultInfo, const SRetrieveTableRsp* pRsp, bool convertUcs4, + bool freeAfterUse); +void setResSchemaInfo(SReqResultInfo* pResInfo, const SSchema* pSchema, int32_t numOfCols); +void doFreeReqResultInfo(SReqResultInfo* pResInfo); SRequestObj* execQuery(STscObj* pTscObj, const char* sql, int sqlLen); static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { @@ -289,7 +299,7 @@ bool persistConnForSpecificMsg(void* parenct, tmsg_t msgType); void processMsgFromServer(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet); STscObj* taos_connect_internal(const char* ip, const char* user, const char* pass, const char* auth, const char* db, - uint16_t port, int connType); + uint16_t port, int connType); SRequestObj* launchQuery(STscObj* pTscObj, const char* sql, int sqlLen); @@ -299,7 +309,7 @@ int32_t getPlan(SRequestObj* pRequest, SQuery* pQuery, SQueryPlan** pPlan, SArra int32_t buildRequest(STscObj* pTscObj, const char* sql, int sqlLen, SRequestObj** pRequest); -void taos_close_internal(void *taos); +void taos_close_internal(void* taos); // --- heartbeat // global, called by mgmt @@ -320,12 +330,12 @@ void hbMgrInitMqHbRspHandle(); SRequestObj* launchQueryImpl(SRequestObj* pRequest, SQuery* pQuery, bool keepQuery, void** res); int32_t scheduleQuery(SRequestObj* pRequest, SQueryPlan* pDag, SArray* pNodeList); -void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData *pResultMeta); +void launchAsyncQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaData* pResultMeta); int32_t refreshMeta(STscObj* pTscObj, SRequestObj* pRequest); int32_t updateQnodeList(SAppInstInfo* pInfo, SArray* pNodeList); void doAsyncQuery(SRequestObj* pRequest, bool forceUpdateMeta); -int32_t removeMeta(STscObj* pTscObj, SArray* tbList);// todo move to clientImpl.c and become a static function -int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog);// todo move to xxx +int32_t removeMeta(STscObj* pTscObj, SArray* tbList); // todo move to clientImpl.c and become a static function +int32_t handleAlterTbExecRes(void* res, struct SCatalog* pCatalog); // todo move to xxx bool qnodeRequired(SRequestObj* pRequest); #ifdef __cplusplus diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index 14649c9fd44be83ecf3d8955e984a2dd1fb857f1..52dec515cedfb86264b77643eb51e285dc01bced 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -244,10 +244,10 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { } SReqResultInfo* pResultInfo = &pRequest->body.resInfo; + pRequest->code = code; if (pRequest->code != TSDB_CODE_SUCCESS) { pResultInfo->numOfRows = 0; - pRequest->code = code; tscError("0x%" PRIx64 " fetch results failed, code:%s, reqId:0x%" PRIx64, pRequest->self, tstrerror(code), pRequest->requestId); } else { @@ -256,7 +256,7 @@ void asyncExecLocalCmd(SRequestObj* pRequest, SQuery* pQuery) { pRequest->requestId); } - pRequest->body.queryFp(pRequest->body.param, pRequest, 0); + pRequest->body.queryFp(pRequest->body.param, pRequest, code); // pRequest->body.fetchFp(pRequest->body.param, pRequest, pResultInfo->numOfRows); } @@ -1442,80 +1442,6 @@ static int32_t doPrepareResPtr(SReqResultInfo* pResInfo) { return TSDB_CODE_SUCCESS; } -static char* parseTagDatatoJson(void* p) { - char* string = NULL; - cJSON* json = cJSON_CreateObject(); - if (json == NULL) { - goto end; - } - - SArray* pTagVals = NULL; - if (tTagToValArray((const STag*)p, &pTagVals) != 0) { - goto end; - } - - int16_t nCols = taosArrayGetSize(pTagVals); - char tagJsonKey[256] = {0}; - for (int j = 0; j < nCols; ++j) { - STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); - // json key encode by binary - memset(tagJsonKey, 0, sizeof(tagJsonKey)); - memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey)); - // json value - char type = pTagVal->type; - if (type == TSDB_DATA_TYPE_NULL) { - cJSON* value = cJSON_CreateNull(); - if (value == NULL) { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - } else if (type == TSDB_DATA_TYPE_NCHAR) { - cJSON* value = NULL; - if (pTagVal->nData > 0) { - char* tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1); - int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); - if (length < 0) { - tscError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, - pTagVal->pData); - taosMemoryFree(tagJsonValue); - goto end; - } - value = cJSON_CreateString(tagJsonValue); - taosMemoryFree(tagJsonValue); - if (value == NULL) { - goto end; - } - } else if (pTagVal->nData == 0) { - value = cJSON_CreateString(""); - } else { - ASSERT(0); - } - - cJSON_AddItemToObject(json, tagJsonKey, value); - } else if (type == TSDB_DATA_TYPE_DOUBLE) { - double jsonVd = *(double*)(&pTagVal->i64); - cJSON* value = cJSON_CreateNumber(jsonVd); - if (value == NULL) { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - } else if (type == TSDB_DATA_TYPE_BOOL) { - char jsonVd = *(char*)(&pTagVal->i64); - cJSON* value = cJSON_CreateBool(jsonVd); - if (value == NULL) { - goto end; - } - cJSON_AddItemToObject(json, tagJsonKey, value); - } else { - ASSERT(0); - } - } - string = cJSON_PrintUnformatted(json); -end: - cJSON_Delete(json); - return string; -} - static int32_t doConvertUCS4(SReqResultInfo* pResultInfo, int32_t numOfRows, int32_t numOfCols, int32_t* colLength) { for (int32_t i = 0; i < numOfCols; ++i) { int32_t type = pResultInfo->fields[i].type; @@ -1579,7 +1505,7 @@ static int32_t estimateJsonLen(SReqResultInfo* pResultInfo, int32_t numOfCols, i char* jsonInnerData = data + CHAR_BYTES; if (jsonInnerType == TSDB_DATA_TYPE_NULL) { len += (VARSTR_HEADER_SIZE + strlen(TSDB_DATA_NULL_STR_L)); - } else if (jsonInnerType & TD_TAG_JSON) { + } else if (tTagIsJson(data)) { len += (VARSTR_HEADER_SIZE + ((const STag*)(data))->len); } else if (jsonInnerType == TSDB_DATA_TYPE_NCHAR) { // value -> "value" len += varDataTLen(jsonInnerData) + CHAR_BYTES * 2; @@ -1663,7 +1589,7 @@ static int32_t doConvertJson(SReqResultInfo* pResultInfo, int32_t numOfCols, int if (jsonInnerType == TSDB_DATA_TYPE_NULL) { sprintf(varDataVal(dst), "%s", TSDB_DATA_NULL_STR_L); varDataSetLen(dst, strlen(varDataVal(dst))); - } else if (jsonInnerType & TD_TAG_JSON) { + } else if (tTagIsJson(data)) { char* jsonString = parseTagDatatoJson(data); STR_TO_VARSTR(dst, jsonString); taosMemoryFree(jsonString); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index c5a237f5eda0a457cfff68120f546cea7b8b430f..22747425f078ed16df9e76a0e1eb0a647f56241c 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -17,6 +17,7 @@ #include "clientInt.h" #include "clientLog.h" #include "clientStmt.h" +#include "functionMgt.h" #include "os.h" #include "query.h" #include "scheduler.h" @@ -25,7 +26,6 @@ #include "tref.h" #include "trpc.h" #include "version.h" -#include "functionMgt.h" #define TSC_VAR_NOT_RELEASE 1 #define TSC_VAR_RELEASED 0 @@ -97,11 +97,11 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha pass = TSDB_DEFAULT_PASS; } - STscObj* pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); + STscObj *pObj = taos_connect_internal(ip, user, pass, NULL, db, port, CONN_TYPE__QUERY); if (pObj) { return pObj->id; } - + return NULL; } @@ -111,41 +111,40 @@ void taos_close_internal(void *taos) { } STscObj *pTscObj = (STscObj *)taos; - tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t*)pTscObj->id, pTscObj->numOfReqs); + tscDebug("0x%" PRIx64 " try to close connection, numOfReq:%d", *(int64_t *)pTscObj->id, pTscObj->numOfReqs); - taosRemoveRef(clientConnRefPool, *(int64_t*)pTscObj->id); + taosRemoveRef(clientConnRefPool, *(int64_t *)pTscObj->id); } void taos_close(TAOS *taos) { if (taos == NULL) { return; } - - STscObj* pObj = acquireTscObj(*(int64_t*)taos); + + STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { return; } - + taos_close_internal(pObj); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); taosMemoryFree(taos); } - -int taos_errno(TAOS_RES *tres) { - if (tres == NULL) { +int taos_errno(TAOS_RES *res) { + if (res == NULL || TD_RES_TMQ_META(res)) { return terrno; } - if (TD_RES_TMQ(tres)) { + if (TD_RES_TMQ(res)) { return 0; } - return ((SRequestObj *)tres)->code; + return ((SRequestObj *)res)->code; } const char *taos_errstr(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return (const char *)tstrerror(terrno); } @@ -179,11 +178,15 @@ void taos_free_result(TAOS_RES *res) { if (pRsp->rsp.withSchema) taosArrayDestroyP(pRsp->rsp.blockSchema, (FDelete)tDeleteSSchemaWrapper); pRsp->resInfo.pRspMsg = NULL; doFreeReqResultInfo(&pRsp->resInfo); + } else if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj *pRspObj = (SMqMetaRspObj *)res; + taosMemoryFree(pRspObj->metaRsp.metaRsp); + taosMemoryFree(pRspObj); } } int taos_field_count(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return 0; } @@ -194,7 +197,7 @@ int taos_field_count(TAOS_RES *res) { int taos_num_fields(TAOS_RES *res) { return taos_field_count(res); } TAOS_FIELD *taos_fetch_fields(TAOS_RES *res) { - if (taos_num_fields(res) == 0) { + if (taos_num_fields(res) == 0 || TD_RES_TMQ_META(res)) { return NULL; } @@ -215,8 +218,8 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); if (pTscObj == NULL || sql == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; @@ -229,21 +232,21 @@ TAOS_RES *taos_query(TAOS *taos, const char *sql) { taos_query_a(taos, sql, syncQueryFn, param); tsem_wait(¶m->sem); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); return param->pRequest; #else size_t sqlLen = strlen(sql); if (sqlLen > (size_t)TSDB_MAX_ALLOWED_SQL_LEN) { - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); tscError("sql string exceeds max length:%d", TSDB_MAX_ALLOWED_SQL_LEN); terrno = TSDB_CODE_TSC_EXCEED_SQL_LIMIT; return NULL; } - TAOS_RES* pRes = execQuery(pTscObj, sql, sqlLen); + TAOS_RES *pRes = execQuery(pTscObj, sql, sqlLen); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); return pRes; #endif @@ -287,6 +290,8 @@ TAOS_ROW taos_fetch_row(TAOS_RES *res) { pResultInfo->current += 1; return pResultInfo->row; } + } else if (TD_RES_TMQ_META(res)) { + return NULL; } else { // assert to avoid un-initialization error ASSERT(0); @@ -380,7 +385,7 @@ int taos_print_row(char *str, TAOS_ROW row, TAOS_FIELD *fields, int num_fields) } int *taos_fetch_lengths(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return NULL; } @@ -389,7 +394,7 @@ int *taos_fetch_lengths(TAOS_RES *res) { } TAOS_ROW *taos_result_block(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { terrno = TSDB_CODE_INVALID_PARA; return NULL; } @@ -438,7 +443,7 @@ const char *taos_data_type(int type) { const char *taos_get_client_info() { return version; } int taos_affected_rows(TAOS_RES *res) { - if (res == NULL || TD_RES_TMQ(res)) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) { return 0; } @@ -448,7 +453,7 @@ int taos_affected_rows(TAOS_RES *res) { } int taos_result_precision(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return TSDB_TIME_PRECISION_MILLI; } @@ -463,15 +468,15 @@ int taos_result_precision(TAOS_RES *res) { } int taos_select_db(TAOS *taos, const char *db) { - STscObj* pObj = acquireTscObj(*(int64_t*)taos); + STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (pObj == NULL) { - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); terrno = TSDB_CODE_TSC_DISCONNECTED; return TSDB_CODE_TSC_DISCONNECTED; } if (db == NULL || strlen(db) == 0) { - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); terrno = TSDB_CODE_TSC_INVALID_INPUT; return terrno; } @@ -483,19 +488,19 @@ int taos_select_db(TAOS *taos, const char *db) { int32_t code = taos_errno(pRequest); taos_free_result(pRequest); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); return code; } void taos_stop_query(TAOS_RES *res) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ(res) || TD_RES_TMQ_META(res)) { return; } SRequestObj *pRequest = (SRequestObj *)res; pRequest->killed = true; - - int32_t numOfFields = taos_num_fields(pRequest); + + int32_t numOfFields = taos_num_fields(pRequest); // It is not a query, no need to stop. if (numOfFields == 0) { tscDebug("request %" PRIx64 " no need to be killed since not query", pRequest->requestId); @@ -510,6 +515,9 @@ void taos_stop_query(TAOS_RES *res) { } bool taos_is_null(TAOS_RES *res, int32_t row, int32_t col) { + if (res == NULL || TD_RES_TMQ_META(res)) { + return true; + } SReqResultInfo *pResultInfo = tscGetCurResInfo(res); if (col >= pResultInfo->numOfCols || col < 0 || row >= pResultInfo->numOfRows || row < 0) { return true; @@ -532,7 +540,7 @@ int taos_fetch_block(TAOS_RES *res, TAOS_ROW *rows) { } int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return 0; } @@ -575,7 +583,7 @@ int taos_fetch_block_s(TAOS_RES *res, int *numOfRows, TAOS_ROW *rows) { } int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return 0; } @@ -615,7 +623,7 @@ int taos_fetch_raw_block(TAOS_RES *res, int *numOfRows, void **pData) { } int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { - if (res == NULL) { + if (res == NULL || TD_RES_TMQ_META(res)) { return 0; } @@ -636,7 +644,7 @@ int *taos_get_column_data_offset(TAOS_RES *res, int columnIndex) { int taos_validate_sql(TAOS *taos, const char *sql) { return true; } void taos_reset_current_db(TAOS *taos) { - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); if (pTscObj == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; return; @@ -644,17 +652,17 @@ void taos_reset_current_db(TAOS *taos) { resetConnectDB(pTscObj); - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); } const char *taos_get_server_info(TAOS *taos) { - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); if (pTscObj == NULL) { terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); return pTscObj->ver; } @@ -682,7 +690,7 @@ static void destorySqlParseWrapper(SqlParseWrapper *pWrapper) { void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { tscDebug("enter meta callback, code %s", tstrerror(code)); - + SqlParseWrapper *pWrapper = (SqlParseWrapper *)param; SQuery *pQuery = pWrapper->pQuery; SRequestObj *pRequest = pWrapper->pRequest; @@ -723,11 +731,11 @@ void retrieveMetaCallback(SMetaData *pResultMeta, void *param, int32_t code) { } void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param) { - STscObj* pTscObj = acquireTscObj(*(int64_t*)taos); + STscObj *pTscObj = acquireTscObj(*(int64_t *)taos); if (pTscObj == NULL || sql == NULL || NULL == fp) { terrno = TSDB_CODE_INVALID_PARA; if (pTscObj) { - releaseTscObj(*(int64_t*)taos); + releaseTscObj(*(int64_t *)taos); } else { terrno = TSDB_CODE_TSC_DISCONNECTED; } @@ -745,7 +753,7 @@ void taos_query_a(TAOS *taos, const char *sql, __taos_async_fn_t fp, void *param } SRequestObj *pRequest = NULL; - int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); + int32_t code = buildRequest(pTscObj, sql, sqlLen, &pRequest); if (code != TSDB_CODE_SUCCESS) { terrno = code; fp(param, NULL, terrno); @@ -849,8 +857,8 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { SReqResultInfo *pResultInfo = &pRequest->body.resInfo; - tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, - pRequest->self, code, tstrerror(code), pRequest->requestId); + tscDebug("0x%" PRIx64 " enter scheduler fetch cb, code:%d - %s, reqId:0x%" PRIx64, pRequest->self, code, + tstrerror(code), pRequest->requestId); pResultInfo->pData = pResult; pResultInfo->numOfRows = 0; @@ -884,9 +892,11 @@ static void fetchCallback(void *pResult, void *param, int32_t code) { void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { ASSERT(res != NULL && fp != NULL); + ASSERT(TD_RES_QUERY(res)); SRequestObj *pRequest = res; pRequest->body.fetchFp = fp; + pRequest->body.param = param; SReqResultInfo *pResultInfo = &pRequest->body.resInfo; if (taos_num_fields(pRequest) == 0) { @@ -909,6 +919,7 @@ void taos_fetch_rows_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { ASSERT(res != NULL && fp != NULL); + ASSERT(TD_RES_QUERY(res)); SRequestObj *pRequest = res; pRequest->body.resInfo.convertUcs4 = false; @@ -922,6 +933,7 @@ void taos_fetch_raw_block_a(TAOS_RES *res, __taos_async_fn_t fp, void *param) { const void *taos_get_raw_block(TAOS_RES *res) { ASSERT(res != NULL); + ASSERT(TD_RES_QUERY(res)); SRequestObj *pRequest = res; return pRequest->body.resInfo.pData; @@ -948,16 +960,16 @@ int taos_load_table_info(TAOS *taos, const char *tableNameList) { } TAOS_STMT *taos_stmt_init(TAOS *taos) { - STscObj* pObj = acquireTscObj(*(int64_t*)taos); + STscObj *pObj = acquireTscObj(*(int64_t *)taos); if (NULL == pObj) { tscError("invalid parameter for %s", __FUNCTION__); terrno = TSDB_CODE_TSC_DISCONNECTED; return NULL; } - TAOS_STMT* pStmt = stmtInit(pObj); - - releaseTscObj(*(int64_t*)taos); + TAOS_STMT *pStmt = stmtInit(pObj); + + releaseTscObj(*(int64_t *)taos); return pStmt; } diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index 6fa4e6bffb35c11c4ae04f435dc34a67baddbb24..7d2bf019d2f7a35aad42aaa6ae3405f89b26d5ce 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -2389,17 +2389,19 @@ static int32_t isSchemalessDb(STscObj *taos, SRequestObj *request) { static void smlInsertCallback(void *param, void *res, int32_t code) { SRequestObj *pRequest = (SRequestObj *)res; SSmlHandle *info = (SSmlHandle *)param; + int32_t rows = taos_affected_rows(pRequest); uDebug("SML:0x%" PRIx64 " result. code:%d, msg:%s", info->id, pRequest->code, pRequest->msgBuf); // lock + taosThreadSpinLock(&info->params->lock); + info->params->request->body.resInfo.numOfRows += rows; if (code != TSDB_CODE_SUCCESS) { - taosThreadSpinLock(&info->params->lock); info->params->request->code = code; - taosThreadSpinUnlock(&info->params->lock); } + taosThreadSpinUnlock(&info->params->lock); // unlock - printf("SML:0x%" PRIx64 " insert finished, code: %d, total: %d\n", info->id, code, info->affectedRows); + uDebug("SML:0x%" PRIx64 " insert finished, code: %d, rows: %d, total: %d", info->id, code, rows, info->affectedRows); Params *pParam = info->params; bool isLast = info->isLast; info->cost.endTime = taosGetTimestampUs(); diff --git a/source/client/src/tmq.c b/source/client/src/tmq.c index 638b4f1ea559fed1d76009f1f1141aa1c9e36e07..637a7ee5dd1073c69de38bcba2f2e973eb49d987 100644 --- a/source/client/src/tmq.c +++ b/source/client/src/tmq.c @@ -54,6 +54,7 @@ struct tmq_conf_t { int8_t autoCommit; int8_t resetOffset; int8_t withTbName; + int8_t useSnapshot; uint16_t port; int32_t autoCommitInterval; char* ip; @@ -69,6 +70,7 @@ struct tmq_t { char groupId[TSDB_CGROUP_LEN]; char clientId[256]; int8_t withTbName; + int8_t useSnapshot; int8_t autoCommit; int32_t autoCommitInterval; int32_t resetOffsetCfg; @@ -149,7 +151,10 @@ typedef struct { int32_t epoch; SMqClientVg* vgHandle; SMqClientTopic* topicHandle; - SMqDataBlkRsp msg; + union { + SMqDataBlkRsp dataRsp; + SMqMetaRsp metaRsp; + }; } SMqPollRspWrapper; typedef struct { @@ -279,6 +284,18 @@ tmq_conf_res_t tmq_conf_set(tmq_conf_t* conf, const char* key, const char* value } } + if (strcmp(key, "experiment.use.snapshot") == 0) { + if (strcmp(value, "true") == 0) { + conf->useSnapshot = true; + return TMQ_CONF_OK; + } else if (strcmp(value, "false") == 0) { + conf->useSnapshot = false; + return TMQ_CONF_OK; + } else { + return TMQ_CONF_INVALID; + } + } + if (strcmp(key, "td.connect.ip") == 0) { conf->ip = strdup(value); return TMQ_CONF_OK; @@ -401,8 +418,17 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ int32_t code = -1; if (msg != NULL) { - SMqRspObj* pRspObj = (SMqRspObj*)msg; - if (!TD_RES_TMQ(pRspObj)) { + char* topic; + int32_t vgId; + if (TD_RES_TMQ(msg)) { + SMqRspObj* pRspObj = (SMqRspObj*)msg; + topic = pRspObj->topic; + vgId = pRspObj->vgId; + } else if (TD_RES_TMQ_META(msg)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)msg; + topic = pMetaRspObj->topic; + vgId = pMetaRspObj->vgId; + } else { return TSDB_CODE_TMQ_INVALID_MSG; } @@ -421,10 +447,10 @@ int32_t tmqCommitInner2(tmq_t* tmq, const TAOS_RES* msg, int8_t automatic, int8_ for (int32_t i = 0; i < taosArrayGetSize(tmq->clientTopics); i++) { SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, pRspObj->topic) == 0) { + if (strcmp(pTopic->topicName, topic) == 0) { for (int32_t j = 0; j < taosArrayGetSize(pTopic->vgs); j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->vgId == pRspObj->vgId) { + if (pVg->vgId == vgId) { if (pVg->currentOffset < 0 || pVg->committedOffset == pVg->currentOffset) { tscDebug("consumer %ld skip commit for topic %s vg %d, current offset is %ld, committed offset is %ld", tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset, pVg->committedOffset); @@ -941,6 +967,7 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { strcpy(pTmq->clientId, conf->clientId); strcpy(pTmq->groupId, conf->groupId); pTmq->withTbName = conf->withTbName; + pTmq->useSnapshot = conf->useSnapshot; pTmq->autoCommit = conf->autoCommit; pTmq->autoCommitInterval = conf->autoCommitInterval; pTmq->commitCb = conf->commitCb; @@ -1131,6 +1158,9 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { tscWarn("mismatch rsp from vg %d, epoch %d, current epoch %d", vgId, msgEpoch, tmqEpoch); } + // handle meta rsp + int8_t rspType = ((SMqRspHead*)pMsg->pData)->mqMsgType; + SMqPollRspWrapper* pRspWrapper = taosAllocateQitem(sizeof(SMqPollRspWrapper), DEF_QITEM); if (pRspWrapper == NULL) { taosMemoryFree(pMsg->pData); @@ -1138,17 +1168,23 @@ int32_t tmqPollCb(void* param, const SDataBuf* pMsg, int32_t code) { goto CREATE_MSG_FAIL; } - pRspWrapper->tmqRspType = TMQ_MSG_TYPE__POLL_RSP; + pRspWrapper->tmqRspType = rspType; pRspWrapper->vgHandle = pVg; pRspWrapper->topicHandle = pTopic; - memcpy(&pRspWrapper->msg, pMsg->pData, sizeof(SMqRspHead)); + if (rspType == TMQ_MSG_TYPE__POLL_RSP) { + memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->dataRsp); + } else { + ASSERT(rspType == TMQ_MSG_TYPE__POLL_META_RSP); + memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); + tDecodeSMqMetaRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->metaRsp); + } - tDecodeSMqDataBlkRsp(POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), &pRspWrapper->msg); taosMemoryFree(pMsg->pData); - tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld", tmq->consumerId, pVg->vgId, - pRspWrapper->msg.reqOffset, pRspWrapper->msg.rspOffset); + tscDebug("consumer %ld recv poll: vg %d, req offset %ld, rsp offset %ld, type %d", tmq->consumerId, pVg->vgId, + pRspWrapper->dataRsp.reqOffset, pRspWrapper->dataRsp.rspOffset, rspType); taosWriteQitem(tmq->mqueue, pRspWrapper); tsem_post(&tmq->rspSem); @@ -1511,11 +1547,24 @@ SMqPollReq* tmqBuildConsumeReqImpl(tmq_t* tmq, int64_t timeout, SMqClientTopic* pReq->currentOffset = reqOffset; pReq->reqId = generateRequestId(); + pReq->useSnapshot = tmq->useSnapshot; + pReq->head.vgId = htonl(pVg->vgId); pReq->head.contLen = htonl(sizeof(SMqPollReq)); return pReq; } +SMqMetaRspObj* tmqBuildMetaRspFromWrapper(SMqPollRspWrapper* pWrapper) { + SMqMetaRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqMetaRspObj)); + pRspObj->resType = RES_TYPE__TMQ_META; + tstrncpy(pRspObj->topic, pWrapper->topicHandle->topicName, TSDB_TOPIC_FNAME_LEN); + tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); + pRspObj->vgId = pWrapper->vgHandle->vgId; + + memcpy(&pRspObj->metaRsp, &pWrapper->metaRsp, sizeof(SMqMetaRsp)); + return pRspObj; +} + SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { SMqRspObj* pRspObj = taosMemoryCalloc(1, sizeof(SMqRspObj)); pRspObj->resType = RES_TYPE__TMQ; @@ -1523,11 +1572,11 @@ SMqRspObj* tmqBuildRspFromWrapper(SMqPollRspWrapper* pWrapper) { tstrncpy(pRspObj->db, pWrapper->topicHandle->db, TSDB_DB_FNAME_LEN); pRspObj->vgId = pWrapper->vgHandle->vgId; pRspObj->resIter = -1; - memcpy(&pRspObj->rsp, &pWrapper->msg, sizeof(SMqDataBlkRsp)); + memcpy(&pRspObj->rsp, &pWrapper->dataRsp, sizeof(SMqDataBlkRsp)); pRspObj->resInfo.totalRows = 0; pRspObj->resInfo.precision = TSDB_TIME_PRECISION_MILLI; - if (!pWrapper->msg.withSchema) { + if (!pWrapper->dataRsp.withSchema) { setResSchemaInfo(&pRspObj->resInfo, pWrapper->topicHandle->schema.pSchema, pWrapper->topicHandle->schema.nCols); } @@ -1625,7 +1674,7 @@ int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* pReset) return 0; } -SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { +void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { while (1) { SMqRspWrapper* rspWrapper = NULL; taosGetQitem(tmq->qall, (void**)&rspWrapper); @@ -1643,12 +1692,12 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; /*atomic_sub_fetch_32(&tmq->readyRequest, 1);*/ int32_t consumerEpoch = atomic_load_32(&tmq->epoch); - if (pollRspWrapper->msg.head.epoch == consumerEpoch) { + if (pollRspWrapper->dataRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ - pVg->currentOffset = pollRspWrapper->msg.rspOffset; + pVg->currentOffset = pollRspWrapper->dataRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); - if (pollRspWrapper->msg.blockNum == 0) { + if (pollRspWrapper->dataRsp.blockNum == 0) { taosFreeQitem(pollRspWrapper); rspWrapper = NULL; continue; @@ -1658,8 +1707,25 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); return pRsp; } else { - tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", pollRspWrapper->msg.head.epoch, - consumerEpoch); + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", + pollRspWrapper->dataRsp.head.epoch, consumerEpoch); + taosFreeQitem(pollRspWrapper); + } + } else if (rspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)rspWrapper; + int32_t consumerEpoch = atomic_load_32(&tmq->epoch); + if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { + SMqClientVg* pVg = pollRspWrapper->vgHandle; + /*printf("vg %d offset %ld up to %ld\n", pVg->vgId, pVg->currentOffset, rspMsg->msg.rspOffset);*/ + pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // build rsp + SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); + taosFreeQitem(pollRspWrapper); + return pRsp; + } else { + tscDebug("msg discard since epoch mismatch: msg epoch %d, consumer epoch %d\n", + pollRspWrapper->metaRsp.head.epoch, consumerEpoch); taosFreeQitem(pollRspWrapper); } } else { @@ -1676,8 +1742,8 @@ SMqRspObj* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { - SMqRspObj* rspObj; - int64_t startTime = taosGetTimestampMs(); + void* rspObj; + int64_t startTime = taosGetTimestampMs(); #if 0 tmqHandleAllDelayedTask(tmq); @@ -1747,10 +1813,23 @@ const char* tmq_err2str(int32_t err) { } } +tmq_res_t tmq_get_res_type(TAOS_RES* res) { + if (TD_RES_TMQ(res)) { + return TMQ_RES_DATA; + } else if (TD_RES_TMQ_META(res)) { + return TMQ_RES_TABLE_META; + } else { + return TMQ_RES_INVALID; + } +} + const char* tmq_get_topic_name(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return strchr(pRspObj->topic, '.') + 1; + } else if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + return strchr(pMetaRspObj->topic, '.') + 1; } else { return NULL; } @@ -1760,6 +1839,9 @@ const char* tmq_get_db_name(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return strchr(pRspObj->db, '.') + 1; + } else if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + return strchr(pMetaRspObj->db, '.') + 1; } else { return NULL; } @@ -1769,6 +1851,9 @@ int32_t tmq_get_vgroup_id(TAOS_RES* res) { if (TD_RES_TMQ(res)) { SMqRspObj* pRspObj = (SMqRspObj*)res; return pRspObj->vgId; + } else if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + return pMetaRspObj->vgId; } else { return -1; } @@ -1786,6 +1871,16 @@ const char* tmq_get_table_name(TAOS_RES* res) { return NULL; } +int32_t tmq_get_raw_meta(TAOS_RES* res, void** raw_meta, int32_t* raw_meta_len) { + if (TD_RES_TMQ_META(res)) { + SMqMetaRspObj* pMetaRspObj = (SMqMetaRspObj*)res; + *raw_meta = pMetaRspObj->metaRsp.metaRsp; + *raw_meta_len = pMetaRspObj->metaRsp.metaRspLen; + return 0; + } + return -1; +} + void tmq_commit_async(tmq_t* tmq, const TAOS_RES* msg, tmq_commit_cb* cb, void* param) { tmqCommitInner2(tmq, msg, 0, 1, cb, param); } diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index b6ec75f8ebb9d21967a22a5dfbf453394710ce10..36dcab5c192ea42a6ac4a98da16b5a48fd642b1d 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -43,6 +43,26 @@ void showDB(TAOS* pConn) { } } +void printResult(TAOS_RES* pRes) { + TAOS_ROW pRow = NULL; + TAOS_FIELD* pFields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_num_fields(pRes); + + int32_t n = 0; + char str[512] = {0}; + while ((pRow = taos_fetch_row(pRes)) != NULL) { + int32_t* length = taos_fetch_lengths(pRes); + for(int32_t i = 0; i < numOfFields; ++i) { + printf("(%d):%d " , i, length[i]); + } + printf("\n"); + + int32_t code = taos_print_row(str, pRow, pFields, numOfFields); + printf("%s\n", str); + memset(str, 0, sizeof(str)); + } +} + void fetchCallback(void* param, void* res, int32_t numOfRow) { #if 0 printf("numOfRow = %d \n", numOfRow); @@ -729,48 +749,31 @@ TEST(testCase, projection_query_tables) { // taos_close(pConn); //} -//TEST(testCase, agg_query_tables) { -// TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); -// ASSERT_NE(pConn, nullptr); -// -// TAOS_RES* pRes = taos_query(pConn, "use abc1"); -// if (taos_errno(pRes) != 0) { -// printf("failed to use db, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// taos_free_result(pRes); -// -// pRes = taos_query(pConn, "show stables"); -// if (taos_errno(pRes) != 0) { -// printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); -// taos_free_result(pRes); -// ASSERT_TRUE(false); -// } -// -// TAOS_ROW pRow = NULL; -// TAOS_FIELD* pFields = taos_fetch_fields(pRes); -// int32_t numOfFields = taos_num_fields(pRes); -// -// int32_t n = 0; -// char str[512] = {0}; -// while ((pRow = taos_fetch_row(pRes)) != NULL) { -// int32_t* length = taos_fetch_lengths(pRes); -// for(int32_t i = 0; i < numOfFields; ++i) { -// printf("(%d):%d " , i, length[i]); -// } -// printf("\n"); -// -// int32_t code = taos_print_row(str, pRow, pFields, numOfFields); -// printf("%s\n", str); -// memset(str, 0, sizeof(str)); -// } -// -// taos_free_result(pRes); -// taos_close(pConn); -//} -#endif +TEST(testCase, agg_query_tables) { + TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_NE(pConn, nullptr); + + TAOS_RES* pRes = taos_query(pConn, "use abc1"); + if (taos_errno(pRes) != 0) { + printf("failed to use db, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + taos_free_result(pRes); + + pRes = taos_query(pConn, "show table distributed st1"); + if (taos_errno(pRes) != 0) { + printf("failed to select from table, reason:%s\n", taos_errstr(pRes)); + taos_free_result(pRes); + ASSERT_TRUE(false); + } + + printResult(pRes); + taos_free_result(pRes); + taos_close(pConn); +} +#endif /* --- copy the following script in the shell to setup the environment --- @@ -786,7 +789,7 @@ TEST(testCase, async_api_test) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); ASSERT_NE(pConn, nullptr); - taos_query(pConn, "use table_alltype_hyperloglog"); + taos_query(pConn, "use abc1"); #if 0 TAOS_RES* pRes = taos_query(pConn, "insert into tu(ts) values('2022-02-27 12:12:61')"); if (taos_errno(pRes) != 0) { @@ -812,7 +815,7 @@ TEST(testCase, async_api_test) { } #endif - taos_query_a(pConn, "select HYPERLOGLOG(q_ts) from stable_1_2 where ts between 1630000001000 and 1630100001000 interval(19d) Fill(NONE);", queryCallback, pConn); + taos_query_a(pConn, "select count(*) from tu", queryCallback, pConn); getchar(); taos_close(pConn); } diff --git a/source/client/test/smlTest.cpp b/source/client/test/smlTest.cpp index fffb03d9a5d7c797174daaad1b72ebb7dd444868..832564e0dbe8a2914a6d1cc852919282b8c44ba6 100644 --- a/source/client/test/smlTest.cpp +++ b/source/client/test/smlTest.cpp @@ -1272,10 +1272,10 @@ TEST(testCase, sml_dup_time_Test) { const char *sql[] = { //"test_ms,t0=t c0=f 1626006833641", "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=false,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000" + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000" }; pRes = taos_query(taos, "use dup_time"); taos_free_result(pRes); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index e7b6342150a4b48242eac408c021ad9cc2bfa170..455f204542f68b3d5d23063b79c5a26bea7538c2 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -231,7 +231,13 @@ static const SSysDbTableSchema transSchema[] = { static const SSysDbTableSchema configSchema[] = { {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, - {.name = "value", .bytes = TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, +}; + +static const SSysDbTableSchema variablesSchema[] = { + {.name = "dnode_id", .bytes = 4, .type = TSDB_DATA_TYPE_INT}, + {.name = "name", .bytes = TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, + {.name = "value", .bytes = TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR}, }; static const SSysTableMeta infosMeta[] = { @@ -253,6 +259,7 @@ static const SSysTableMeta infosMeta[] = { {TSDB_INS_TABLE_LICENCES, grantsSchema, tListLen(grantsSchema)}, {TSDB_INS_TABLE_VGROUPS, vgroupsSchema, tListLen(vgroupsSchema)}, {TSDB_INS_TABLE_CONFIGS, configSchema, tListLen(configSchema)}, + {TSDB_INS_TABLE_DNODE_VARIABLES, variablesSchema, tListLen(variablesSchema)}, }; static const SSysDbTableSchema connectionsSchema[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index 3c3d3e953dc567f7f9edb83d56d6ff48dca9edf6..8317c6c79daedd59a99652b638ce40e64f4ebdaf 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -110,7 +110,7 @@ int32_t getJsonValueLen(const char* data) { dataLen = DOUBLE_BYTES + CHAR_BYTES; } else if (*data == TSDB_DATA_TYPE_BOOL) { dataLen = CHAR_BYTES + CHAR_BYTES; - } else if (*data & TD_TAG_JSON) { // json string + } else if (tTagIsJson(data)) { // json string dataLen = ((STag*)(data))->len; } else { ASSERT(0); @@ -214,7 +214,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } } -int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, int32_t* capacity, +int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, uint32_t* capacity, const SColumnInfoData* pSource, uint32_t numOfRow2) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRow2 == 0) { @@ -263,6 +263,7 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in pColumnInfoData->varmeta.length = len + oldLen; } else { if (finalNumOfRows > *capacity) { + ASSERT(finalNumOfRows*pColumnInfoData->info.bytes); char* tmp = taosMemoryRealloc(pColumnInfoData->pData, finalNumOfRows * pColumnInfoData->info.bytes); if (tmp == NULL) { return TSDB_CODE_VND_OUT_OF_MEMORY; @@ -292,22 +293,18 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, uint32_t numOfRow1, in return numOfRow1 + numOfRow2; } -int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows) { +int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* pSource, int32_t numOfRows, const SDataBlockInfo* pBlockInfo) { ASSERT(pColumnInfoData != NULL && pSource != NULL && pColumnInfoData->info.type == pSource->info.type); if (numOfRows <= 0) { return numOfRows; } - if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { - // Handle the bitmap - char* p = taosMemoryRealloc(pColumnInfoData->varmeta.offset, sizeof(int32_t) * numOfRows); - if (p == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } + if (pBlockInfo != NULL) { + ASSERT(pBlockInfo->capacity >= numOfRows); + } - pColumnInfoData->varmeta.offset = (int32_t*)p; + if (IS_VAR_DATA_TYPE(pColumnInfoData->info.type)) { memcpy(pColumnInfoData->varmeta.offset, pSource->varmeta.offset, sizeof(int32_t) * numOfRows); - if (pColumnInfoData->varmeta.allocLen < pSource->varmeta.length) { char* tmp = taosMemoryRealloc(pColumnInfoData->pData, pSource->varmeta.length); if (tmp == NULL) { @@ -318,27 +315,11 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p pColumnInfoData->varmeta.allocLen = pSource->varmeta.length; } - memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); pColumnInfoData->varmeta.length = pSource->varmeta.length; + memcpy(pColumnInfoData->pData, pSource->pData, pSource->varmeta.length); } else { - char* tmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(numOfRows)); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->nullbitmap = tmp; memcpy(pColumnInfoData->nullbitmap, pSource->nullbitmap, BitmapLen(numOfRows)); - - if (pSource->pData) { - int32_t newSize = numOfRows * pColumnInfoData->info.bytes; - tmp = taosMemoryRealloc(pColumnInfoData->pData, newSize); - if (tmp == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pColumnInfoData->pData = tmp; - memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); - } + memcpy(pColumnInfoData->pData, pSource->pData, pSource->info.bytes * numOfRows); } pColumnInfoData->hasNull = pSource->hasNull; @@ -347,8 +328,7 @@ int32_t colDataAssign(SColumnInfoData* pColumnInfoData, const SColumnInfoData* p } size_t blockDataGetNumOfCols(const SSDataBlock* pBlock) { - ASSERT(pBlock && pBlock->info.numOfCols == taosArrayGetSize(pBlock->pDataBlock)); - return pBlock->info.numOfCols; + return taosArrayGetSize(pBlock->pDataBlock); } size_t blockDataGetNumOfRows(const SSDataBlock* pBlock) { return pBlock->info.rows; } @@ -358,7 +338,8 @@ int32_t blockDataUpdateTsWindow(SSDataBlock* pDataBlock, int32_t tsColumnIndex) return 0; } - if (pDataBlock->info.numOfCols <= 0) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (numOfCols <= 0) { return -1; } @@ -382,7 +363,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { assert(pSrc != NULL && pDest != NULL); int32_t capacity = pDest->info.capacity; - for (int32_t i = 0; i < pDest->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDest->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol2 = taosArrayGet(pDest->pDataBlock, i); SColumnInfoData* pCol1 = taosArrayGet(pSrc->pDataBlock, i); @@ -398,9 +380,8 @@ int32_t blockDataMerge(SSDataBlock* pDest, const SSDataBlock* pSrc) { size_t blockDataGetSize(const SSDataBlock* pBlock) { assert(pBlock != NULL); - size_t total = 0; - int32_t numOfCols = pBlock->info.numOfCols; - + size_t total = 0; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); total += colDataGetFullLength(pColInfoData, pBlock->info.rows); @@ -415,7 +396,7 @@ int32_t blockDataSplitRows(SSDataBlock* pBlock, bool hasVarCol, int32_t startInd int32_t pageSize) { ASSERT(pBlock != NULL && stopIndex != NULL); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; int32_t bitmapChar = 1; @@ -476,33 +457,24 @@ SSDataBlock* blockDataExtractBlock(SSDataBlock* pBlock, int32_t startIndex, int3 return NULL; } - SSDataBlock* pDst = taosMemoryCalloc(1, sizeof(SSDataBlock)); + SSDataBlock* pDst = createDataBlock(); if (pDst == NULL) { return NULL; } pDst->info = pBlock->info; - pDst->info.rows = 0; - pDst->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData colInfo = {0}; SColumnInfoData* pSrcCol = taosArrayGet(pBlock->pDataBlock, i); colInfo.info = pSrcCol->info; - - if (IS_VAR_DATA_TYPE(pSrcCol->info.type)) { - SVarColAttr* pAttr = &colInfo.varmeta; - pAttr->offset = taosMemoryCalloc(rowCount, sizeof(int32_t)); - } else { - colInfo.nullbitmap = taosMemoryCalloc(1, BitmapLen(rowCount)); - colInfo.pData = taosMemoryCalloc(rowCount, colInfo.info.bytes); - } - - taosArrayPush(pDst->pDataBlock, &colInfo); + blockDataAppendColInfo(pDst, &colInfo); } - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + blockDataEnsureCapacity(pDst, rowCount); + + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pDstCol = taosArrayGet(pDst->pDataBlock, i); @@ -540,7 +512,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { // write the number of rows *(uint32_t*)buf = pBlock->info.rows; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int32_t numOfRows = pBlock->info.rows; char* pStart = buf + sizeof(uint32_t); @@ -570,7 +542,7 @@ int32_t blockDataToBuf(char* buf, const SSDataBlock* pBlock) { int32_t blockDataFromBuf(SSDataBlock* pBlock, const char* buf) { pBlock->info.rows = *(int32_t*)buf; - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); const char* pStart = buf + sizeof(uint32_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -619,7 +591,8 @@ int32_t blockDataFromBuf1(SSDataBlock* pBlock, const char* buf, size_t capacity) pBlock->info.rows = *(int32_t*)buf; pBlock->info.groupId = *(uint64_t*)(buf + sizeof(int32_t)); - int32_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + const char* pStart = buf + sizeof(uint32_t) + sizeof(uint64_t); for (int32_t i = 0; i < numOfCols; ++i) { @@ -665,7 +638,7 @@ size_t blockDataGetRowSize(SSDataBlock* pBlock) { if (pBlock->info.rowSize == 0) { size_t rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -692,7 +665,7 @@ double blockDataGetSerialRowSize(const SSDataBlock* pBlock) { ASSERT(pBlock != NULL); double rowSize = 0; - size_t numOfCols = pBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); rowSize += pColInfo->info.bytes; @@ -761,7 +734,7 @@ int32_t dataBlockCompar(const void* p1, const void* p2, const void* param) { static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, const SSDataBlock* pSrcBlock, int32_t tupleIndex) { int32_t code = 0; - int32_t numOfCols = pSrcBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pDstCols[i]; @@ -784,7 +757,7 @@ static int32_t doAssignOneTuple(SColumnInfoData* pDstCols, int32_t numOfRows, co return TSDB_CODE_SUCCESS; } -static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, int32_t* index) { +static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataBlock, const int32_t* index) { #if 0 for (int32_t i = 0; i < pDataBlock->info.rows; ++i) { int32_t code = doAssignOneTuple(pCols, i, pDataBlock, index[i]); @@ -793,7 +766,8 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB } } #else - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = &pCols[i]; SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); @@ -820,7 +794,7 @@ static int32_t blockDataAssign(SColumnInfoData* pCols, const SSDataBlock* pDataB static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { int32_t rows = pDataBlock->info.rows; - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); SColumnInfoData* pCols = taosMemoryCalloc(numOfCols, sizeof(SColumnInfoData)); if (pCols == NULL) { @@ -847,7 +821,7 @@ static SColumnInfoData* createHelpColInfoData(const SSDataBlock* pDataBlock) { } static void copyBackToBlock(SSDataBlock* pDataBlock, SColumnInfoData* pCols) { - int32_t numOfCols = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, i); @@ -908,8 +882,10 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { } } + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + if (taosArrayGetSize(pOrderInfo) == 1 && (!sortColumnHasNull)) { - if (pDataBlock->info.numOfCols == 1) { + if (numOfCols == 1) { if (!varTypeSort) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, 0); SBlockOrderInfo* pOrder = taosArrayGet(pOrderInfo, 0); @@ -925,7 +901,8 @@ int32_t blockDataSort(SSDataBlock* pDataBlock, SArray* pOrderInfo) { return TSDB_CODE_SUCCESS; } else { // var data type } - } else if (pDataBlock->info.numOfCols == 2) { + } else if (numOfCols == 2) { + } } @@ -1119,33 +1096,39 @@ int32_t blockDataSort_rv(SSDataBlock* pDataBlock, SArray* pOrderInfo, bool nullF void blockDataCleanup(SSDataBlock* pDataBlock) { pDataBlock->info.rows = 0; - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); colInfoDataCleanup(p, pDataBlock->info.capacity); } } -int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, uint32_t numOfRows) { - if (0 == numOfRows || numOfRows <= existRows) { +static int32_t doEnsureCapacity(SColumnInfoData* pColumn, const SDataBlockInfo *pBlockInfo, uint32_t numOfRows) { + ASSERT(numOfRows > 0 && pBlockInfo->capacity >= pBlockInfo->rows); + if (numOfRows < pBlockInfo->capacity) { return TSDB_CODE_SUCCESS; } + // todo temp disable it +// ASSERT(pColumn->info.bytes != 0); + + int32_t existedRows = pBlockInfo->rows; + if (IS_VAR_DATA_TYPE(pColumn->info.type)) { char* tmp = taosMemoryRealloc(pColumn->varmeta.offset, sizeof(int32_t) * numOfRows); - if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } pColumn->varmeta.offset = (int32_t*)tmp; - memset(&pColumn->varmeta.offset[existRows], 0, sizeof(int32_t) * (numOfRows - existRows)); + memset(&pColumn->varmeta.offset[existedRows], 0, sizeof(int32_t) * (numOfRows - existedRows)); } else { char* tmp = taosMemoryRealloc(pColumn->nullbitmap, BitmapLen(numOfRows)); if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - int32_t oldLen = BitmapLen(existRows); + int32_t oldLen = BitmapLen(existedRows); pColumn->nullbitmap = tmp; memset(&pColumn->nullbitmap[oldLen], 0, BitmapLen(numOfRows) - oldLen); @@ -1158,8 +1141,8 @@ int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, size_t existRows, ui if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - memset(tmp + pColumn->info.bytes * existRows, 0, pColumn->info.bytes * (numOfRows - existRows)); - + + memset(tmp + pColumn->info.bytes * existedRows, 0, pColumn->info.bytes * (numOfRows - existedRows)); pColumn->pData = tmp; } @@ -1176,17 +1159,27 @@ void colInfoDataCleanup(SColumnInfoData* pColumn, uint32_t numOfRows) { } } +int32_t colInfoDataEnsureCapacity(SColumnInfoData* pColumn, uint32_t numOfRows) { + SDataBlockInfo info = {0}; + return doEnsureCapacity(pColumn, &info, numOfRows); +} + int32_t blockDataEnsureCapacity(SSDataBlock* pDataBlock, uint32_t numOfRows) { int32_t code = 0; + ASSERT(numOfRows > 0); + if (numOfRows == 0) { return TSDB_CODE_SUCCESS; } - pDataBlock->info.capacity = numOfRows; + if (pDataBlock->info.capacity < numOfRows) { + pDataBlock->info.capacity = numOfRows; + } - for (int32_t i = 0; i < pDataBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - code = colInfoDataEnsureCapacity(p, pDataBlock->info.rows, numOfRows); + code = doEnsureCapacity(p, &pDataBlock->info, numOfRows); if (code) { return code; } @@ -1210,40 +1203,34 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return NULL; } - int32_t numOfCols = pDataBlock->info.numOfCols; - - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - - pBlock->info.numOfCols = numOfCols; - pBlock->info.hasVarCol = pDataBlock->info.hasVarCol; - pBlock->info.rowSize = pDataBlock->info.rowSize; - pBlock->info.groupId = pDataBlock->info.groupId; - pBlock->info.childId = pDataBlock->info.childId; - pBlock->info.type = pDataBlock->info.type; + SSDataBlock* pBlock = createDataBlock(); + pBlock->info = pDataBlock->info; + pBlock->info.rows = 0; + pBlock->info.capacity = 0; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData colInfo = {0}; SColumnInfoData* p = taosArrayGet(pDataBlock->pDataBlock, i); - colInfo.info = p->info; - colInfo.hasNull = true; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = {.hasNull = true, .info = p->info}; + blockDataAppendColInfo(pBlock, &colInfo); } if (copyData) { + int32_t code = blockDataEnsureCapacity(pBlock, pDataBlock->info.rows); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + blockDataDestroy(pBlock); + return NULL; + } + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrc = taosArrayGet(pDataBlock->pDataBlock, i); - - int32_t code = colInfoDataEnsureCapacity(pDst, 0, pDataBlock->info.rows); - if (code != TSDB_CODE_SUCCESS) { - return NULL; - } - if (pSrc->pData == NULL) { continue; } - colDataAssign(pDst, pSrc, pDataBlock->info.rows); + + colDataAssign(pDst, pSrc, pDataBlock->info.rows, &pDataBlock->info); } pBlock->info.rows = pDataBlock->info.rows; @@ -1253,16 +1240,75 @@ SSDataBlock* createOneDataBlock(const SSDataBlock* pDataBlock, bool copyData) { return pBlock; } +SSDataBlock* createDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + if (pBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + } + + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pBlock); + } + + return pBlock; +} + +int32_t blockDataAppendColInfo(SSDataBlock* pBlock, SColumnInfoData* pColInfoData) { + ASSERT(pBlock != NULL && pColInfoData != NULL); + if (pBlock->pDataBlock == NULL) { + pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + if (pBlock->pDataBlock == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + } + + void* p = taosArrayPush(pBlock->pDataBlock, pColInfoData); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + // todo disable it temporarily +// ASSERT(pColInfoData->info.type != 0); + if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { + pBlock->info.hasVarCol = true; + } + + pBlock->info.rowSize += pColInfoData->info.bytes; + return TSDB_CODE_SUCCESS; +} + +SColumnInfoData createColumnInfoData(int16_t type, int32_t bytes, int16_t colId) { + SColumnInfoData col = {.hasNull = true}; + col.info.colId = colId; + col.info.type = type; + col.info.bytes = bytes; + + return col; +} + +SColumnInfoData* bdGetColumnInfoData(SSDataBlock* pBlock, int32_t index) { + ASSERT(pBlock != NULL); + if (index >= taosArrayGetSize(pBlock->pDataBlock)) { + return NULL; + } + + return taosArrayGet(pBlock->pDataBlock, index); +} + size_t blockDataGetCapacityInRow(const SSDataBlock* pBlock, size_t pageSize) { - int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(pBlock->info.numOfCols); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t payloadSize = pageSize - blockDataGetSerialMetaSize(numOfCols); int32_t rowSize = pBlock->info.rowSize; - int32_t nRows = payloadSize / rowSize; // the true value must be less than the value of nRows int32_t additional = 0; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pCol->info.type)) { additional += nRows * sizeof(int32_t); @@ -1346,7 +1392,8 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { if (pBlock->info.rows <= n) { blockDataCleanup(pBlock); } else { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataTrimFirstNRows(pColInfoData, n, pBlock->info.rows); } @@ -1358,7 +1405,7 @@ int32_t blockDataTrimFirstNRows(SSDataBlock* pBlock, size_t n) { int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { int64_t tbUid = pBlock->info.uid; - int16_t numOfCols = pBlock->info.numOfCols; + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); int16_t hasVarCol = pBlock->info.hasVarCol; int32_t rows = pBlock->info.rows; int32_t sz = taosArrayGetSize(pBlock->pDataBlock); @@ -1393,8 +1440,10 @@ int32_t tEncodeDataBlock(void** buf, const SSDataBlock* pBlock) { void* tDecodeDataBlock(const void* buf, SSDataBlock* pBlock) { int32_t sz; - buf = taosDecodeFixedI64(buf, &pBlock->info.uid); - buf = taosDecodeFixedI16(buf, &pBlock->info.numOfCols); + int16_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + + buf = taosDecodeFixedU64(buf, &pBlock->info.uid); + buf = taosDecodeFixedI16(buf, &numOfCols); buf = taosDecodeFixedI16(buf, &pBlock->info.hasVarCol); buf = taosDecodeFixedI32(buf, &pBlock->info.rows); buf = taosDecodeFixedI32(buf, &sz); @@ -1503,12 +1552,13 @@ void blockDebugShowData(const SArray* dataBlocks, const char* flag) { int32_t sz = taosArrayGetSize(dataBlocks); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(dataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + size_t numOfCols = taosArrayGetSize(pDataBlock->pDataBlock); + int32_t rows = pDataBlock->info.rows; printf("%s |block type %d |child id %d|\n", flag, (int32_t)pDataBlock->info.type, pDataBlock->info.childId); for (int32_t j = 0; j < rows; j++) { printf("%s |", flag); - for (int32_t k = 0; k < colNum; k++) { + for (int32_t k = 0; k < numOfCols; k++) { SColumnInfoData* pColInfoData = taosArrayGet(pDataBlock->pDataBlock, k); void* var = POINTER_SHIFT(pColInfoData->pData, j * pColInfoData->info.bytes); if (colDataIsNull(pColInfoData, rows, j, NULL)) { @@ -1561,7 +1611,9 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks int32_t bufSize = sizeof(SSubmitReq); for (int32_t i = 0; i < sz; ++i) { SDataBlockInfo* pBlkInfo = &((SSDataBlock*)taosArrayGet(pDataBlocks, i))->info; - bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(pBlkInfo->numOfCols)); + + int32_t numOfCols = taosArrayGetSize(pDataBlocks); + bufSize += pBlkInfo->rows * (TD_ROW_HEAD_LEN + pBlkInfo->rowSize + BitmapLen(numOfCols)); bufSize += sizeof(SSubmitBlk); } @@ -1579,10 +1631,10 @@ int32_t buildSubmitReqFromDataBlock(SSubmitReq** pReq, const SArray* pDataBlocks for (int32_t i = 0; i < sz; ++i) { SSDataBlock* pDataBlock = taosArrayGet(pDataBlocks, i); - int32_t colNum = pDataBlock->info.numOfCols; + int32_t colNum = taosArrayGetSize(pDataBlock->pDataBlock); int32_t rows = pDataBlock->info.rows; - int32_t rowSize = pDataBlock->info.rowSize; - int64_t groupId = pDataBlock->info.groupId; +// int32_t rowSize = pDataBlock->info.rowSize; +// int64_t groupId = pDataBlock->info.groupId; if (colNum <= 1) { // invalid if only with TS col @@ -1800,8 +1852,6 @@ void blockCompressEncode(const SSDataBlock* pBlock, char* data, int32_t* dataLen } const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t numOfRows, const char* pData) { - blockDataEnsureCapacity(pBlock, numOfRows); - const char* pStart = pData; int32_t dataLen = *(int32_t*)pStart; @@ -1815,9 +1865,6 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t taosArraySetSize(pBlock->pDataBlock, numOfCols); } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); - ASSERT(pBlock->pDataBlock->size >= numOfCols); - for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); pColInfoData->info.type = *(int16_t*)pStart; @@ -1842,28 +1889,26 @@ const char* blockCompressDecode(SSDataBlock* pBlock, int32_t numOfCols, int32_t SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); if (IS_VAR_DATA_TYPE(pColInfoData->info.type)) { - pColInfoData->varmeta.length = colLen[i]; - pColInfoData->varmeta.allocLen = colLen[i]; - memcpy(pColInfoData->varmeta.offset, pStart, sizeof(int32_t) * numOfRows); pStart += sizeof(int32_t) * numOfRows; - if (colLen[i] > 0) { - taosMemoryFreeClear(pColInfoData->pData); - pColInfoData->pData = taosMemoryMalloc(colLen[i]); + if (colLen[i] > 0 && pColInfoData->varmeta.allocLen < colLen[i]) { + char* tmp = taosMemoryRealloc(pColInfoData->pData, colLen[i]); + if (tmp == NULL) { + return NULL; + } + + pColInfoData->pData = tmp; + pColInfoData->varmeta.allocLen = colLen[i]; } + + pColInfoData->varmeta.length = colLen[i]; } else { - if (pColInfoData->nullbitmap == NULL) { - pColInfoData->nullbitmap = taosMemoryCalloc(1, BitmapLen(numOfRows)); - } memcpy(pColInfoData->nullbitmap, pStart, BitmapLen(numOfRows)); pStart += BitmapLen(numOfRows); } if (colLen[i] > 0) { - if (pColInfoData->pData == NULL) { - pColInfoData->pData = taosMemoryCalloc(1, colLen[i]); - } memcpy(pColInfoData->pData, pStart, colLen[i]); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index 1ddb606ccfa2f721e5de4b704a2c63ff2fd4fe6e..8460a27a0e6cf598437ab66434eba3103615cb9c 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -924,6 +924,18 @@ static int32_t tGetTagVal(uint8_t *p, STagVal *pTagVal, int8_t isJson) { return n; } + +bool tTagIsJson(const void *pTag){ + return (((const STag *)pTag)->flags & TD_TAG_JSON); +} + +bool tTagIsJsonNull(void *data){ + STag *pTag = (STag*)data; + int8_t isJson = tTagIsJson(pTag); + if(!isJson) return false; + return ((STag*)data)->nTag == 0; +} + int32_t tTagNew(SArray *pArray, int32_t version, int8_t isJson, STag **ppTag) { int32_t code = 0; uint8_t *p = NULL; diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 269c92a670ffed95c2cd2b5d1f76bf97d5615f1b..2172a6fe5030b1d2899d0515b98023ed8f231a7c 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -187,6 +187,9 @@ bool tsStartUdfd = true; // internal int32_t tsTransPullupInterval = 2; int32_t tsMqRebalanceInterval = 2; +int32_t tsTtlUnit = 86400; +int32_t tsTtlPushInterval = 60; + void taosAddDataDir(int32_t index, char *v1, int32_t level, int32_t primary) { tstrncpy(tsDiskCfg[index].dir, v1, TSDB_FILENAME_LEN); @@ -467,6 +470,8 @@ static int32_t taosAddServerCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "transPullupInterval", tsTransPullupInterval, 1, 10000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "mqRebalanceInterval", tsMqRebalanceInterval, 1, 10000, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlUnit", tsTtlUnit, 1, 86400*365, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "ttlPushInterval", tsTtlPushInterval, 1, 10000, 1) != 0) return -1; if (cfgAddBool(pCfg, "udf", tsStartUdfd, 0) != 0) return -1; return 0; @@ -619,6 +624,8 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { tsTransPullupInterval = cfgGetItem(pCfg, "transPullupInterval")->i32; tsMqRebalanceInterval = cfgGetItem(pCfg, "mqRebalanceInterval")->i32; + tsTtlUnit = cfgGetItem(pCfg, "ttlUnit")->i32; + tsTtlPushInterval = cfgGetItem(pCfg, "ttlPushInterval")->i32; tsStartUdfd = cfgGetItem(pCfg, "udf")->bval; @@ -631,7 +638,7 @@ static int32_t taosSetServerCfg(SConfig *pCfg) { int32_t taosCreateLog(const char *logname, int32_t logFileNum, const char *cfgDir, const char **envCmd, const char *envFile, char *apolloUrl, SArray *pArgs, bool tsc) { - osDefaultInit(); + if (tsCfg == NULL) osDefaultInit(); SConfig *pCfg = cfgInit(); if (pCfg == NULL) return -1; @@ -753,9 +760,9 @@ void taosCleanupCfg() { } void taosCfgDynamicOptions(const char *option, const char *value) { - if (strcasecmp(option, "debugFlag") == 0) { - int32_t debugFlag = atoi(value); - taosSetAllDebugFlag(debugFlag); + if (strncasecmp(option, "debugFlag", 9) == 0) { + int32_t flag = atoi(value); + taosSetAllDebugFlag(flag); } if (strcasecmp(option, "resetlog") == 0) { diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index d078d22cdf36a2ab365c757217e28f750ec137b3..4ad11f2998df6a6ee5436e0895c90bfcdab458b2 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -127,7 +127,7 @@ int32_t tEncodeSEpSet(SEncoder *pEncoder, const SEpSet *pEp) { if (tEncodeI8(pEncoder, pEp->numOfEps) < 0) return -1; for (int32_t i = 0; i < TSDB_MAX_REPLICA; i++) { if (tEncodeU16(pEncoder, pEp->eps[i].port) < 0) return -1; - if (tEncodeCStr(pEncoder, pEp->eps[i].fqdn) < 0) return -1; + if (tEncodeCStrWithLen(pEncoder, pEp->eps[i].fqdn, TSDB_FQDN_LEN) < 0) return -1; } return 0; } @@ -532,6 +532,14 @@ int32_t tSerializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pReq if (pReq->ast2Len > 0) { if (tEncodeBinary(&encoder, pReq->pAst2, pReq->ast2Len) < 0) return -1; } + + int32_t numOfFuncs = taosArrayGetSize(pReq->pFuncs); + if (tEncodeI32(&encoder, numOfFuncs) < 0) return -1; + for (int32_t i = 0; i < numOfFuncs; ++i) { + const char *pFunc = taosArrayGet(pReq->pFuncs, i); + if (tEncodeCStr(&encoder, pFunc) < 0) return -1; + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -606,6 +614,21 @@ int32_t tDeserializeSMCreateStbReq(void *buf, int32_t bufLen, SMCreateStbReq *pR if (tDecodeCStrTo(&decoder, pReq->pAst2) < 0) return -1; } + int32_t numOfFuncs = 0; + if (tDecodeI32(&decoder, &numOfFuncs) < 0) return -1; + if (numOfFuncs > 0) { + pReq->pFuncs = taosArrayInit(numOfFuncs, TSDB_FUNC_NAME_LEN); + if (NULL == pReq->pFuncs) return -1; + } + for (int32_t i = 0; i < numOfFuncs; ++i) { + char* pFunc = NULL; + if (tDecodeCStrAlloc(&decoder, &pFunc) < 0) return -1; + if (taosArrayPush(pReq->pFuncs, pFunc) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -618,8 +641,7 @@ void tFreeSMCreateStbReq(SMCreateStbReq *pReq) { taosMemoryFreeClear(pReq->comment); taosMemoryFreeClear(pReq->pAst1); taosMemoryFreeClear(pReq->pAst2); - pReq->pColumns = NULL; - pReq->pTags = NULL; + taosArrayDestroy(pReq->pFuncs); } int32_t tSerializeSMDropStbReq(void *buf, int32_t bufLen, SMDropStbReq *pReq) { @@ -1478,6 +1500,33 @@ int32_t tDeserializeSMCfgDnodeReq(void *buf, int32_t bufLen, SMCfgDnodeReq *pReq return 0; } +int32_t tSerializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->config) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->value) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDCfgDnodeReq(void *buf, int32_t bufLen, SDCfgDnodeReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->config) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->value) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSCreateDnodeReq(void *buf, int32_t bufLen, SCreateDnodeReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -1757,6 +1806,165 @@ void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) { taosArrayDestroy(pRsp->pFuncInfos); } +int32_t tSerializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + if (buf != NULL) { + buf = (char *)buf + headLen; + bufLen -= headLen; + } + + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->dbFName) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->tbName) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + + if (buf != NULL) { + SMsgHead *pHead = (SMsgHead *)((char *)buf - headLen); + pHead->vgId = htonl(pReq->header.vgId); + pHead->contLen = htonl(tlen + headLen); + } + + return tlen + headLen; +} + +int32_t tDeserializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq) { + int32_t headLen = sizeof(SMsgHead); + + SMsgHead *pHead = buf; + pHead->vgId = pReq->header.vgId; + pHead->contLen = pReq->header.contLen; + + SDecoder decoder = {0}; + tDecoderInit(&decoder, (char *)buf + headLen, bufLen - headLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->dbFName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->tbName) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + +int32_t tSerializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->tbName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->stbName) < 0) return -1; + if (tEncodeCStr(&encoder, pRsp->dbFName) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfTags) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->numOfColumns) < 0) return -1; + if (tEncodeI8(&encoder, pRsp->tableType) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->delay1) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->delay2) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->watermark1) < 0) return -1; + if (tEncodeI64(&encoder, pRsp->watermark2) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->ttl) < 0) return -1; + + int32_t numOfFuncs = taosArrayGetSize(pRsp->pFuncs); + if (tEncodeI32(&encoder, numOfFuncs) < 0) return -1; + for (int32_t i = 0; i < numOfFuncs; ++i) { + const char *pFunc = taosArrayGet(pRsp->pFuncs, i); + if (tEncodeCStr(&encoder, pFunc) < 0) return -1; + } + + if (tEncodeI32(&encoder, pRsp->commentLen) < 0) return -1; + if (pRsp->commentLen > 0) { + if (tEncodeCStr(&encoder, pRsp->pComment) < 0) return -1; + } + + for (int32_t i = 0; i < pRsp->numOfColumns + pRsp->numOfTags; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + if (tEncodeSSchema(&encoder, pSchema) < 0) return -1; + } + + if (tEncodeI32(&encoder, pRsp->tagsLen) < 0) return -1; + if (tEncodeBinary(&encoder, pRsp->pTags, pRsp->tagsLen) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSTableCfgRsp(void *buf, int32_t bufLen, STableCfgRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->tbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->stbName) < 0) return -1; + if (tDecodeCStrTo(&decoder, pRsp->dbFName) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfTags) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->numOfColumns) < 0) return -1; + if (tDecodeI8(&decoder, &pRsp->tableType) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->delay1) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->delay2) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->watermark1) < 0) return -1; + if (tDecodeI64(&decoder, &pRsp->watermark2) < 0) return -1; + if (tDecodeI32(&decoder, &pRsp->ttl) < 0) return -1; + + int32_t numOfFuncs = 0; + if (tDecodeI32(&decoder, &numOfFuncs) < 0) return -1; + if (numOfFuncs > 0) { + pRsp->pFuncs = taosArrayInit(numOfFuncs, TSDB_FUNC_NAME_LEN); + if (NULL == pRsp->pFuncs) return -1; + } + for (int32_t i = 0; i < numOfFuncs; ++i) { + char pFunc[TSDB_FUNC_NAME_LEN]; + if (tDecodeCStrTo(&decoder, pFunc) < 0) return -1; + if (taosArrayPush(pRsp->pFuncs, pFunc) == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + } + + if (tDecodeI32(&decoder, &pRsp->commentLen) < 0) return -1; + if (pRsp->commentLen > 0) { + if (tDecodeCStrAlloc(&decoder, &pRsp->pComment) < 0) return -1; + } else { + pRsp->pComment = NULL; + } + + int32_t totalCols = pRsp->numOfTags + pRsp->numOfColumns; + pRsp->pSchemas = taosMemoryMalloc(sizeof(SSchema) * totalCols); + if (pRsp->pSchemas == NULL) return -1; + + for (int32_t i = 0; i < totalCols; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + if (tDecodeSSchema(&decoder, pSchema) < 0) return -1; + } + + if (tDecodeI32(&decoder, &pRsp->tagsLen) < 0) return -1; + if (tDecodeBinaryAlloc(&decoder, (void**)&pRsp->pTags, NULL) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +void tFreeSTableCfgRsp(STableCfgRsp *pRsp) { + if (NULL == pRsp) { + return; + } + + taosMemoryFreeClear(pRsp->pComment); + taosMemoryFreeClear(pRsp->pSchemas); + taosMemoryFreeClear(pRsp->pTags); + + taosArrayDestroy(pRsp->pFuncs); +} + int32_t tSerializeSCreateDbReq(void *buf, int32_t bufLen, SCreateDbReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2013,6 +2221,32 @@ int32_t tDeserializeSQnodeListReq(void *buf, int32_t bufLen, SQnodeListReq *pReq return 0; } +int32_t tSerializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->rowNum) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDnodeListReq(void *buf, int32_t bufLen, SDnodeListReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->rowNum) < 0) return -1; + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + + int32_t tSerializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2056,6 +2290,50 @@ int32_t tDeserializeSQnodeListRsp(void *buf, int32_t bufLen, SQnodeListRsp *pRsp void tFreeSQnodeListRsp(SQnodeListRsp *pRsp) { taosArrayDestroy(pRsp->qnodeList); } +int32_t tSerializeSDnodeListRsp(void *buf, int32_t bufLen, SDnodeListRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + int32_t num = taosArrayGetSize(pRsp->dnodeList); + if (tEncodeI32(&encoder, num) < 0) return -1; + for (int32_t i = 0; i < num; ++i) { + SEpSet *pEpSet = taosArrayGet(pRsp->dnodeList, i); + if (tEncodeSEpSet(&encoder, pEpSet) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSDnodeListRsp(void *buf, int32_t bufLen, SDnodeListRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + int32_t num = 0; + if (tDecodeI32(&decoder, &num) < 0) return -1; + if (NULL == pRsp->dnodeList) { + pRsp->dnodeList = taosArrayInit(num, sizeof(SEpSet)); + if (NULL == pRsp->dnodeList) return -1; + } + + for (int32_t i = 0; i < num; ++i) { + SEpSet epSet = {0}; + if (tDecodeSEpSet(&decoder, &epSet) < 0) return -1; + taosArrayPush(pRsp->dnodeList, &epSet); + } + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +void tFreeSDnodeListRsp(SDnodeListRsp *pRsp) { taosArrayDestroy(pRsp->dnodeList); } + + int32_t tSerializeSCompactDbReq(void *buf, int32_t bufLen, SCompactDbReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2552,6 +2830,31 @@ void tFreeSTableIndexInfo(void *info) { taosMemoryFree(pInfo->expr); } +int32_t tSerializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeI32(&encoder, pReq->useless) < 0) return -1; + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSShowVariablesReq(void *buf, int32_t bufLen, SShowVariablesReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeI32(&decoder, &pReq->useless) < 0) return -1; + + tEndDecode(&decoder); + tDecoderClear(&decoder); + return 0; +} + int32_t tSerializeSShowReq(void *buf, int32_t bufLen, SShowReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -2956,6 +3259,7 @@ int32_t tSerializeSCMCreateTopicReq(void *buf, int32_t bufLen, const SCMCreateTo if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; if (tEncodeI8(&encoder, pReq->igExists) < 0) return -1; if (tEncodeI8(&encoder, pReq->subType) < 0) return -1; + if (tEncodeI8(&encoder, pReq->withMeta) < 0) return -1; if (tEncodeCStr(&encoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { @@ -2985,6 +3289,7 @@ int32_t tDeserializeSCMCreateTopicReq(void *buf, int32_t bufLen, SCMCreateTopicR if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; if (tDecodeI8(&decoder, &pReq->igExists) < 0) return -1; if (tDecodeI8(&decoder, &pReq->subType) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->withMeta) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->subDbName) < 0) return -1; if (TOPIC_SUB_TYPE__DB == pReq->subType) { } else if (TOPIC_SUB_TYPE__TABLE == pReq->subType) { @@ -3052,7 +3357,7 @@ int32_t tSerializeSConnectReq(void *buf, int32_t bufLen, SConnectReq *pReq) { if (tEncodeCStr(&encoder, pReq->app) < 0) return -1; if (tEncodeCStr(&encoder, pReq->db) < 0) return -1; if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; - if (tEncodeCStr(&encoder, pReq->passwd) < 0) return -1; + if (tEncodeCStrWithLen(&encoder, pReq->passwd, TSDB_PASSWORD_LEN) < 0) return -1; if (tEncodeI64(&encoder, pReq->startTime) < 0) return -1; tEndEncode(&encoder); @@ -4267,6 +4572,35 @@ int32_t tDeserializeSCMCreateStreamReq(void *buf, int32_t bufLen, SCMCreateStrea return 0; } +int32_t tSerializeSMDropStreamReq(void *buf, int32_t bufLen, const SMDropStreamReq *pReq) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + if (tEncodeCStr(&encoder, pReq->name) < 0) return -1; + if (tEncodeI8(&encoder, pReq->igNotExists) < 0) return -1; + + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSMDropStreamReq(void *buf, int32_t bufLen, SMDropStreamReq *pReq) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + if (tDecodeCStrTo(&decoder, pReq->name) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->igNotExists) < 0) return -1; + + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + void tFreeSCMCreateStreamReq(SCMCreateStreamReq *pReq) { taosMemoryFreeClear(pReq->sql); taosMemoryFreeClear(pReq->ast); diff --git a/source/common/src/tname.c b/source/common/src/tname.c index fd055135799a5e508ec535b43d46e9246c8d644e..ce454d439d9590c1068206eaab94f07ec3e86a16 100644 --- a/source/common/src/tname.c +++ b/source/common/src/tname.c @@ -240,6 +240,15 @@ bool tNameDBNameEqual(SName* left, SName* right) { return (0 == strcmp(left->dbname, right->dbname)); } +bool tNameTbNameEqual(SName* left, SName* right) { + bool equal = tNameDBNameEqual(left, right); + if (equal) { + return (0 == strcmp(left->tname, right->tname)); + } + + return equal; +} + int32_t tNameFromString(SName* dst, const char* str, uint32_t type) { assert(dst != NULL && str != NULL && strlen(str) > 0); @@ -312,8 +321,10 @@ void buildChildTableName(RandTableName* rName) { taosStringBuilderAppendStringLen(&sb, rName->sTableName, rName->sTableNameLen); taosArraySort(rName->tags, compareKv); for (int j = 0; j < taosArrayGetSize(rName->tags); ++j) { + taosStringBuilderAppendChar(&sb, ','); SSmlKv* tagKv = taosArrayGetP(rName->tags, j); taosStringBuilderAppendStringLen(&sb, tagKv->key, tagKv->keyLen); + taosStringBuilderAppendChar(&sb, '='); if (IS_VAR_DATA_TYPE(tagKv->type)) { taosStringBuilderAppendStringLen(&sb, tagKv->value, tagKv->length); } else { @@ -326,9 +337,14 @@ void buildChildTableName(RandTableName* rName) { tMD5Init(&context); tMD5Update(&context, (uint8_t*)keyJoined, (uint32_t)len); tMD5Final(&context); - uint64_t digest1 = *(uint64_t*)(context.digest); - uint64_t digest2 = *(uint64_t*)(context.digest + 8); - snprintf(rName->childTableName, TSDB_TABLE_NAME_LEN, "t_%016" PRIx64 "%016" PRIx64, digest1, digest2); + + char temp[8] = {0}; + rName->childTableName[0] = 't'; + rName->childTableName[1] = '_'; + for(int i = 0; i < 16; i++){ + sprintf(temp, "%02x", context.digest[i]); + strcat(rName->childTableName, temp); + } taosStringBuilderDestroy(&sb); - rName->uid = digest1; + rName->uid = *(uint64_t*)(context.digest); } diff --git a/source/common/src/ttime.c b/source/common/src/ttime.c index 0b59e9b6cc611a9da3362fbef3da410376c0ea58..8fbdeb06545d9bdd31db2e630b8a7b7c52281587 100644 --- a/source/common/src/ttime.c +++ b/source/common/src/ttime.c @@ -76,22 +76,28 @@ void deltaToUtcInitOnce() { static int64_t parseFraction(char* str, char** end, int32_t timePrec); static int32_t parseTimeWithTz(const char* timestr, int64_t* time, int32_t timePrec, char delim); -static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec); -static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec); +static int32_t parseLocaltime(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); +static int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim); static char* forwardToTimeStringEnd(char* str); static bool checkTzPresent(const char* str, int32_t len); -static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec) = {parseLocaltime, - parseLocaltimeDst}; +static int32_t (*parseLocaltimeFp[])(char* timestr, int32_t len, int64_t* utime, int32_t timePrec, char delim) = {parseLocaltime, + parseLocaltimeDst}; int32_t taosParseTime(const char* timestr, int64_t* utime, int32_t len, int32_t timePrec, int8_t day_light) { /* parse datatime string in with tz */ if (strnchr(timestr, 'T', len, false) != NULL) { - return parseTimeWithTz(timestr, utime, timePrec, 'T'); - } else if (checkTzPresent(timestr, len)) { - return parseTimeWithTz(timestr, utime, timePrec, 0); + if (checkTzPresent(timestr, len)) { + return parseTimeWithTz(timestr, utime, timePrec, 'T'); + } else { + return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 'T'); + } } else { - return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec); + if (checkTzPresent(timestr, len)) { + return parseTimeWithTz(timestr, utime, timePrec, 0); + } else { + return (*parseLocaltimeFp[day_light])((char*)timestr, len, utime, timePrec, 0); + } } } @@ -333,13 +339,25 @@ static FORCE_INLINE bool validateTm(struct tm* pTm) { return true; } -int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec) { +int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) { *time = 0; struct tm tm = {0}; - char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + char *str; + if (delim == 'T') { + str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); + } else if (delim == 0) { + str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + } else { + str = NULL; + } + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - return -1; + //if parse failed, try "%Y-%m-%d" format + str = taosStrpTime(timestr, "%Y-%m-%d", &tm); + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { + return -1; + } } #ifdef _MSC_VER @@ -367,14 +385,26 @@ int32_t parseLocaltime(char* timestr, int32_t len, int64_t* time, int32_t timePr return 0; } -int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec) { +int32_t parseLocaltimeDst(char* timestr, int32_t len, int64_t* time, int32_t timePrec, char delim) { *time = 0; struct tm tm = {0}; tm.tm_isdst = -1; - char* str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + char *str; + if (delim == 'T') { + str = taosStrpTime(timestr, "%Y-%m-%dT%H:%M:%S", &tm); + } else if (delim == 0) { + str = taosStrpTime(timestr, "%Y-%m-%d %H:%M:%S", &tm); + } else { + str = NULL; + } + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { - return -1; + //if parse failed, try "%Y-%m-%d" format + str = taosStrpTime(timestr, "%Y-%m-%d", &tm); + if (str == NULL || (((str - timestr) < len) && (*str != '.')) || !validateTm(&tm)) { + return -1; + } } /* mktime will be affected by TZ, set by using taos_options */ diff --git a/source/common/test/commonTests.cpp b/source/common/test/commonTests.cpp index 2adb558d011aadc5e75c3f04d6c5ab6dfb4888d6..8460d71e56ea70b5b5bc5c5a536476c98e007716 100644 --- a/source/common/test/commonTests.cpp +++ b/source/common/test/commonTests.cpp @@ -88,26 +88,15 @@ TEST(testCase, toInteger_test) { } TEST(testCase, Datablock_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; + SSDataBlock* b = createDataBlock(); - infoData.pData = (char*)taosMemoryCalloc(40, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (40 / 8)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); taosArrayPush(b->pDataBlock, &infoData); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; - - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(40, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); + blockDataEnsureCapacity(b, 40); char* str = "the value of: %d"; char buf[128] = {0}; @@ -222,28 +211,17 @@ TEST(testCase, non_var_dataBlock_split_test) { #endif TEST(testCase, var_dataBlock_split_test) { - SSDataBlock* b = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - b->info.numOfCols = 2; - b->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - int32_t numOfRows = 1000000; - SColumnInfoData infoData = {0}; - infoData.info.bytes = 4; - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.colId = 1; + SSDataBlock* b = createDataBlock(); - infoData.pData = (char*)taosMemoryCalloc(numOfRows, infoData.info.bytes); - infoData.nullbitmap = (char*)taosMemoryCalloc(1, sizeof(char) * (numOfRows / 8)); - taosArrayPush(b->pDataBlock, &infoData); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 1); + blockDataAppendColInfo(b, &infoData); - SColumnInfoData infoData1 = {0}; - infoData1.info.bytes = 40; - infoData1.info.type = TSDB_DATA_TYPE_BINARY; - infoData1.info.colId = 2; + SColumnInfoData infoData1 = createColumnInfoData(TSDB_DATA_TYPE_BINARY, 40, 2); + blockDataAppendColInfo(b, &infoData1); - infoData1.varmeta.offset = (int32_t*)taosMemoryCalloc(numOfRows, sizeof(uint32_t)); - taosArrayPush(b->pDataBlock, &infoData1); + blockDataEnsureCapacity(b, numOfRows); char buf[41] = {0}; char buf1[100] = {0}; diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index ee811c0071cbd07c03edb7aaf117c3c4461adebb..4479c06bea5b16701790c43472bb9201e61e57a6 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -45,6 +45,7 @@ int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessAuthRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); // dmWorker.c int32_t dmPutNodeMsgToMgmtQueue(SDnodeMgmt *pMgmt, SRpcMsg *pMsg); diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 3e55469a4a3a7d551b6015b051cac184d38944f7..3d1ece0b6407ccc077692e6e6ff54859ce6fd5c1 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -15,6 +15,10 @@ #define _DEFAULT_SOURCE #include "dmInt.h" +#include "systable.h" + + +extern SConfig *tsCfg; static void dmUpdateDnodeCfg(SDnodeMgmt *pMgmt, SDnodeCfg *pCfg) { if (pMgmt->pData->dnodeId == 0 || pMgmt->pData->clusterId == 0) { @@ -119,8 +123,15 @@ int32_t dmProcessGrantRsp(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t dmProcessConfigReq(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { - dError("config req is received, but not supported yet"); - return TSDB_CODE_OPS_NOT_SUPPORT; + SDCfgDnodeReq cfgReq = {0}; + if (tDeserializeSDCfgDnodeReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + dInfo("start to config, option:%s, value:%s", cfgReq.config, cfgReq.value); + taosCfgDynamicOptions(cfgReq.config, cfgReq.value); + return 0; } static void dmGetServerRunStatus(SDnodeMgmt *pMgmt, SServerStatusRsp *pStatus) { @@ -175,6 +186,130 @@ int32_t dmProcessServerRunStatus(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { return 0; } +SSDataBlock* dmBuildVariablesBlock(void) { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + size_t size = 0; + const SSysTableMeta* pMeta = NULL; + getInfosDbMeta(&pMeta, &size); + + int32_t index = 0; + for (int32_t i = 0; i < size; ++i) { + if (strcmp(pMeta[i].name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) { + index = i; + break; + } + } + + pBlock->pDataBlock = taosArrayInit(pMeta[index].colNum, sizeof(SColumnInfoData)); + + for (int32_t i = 0; i < pMeta[index].colNum; ++i) { + SColumnInfoData colInfoData = {0}; + colInfoData.info.colId = i + 1; + colInfoData.info.type = pMeta[index].schema[i].type; + colInfoData.info.bytes = pMeta[index].schema[i].bytes; + taosArrayPush(pBlock->pDataBlock, &colInfoData); + } + + pBlock->info.hasVarCol = true; + + return pBlock; +} + +int32_t dmAppendVariablesToBlock(SSDataBlock* pBlock, int32_t dnodeId) { + int32_t numOfCfg = taosArrayGetSize(tsCfg->array); + int32_t numOfRows = 0; + blockDataEnsureCapacity(pBlock, numOfCfg); + + for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { + SConfigItem *pItem = taosArrayGet(tsCfg->array, i); + + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, (const char *)&dnodeId, false); + + char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, name, false); + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t valueLen = 0; + cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); + varDataSetLen(value, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, value, false); + + numOfRows++; + } + + + pBlock->info.rows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + + + +int32_t dmProcessRetrieve(SDnodeMgmt *pMgmt, SRpcMsg *pMsg) { + int32_t size = 0; + int32_t rowsRead = 0; + + SRetrieveTableReq retrieveReq = {0}; + if (tDeserializeSRetrieveTableReq(pMsg->pCont, pMsg->contLen, &retrieveReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if (strcasecmp(retrieveReq.tb, TSDB_INS_TABLE_DNODE_VARIABLES)) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + SSDataBlock* pBlock = dmBuildVariablesBlock(); + + dmAppendVariablesToBlock(pBlock, pMgmt->pData->dnodeId); + + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * numOfCols + + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(numOfCols); + + SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + dError("failed to retrieve data since %s", terrstr()); + blockDataDestroy(pBlock); + return -1; + } + + char *pStart = pRsp->data; + *(int32_t *)pStart = htonl(numOfCols); + pStart += sizeof(int32_t); // number of columns + + for (int32_t i = 0; i < numOfCols; ++i) { + SSysTableSchema *pSchema = (SSysTableSchema *)pStart; + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + pSchema->bytes = htonl(pColInfo->info.bytes); + pSchema->colId = htons(pColInfo->info.colId); + pSchema->type = pColInfo->info.type; + + pStart += sizeof(SSysTableSchema); + } + + int32_t len = 0; + blockCompressEncode(pBlock, pStart, &len, numOfCols, false); + + pRsp->numOfRows = htonl(pBlock->info.rows); + pRsp->precision = TSDB_TIME_PRECISION_MILLI; // millisecond time precision + pRsp->completed = 1; + pMsg->info.rsp = pRsp; + pMsg->info.rspLen = size; + dDebug("dnode variables retrieve completed"); + + blockDataDestroy(pBlock); + return TSDB_CODE_SUCCESS; +} + + SArray *dmGetMsgHandles() { int32_t code = -1; SArray *pArray = taosArrayInit(16, sizeof(SMgmtHandle)); @@ -191,6 +326,7 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_BNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; // Requests handled by MNODE if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT_RSP, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 9ffa0e606a6e597e247afa24356794b421e30e5f..89e8aa976ef33b03749a95e55b9640d414a9dca8 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -141,6 +141,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_SERVER_STATUS: code = dmProcessServerRunStatus(pMgmt, pMsg); break; + case TDMT_DND_SYSTABLE_RETRIEVE: + code = dmProcessRetrieve(pMgmt, pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; break; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 0112feedd21a52c78abe505b3e5ae6bce382f97f..d44a7d79bfc98d765065fc6bfa19facac8e1d9ae 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -73,6 +73,7 @@ int32_t mmProcessGetLoadsReq(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { } int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { + const STraceId *trace = &pMsg->info.traceId; SDCreateMnodeReq createReq = {0}; if (tDeserializeSDCreateMnodeReq(pMsg->pCont, pMsg->contLen, &createReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -81,7 +82,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { if (createReq.replica != 1) { terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to create mnode since %s", terrstr()); + dGError("failed to create mnode since %s", terrstr()); return -1; } @@ -91,7 +92,7 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { mgmt.path = pInput->path; mgmt.name = pInput->name; if (mmWriteFile(&mgmt, &createReq.replicas[0], deployed) != 0) { - dError("failed to write mnode file since %s", terrstr()); + dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -99,7 +100,8 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { } int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { - SDDropMnodeReq dropReq = {0}; + const STraceId *trace = &pMsg->info.traceId; + SDDropMnodeReq dropReq = {0}; if (tDeserializeSCreateDropMQSBNodeReq(pMsg->pCont, pMsg->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; return -1; @@ -107,7 +109,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { if (pInput->pData->dnodeId != 0 && dropReq.dnodeId != pInput->pData->dnodeId) { terrno = TSDB_CODE_INVALID_OPTION; - dError("failed to drop mnode since %s", terrstr()); + dGError("failed to drop mnode since %s", terrstr()); return -1; } @@ -117,7 +119,7 @@ int32_t mmProcessDropReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { mgmt.path = pInput->path; mgmt.name = pInput->name; if (mmWriteFile(&mgmt, NULL, deployed) != 0) { - dError("failed to write mnode file since %s", terrstr()); + dGError("failed to write mnode file since %s", terrstr()); return -1; } @@ -159,6 +161,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_QNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_QNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_DNODE_LIST, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_BNODE, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -180,6 +183,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MND_ALTER_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_STB, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_META, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_MND_TABLE_CFG, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_DROP_SMA, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_STREAM, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -216,7 +220,7 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_MQ_VG_DELETE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, mmPutMsgToFetchQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIRM_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c index 60c42d31f5c79363d6bde886e843e4df182f066d..7f3f76b4b60c95aa0adb493bba4f9af70ef9b954 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmWorker.c @@ -18,7 +18,6 @@ static inline int32_t mmAcquire(SMnodeMgmt *pMgmt) { int32_t code = 0; - taosThreadRwlockRdlock(&pMgmt->lock); if (pMgmt->stopped) { code = -1; @@ -48,7 +47,8 @@ static inline void mmSendRsp(SRpcMsg *pMsg, int32_t code) { static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; int32_t code = -1; - STraceId * trace = &pMsg->info.traceId; + + const STraceId *trace = &pMsg->info.traceId; dGTrace("msg:%p, get from mnode queue", pMsg); switch (pMsg->msgType) { @@ -68,7 +68,11 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { mmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + if (code == TSDB_CODE_RPC_REDIRECT) { + mndPostProcessQueryMsg(pMsg); + } + + dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -76,7 +80,9 @@ static void mmProcessRpcMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { SMnodeMgmt *pMgmt = pInfo->ahandle; pMsg->info.node = pMgmt->pMnode; - dTrace("msg:%p, get from mnode-sync queue", pMsg); + + const STraceId *trace = &pMsg->info.traceId; + dGTrace("msg:%p, get from mnode-sync queue", pMsg); SMsgHead *pHead = pMsg->pCont; pHead->contLen = ntohl(pHead->contLen); @@ -84,20 +90,22 @@ static void mmProcessSyncMsg(SQueueInfo *pInfo, SRpcMsg *pMsg) { int32_t code = mndProcessSyncMsg(pMsg); - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static inline int32_t mmPutMsgToWorker(SMnodeMgmt *pMgmt, SSingleWorker *pWorker, SRpcMsg *pMsg) { + const STraceId *trace = &pMsg->info.traceId; + if (mmAcquire(pMgmt) == 0) { - dTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); + dGTrace("msg:%p, put into %s queue, type:%s", pMsg, pWorker->name, TMSG_INFO(pMsg->msgType)); taosWriteQitem(pWorker->queue, pMsg); mmRelease(pMgmt); return 0; } else { - dTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(), - TMSG_INFO(pMsg->msgType)); + dGTrace("msg:%p, failed to put into %s queue since %s, type:%s", pMsg, pWorker->name, terrstr(), + TMSG_INFO(pMsg->msgType)); return -1; } } @@ -116,20 +124,18 @@ int32_t mmPutMsgToReadQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { int32_t mmPutMsgToQueryQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { pMsg->info.node = pMgmt->pMnode; - if (mndPreProcessMsg(pMsg) != 0) { - dError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); + if (mndPreProcessQueryMsg(pMsg) != 0) { + const STraceId *trace = &pMsg->info.traceId; + dGError("msg:%p, failed to pre-process in mnode since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); return -1; } return mmPutMsgToWorker(pMgmt, &pMgmt->queryWorker, pMsg); } int32_t mmPutMsgToFetchQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { - pMsg->info.node = pMgmt->pMnode; - return mmPutMsgToWorker(pMgmt, &pMgmt->fetchWorker, pMsg); } - int32_t mmPutMsgToMonitorQueue(SMnodeMgmt *pMgmt, SRpcMsg *pMsg) { return mmPutMsgToWorker(pMgmt, &pMgmt->monitorWorker, pMsg); } diff --git a/source/dnode/mgmt/mgmt_snode/src/smHandle.c b/source/dnode/mgmt/mgmt_snode/src/smHandle.c index 81576e153e35c490860c4b0b9272dcaa2d798418..7cb41ca77cb77f7069ab8e375cef64840b058615 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smHandle.c +++ b/source/dnode/mgmt/mgmt_snode/src/smHandle.c @@ -95,12 +95,14 @@ SArray *smGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_MON_SM_INFO, smPutNodeMsgToMonitorQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, smPutNodeMsgToMgmtQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, smPutNodeMsgToSharedQueue, 1) == NULL) goto _OVER; code = 0; _OVER: diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index c3ed6d781cc8238ee9c901dc53466ec68da74095..00d83f0ad4d5632e66352014d52cfc56f7bd1bfc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -330,6 +330,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_TABLE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_UPDATE_TAG_VAL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TABLE_CFG, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TABLES_META, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_CANCEL_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DROP_TASK, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; @@ -351,13 +352,15 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_QUERY_HEARTBEAT, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TRIGGER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; - if (dmSetMgmtHandle(pArray, TDMT_VND_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DROP, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DEPLOY, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RUN, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_DISPATCH_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_STREAM_TASK_RECOVER_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_STREAM_RETRIEVE_RSP, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_REPLICA, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_ALTER_CONFIG, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index 71bbc8ddd43aaf6583bcf4758ca018fe798108d4..5e1ef23f1c7819d524f99e72485e942a13ee9ecc 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -28,10 +28,10 @@ static inline void vmSendRsp(SRpcMsg *pMsg, int32_t code) { } static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SVnodeMgmt *pMgmt = pInfo->ahandle; - int32_t code = -1; + SVnodeMgmt *pMgmt = pInfo->ahandle; + int32_t code = -1; + const STraceId *trace = &pMsg->info.traceId; - STraceId *trace = &pMsg->info.traceId; dGTrace("msg:%p, get from vnode-mgmt queue", pMsg); switch (pMsg->msgType) { case TDMT_MON_VM_INFO: @@ -48,50 +48,52 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; - dError("msg:%p, not processed in vnode-mgmt queue", pMsg); + dGError("msg:%p, not processed in vnode-mgmt queue", pMsg); } if (IsReq(pMsg)) { if (code != 0) { if (terrno != 0) code = terrno; - dError("msg:%p, failed to process since %s", pMsg, terrstr()); + dGError("msg:%p, failed to process since %s", pMsg, terrstr()); } vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static void vmProcessQueryQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SVnodeObj *pVnode = pInfo->ahandle; + SVnodeObj *pVnode = pInfo->ahandle; + const STraceId *trace = &pMsg->info.traceId; - dTrace("vgId:%d, msg:%p get from vnode-query queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p get from vnode-query queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessQueryMsg(pVnode->pImpl, pMsg); if (code != 0) { if (terrno != 0) code = terrno; - dError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr()); + dGError("vgId:%d, msg:%p failed to query since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } static void vmProcessFetchQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { - SVnodeObj *pVnode = pInfo->ahandle; + SVnodeObj *pVnode = pInfo->ahandle; + const STraceId *trace = &pMsg->info.traceId; - dTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p get from vnode-fetch queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); + dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -102,16 +104,17 @@ static void vmProcessSyncQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numOf for (int32_t i = 0; i < numOfMsgs; ++i) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; - dTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); + const STraceId *trace = &pMsg->info.traceId; + dGTrace("vgId:%d, msg:%p get from vnode-sync queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessSyncReq(pVnode->pImpl, pMsg, NULL); if (code != 0) { if (terrno != 0) code = terrno; - dError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr()); + dGError("vgId:%d, msg:%p failed to sync since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -123,55 +126,57 @@ static void vmProcessMergeQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO for (int32_t i = 0; i < numOfMsgs; ++i) { if (taosGetQitem(qall, (void **)&pMsg) == 0) continue; - dTrace("vgId:%d, msg:%p get from vnode-merge queue", pVnode->vgId, pMsg); + const STraceId *trace = &pMsg->info.traceId; + dGTrace("vgId:%d, msg:%p get from vnode-merge queue", pVnode->vgId, pMsg); int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr()); + dGError("vgId:%d, msg:%p failed to merge since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dTrace("msg:%p, is freed, code:0x%x", pMsg, code); + dGTrace("msg:%p, is freed, code:0x%x", pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } } static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtype) { - SMsgHead *pHead = pMsg->pCont; - int32_t code = 0; + const STraceId *trace = &pMsg->info.traceId; + SMsgHead *pHead = pMsg->pCont; + int32_t code = 0; pHead->contLen = ntohl(pHead->contLen); pHead->vgId = ntohl(pHead->vgId); SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(), - TMSG_INFO(pMsg->msgType)); + dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s", pHead->vgId, pMsg, terrstr(), + TMSG_INFO(pMsg->msgType)); return terrno != 0 ? terrno : -1; } switch (qtype) { case QUERY_QUEUE: vnodePreprocessQueryMsg(pVnode->pImpl, pMsg); - dTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-query queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pQueryQ, pMsg); break; case FETCH_QUEUE: - dTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-fetch queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pFetchQ, pMsg); break; case WRITE_QUEUE: - dTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-write queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pWriteQ, pMsg); break; case SYNC_QUEUE: - dTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-sync queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pSyncQ, pMsg); break; case APPLY_QUEUE: - dTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); + dGTrace("vgId:%d, msg:%p put into vnode-apply queue", pVnode->vgId, pMsg); taosWriteQitem(pVnode->pApplyQ, pMsg); break; default: @@ -193,13 +198,15 @@ int32_t vmPutMsgToQueryQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsg int32_t vmPutMsgToFetchQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return vmPutMsgToQueue(pMgmt, pMsg, FETCH_QUEUE); } int32_t vmPutMsgToMgmtQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - dTrace("msg:%p, put into vnode-mgmt queue", pMsg); + const STraceId *trace = &pMsg->info.traceId; + dGTrace("msg:%p, put into vnode-mgmt queue", pMsg); taosWriteQitem(pMgmt->mgmtWorker.queue, pMsg); return 0; } int32_t vmPutMsgToMonitorQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { - dTrace("msg:%p, put into vnode-monitor queue", pMsg); + const STraceId *trace = &pMsg->info.traceId; + dGTrace("msg:%p, put into vnode-monitor queue", pMsg); taosWriteQitem(pMgmt->monitorWorker.queue, pMsg); return 0; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmTransport.c b/source/dnode/mgmt/node_mgmt/src/dmTransport.c index 660f512fc520c5657d12c0db11aae58085defc9f..63d2a65df144340624acac260b50bdd1406d418b 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmTransport.c +++ b/source/dnode/mgmt/node_mgmt/src/dmTransport.c @@ -17,9 +17,44 @@ #include "dmMgmt.h" #include "qworker.h" -static void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet); -static void dmSendRsp(SRpcMsg *pMsg); -static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg); +static inline void dmSendRsp(SRpcMsg *pMsg) { + SMgmtWrapper *pWrapper = pMsg->info.wrapper; + if (InChildProc(pWrapper)) { + dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP); + } else { + rpcSendResponse(pMsg); + } +} + +static inline void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { + SEpSet epSet = {0}; + dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet); + + const int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); + pMsg->pCont = rpcMallocCont(contLen); + if (pMsg->pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSEpSet(pMsg->pCont, contLen, &epSet); + pMsg->contLen = contLen; + } +} + +static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { + SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; + int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet); + + rsp.pCont = rpcMallocCont(contLen); + if (rsp.pCont == NULL) { + pMsg->code = TSDB_CODE_OUT_OF_MEMORY; + } else { + tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet); + rsp.contLen = contLen; + } + dmSendRsp(&rsp); + rpcFreeCont(pMsg->pCont); + pMsg->pCont = NULL; +} int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { NodeMsgFp msgFp = pWrapper->msgFps[TMSG_INDEX(pMsg->msgType)]; @@ -28,31 +63,39 @@ int32_t dmProcessNodeMsg(SMgmtWrapper *pWrapper, SRpcMsg *pMsg) { return -1; } - dTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); + const STraceId *trace = &pMsg->info.traceId; + dGTrace("msg:%p, will be processed by %s", pMsg, pWrapper->name); pMsg->info.wrapper = pWrapper; return (*msgFp)(pWrapper->pMgmt, pMsg); } static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { - SDnodeTrans * pTrans = &pDnode->trans; + SDnodeTrans *pTrans = &pDnode->trans; int32_t code = -1; - SRpcMsg * pMsg = NULL; + SRpcMsg *pMsg = NULL; SMgmtWrapper *pWrapper = NULL; SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pRpc->msgType)]; - STraceId *trace = &pRpc->info.traceId; + const STraceId *trace = &pRpc->info.traceId; dGTrace("msg:%s is received, handle:%p len:%d code:0x%x app:%p refId:%" PRId64, TMSG_INFO(pRpc->msgType), pRpc->info.handle, pRpc->contLen, pRpc->code, pRpc->info.ahandle, pRpc->info.refId); - if (pRpc->msgType == TDMT_DND_NET_TEST) { - dmProcessNetTestReq(pDnode, pRpc); - return; - } else if (pRpc->msgType == TDMT_MND_SYSTABLE_RETRIEVE_RSP || pRpc->msgType == TDMT_VND_FETCH_RSP) { - qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0); - return; - } else if (pRpc->msgType == TDMT_MND_STATUS_RSP && pEpSet != NULL) { - dmSetMnodeEpSet(&pDnode->data, pEpSet); - } else { + switch (pRpc->msgType) { + case TDMT_DND_NET_TEST: + dmProcessNetTestReq(pDnode, pRpc); + return; + case TDMT_MND_SYSTABLE_RETRIEVE_RSP: + case TDMT_DND_SYSTABLE_RETRIEVE_RSP: + case TDMT_VND_FETCH_RSP: + qWorkerProcessFetchRsp(NULL, NULL, pRpc, 0); + return; + case TDMT_MND_STATUS_RSP: + if (pEpSet != NULL) { + dmSetMnodeEpSet(&pDnode->data, pEpSet); + } + break; + default: + break; } if (pDnode->status != DND_STAT_RUNNING) { @@ -73,39 +116,43 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { if (pHandle->defaultNtype == NODE_END) { terrno = TSDB_CODE_MSG_NOT_PROCESSED; goto _OVER; - } else { - pWrapper = &pDnode->wrappers[pHandle->defaultNtype]; - if (pHandle->needCheckVgId) { - if (pRpc->contLen > 0) { - SMsgHead *pHead = pRpc->pCont; - int32_t vgId = ntohl(pHead->vgId); - if (vgId == QNODE_HANDLE) { + } + + pWrapper = &pDnode->wrappers[pHandle->defaultNtype]; + if (pHandle->needCheckVgId) { + if (pRpc->contLen > 0) { + const SMsgHead *pHead = pRpc->pCont; + const int32_t vgId = ntohl(pHead->vgId); + switch (vgId) { + case QNODE_HANDLE: pWrapper = &pDnode->wrappers[QNODE]; - } else if (vgId == SNODE_HANDLE) { + break; + case SNODE_HANDLE: pWrapper = &pDnode->wrappers[SNODE]; - } else if (vgId == MNODE_HANDLE) { + break; + case MNODE_HANDLE: pWrapper = &pDnode->wrappers[MNODE]; - } else { - } - } else { - terrno = TSDB_CODE_INVALID_MSG_LEN; - goto _OVER; + break; + default: + break; } + } else { + terrno = TSDB_CODE_INVALID_MSG_LEN; + goto _OVER; } } if (dmMarkWrapper(pWrapper) != 0) { pWrapper = NULL; goto _OVER; - } else { - pRpc->info.wrapper = pWrapper; } + pRpc->info.wrapper = pWrapper; pMsg = taosAllocateQitem(sizeof(SRpcMsg), RPC_QITEM); if (pMsg == NULL) goto _OVER; - memcpy(pMsg, pRpc, sizeof(SRpcMsg)); - dTrace("msg:%p, is created, type:%s handle:%p", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle); + memcpy(pMsg, pRpc, sizeof(SRpcMsg)); + dGTrace("msg:%p, is created, type:%s handle:%p", pMsg, TMSG_INFO(pRpc->msgType), pMsg->info.handle); if (InParentProc(pWrapper)) { code = dmPutToProcCQueue(&pWrapper->proc, pMsg, DND_FUNC_REQ); @@ -115,13 +162,11 @@ static void dmProcessRpcMsg(SDnode *pDnode, SRpcMsg *pRpc, SEpSet *pEpSet) { _OVER: if (code != 0) { - dTrace("failed to process msg:%p since %s, handle:%p", pMsg, terrstr(), pRpc->info.handle); - if (terrno != 0) code = terrno; + dGTrace("msg:%p, failed to process since %s", pMsg, terrstr()); if (IsReq(pRpc)) { SRpcMsg rsp = {.code = code, .info = pRpc->info}; - if ((code == TSDB_CODE_NODE_NOT_DEPLOYED || code == TSDB_CODE_APP_NOT_READY) && pRpc->msgType > TDMT_MND_MSG && pRpc->msgType < TDMT_VND_MSG) { dmBuildMnodeRedirectRsp(pDnode, &rsp); @@ -135,7 +180,7 @@ _OVER: } if (pMsg != NULL) { - dTrace("msg:%p, is freed", pMsg); + dGTrace("msg:%p, is freed", pMsg); taosFreeQitem(pMsg); } rpcFreeCont(pRpc->pCont); @@ -149,11 +194,11 @@ int32_t dmInitMsgHandle(SDnode *pDnode) { for (EDndNodeType ntype = DNODE; ntype < NODE_END; ++ntype) { SMgmtWrapper *pWrapper = &pDnode->wrappers[ntype]; - SArray * pArray = (*pWrapper->func.getHandlesFp)(); + SArray *pArray = (*pWrapper->func.getHandlesFp)(); if (pArray == NULL) return -1; for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { - SMgmtHandle * pMgmt = taosArrayGet(pArray, i); + SMgmtHandle *pMgmt = taosArrayGet(pArray, i); SDnodeHandle *pHandle = &pTrans->msgHandles[TMSG_INDEX(pMgmt->msgType)]; if (pMgmt->needCheckVgId) { pHandle->needCheckVgId = pMgmt->needCheckVgId; @@ -184,45 +229,6 @@ static inline int32_t dmSendReq(const SEpSet *pEpSet, SRpcMsg *pMsg) { } } -static inline void dmSendRsp(SRpcMsg *pMsg) { - SMgmtWrapper *pWrapper = pMsg->info.wrapper; - if (InChildProc(pWrapper)) { - dmPutToProcPQueue(&pWrapper->proc, pMsg, DND_FUNC_RSP); - } else { - rpcSendResponse(pMsg); - } -} - -static void dmBuildMnodeRedirectRsp(SDnode *pDnode, SRpcMsg *pMsg) { - SEpSet epSet = {0}; - dmGetMnodeEpSetForRedirect(&pDnode->data, pMsg, &epSet); - - int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); - pMsg->pCont = rpcMallocCont(contLen); - if (pMsg->pCont == NULL) { - pMsg->code = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSEpSet(pMsg->pCont, contLen, &epSet); - pMsg->contLen = contLen; - } -} - -static inline void dmSendRedirectRsp(SRpcMsg *pMsg, const SEpSet *pNewEpSet) { - SRpcMsg rsp = {.code = TSDB_CODE_RPC_REDIRECT, .info = pMsg->info}; - int32_t contLen = tSerializeSEpSet(NULL, 0, pNewEpSet); - - rsp.pCont = rpcMallocCont(contLen); - if (rsp.pCont == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } else { - tSerializeSEpSet(rsp.pCont, contLen, pNewEpSet); - rsp.contLen = contLen; - } - dmSendRsp(&rsp); - rpcFreeCont(pMsg->pCont); - pMsg->pCont = NULL; -} - static inline void dmRegisterBrokenLinkArg(SRpcMsg *pMsg) { SMgmtWrapper *pWrapper = pMsg->info.wrapper; if (InChildProc(pWrapper)) { diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index c3238d83b3902b1c1c1ab7b1685bf49dd7a4c5ee..01958aa05355e489371679c56f74d73bf2f2db0b 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -40,18 +40,28 @@ #include "wal.h" #include "libs/function/function.h" -// clang-format off #ifdef __cplusplus extern "C" { #endif -#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} -#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} -#define dGTrace(param, ...) do { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ",gtid:%s", __VA_ARGS__, buf);} while(0) + +// clang-format off + +#define dFatal(...) { if (dDebugFlag & DEBUG_FATAL) { taosPrintLog("DND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define dError(...) { if (dDebugFlag & DEBUG_ERROR) { taosPrintLog("DND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define dWarn(...) { if (dDebugFlag & DEBUG_WARN) { taosPrintLog("DND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define dInfo(...) { if (dDebugFlag & DEBUG_INFO) { taosPrintLog("DND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define dDebug(...) { if (dDebugFlag & DEBUG_DEBUG) { taosPrintLog("DND ", DEBUG_DEBUG, dDebugFlag, __VA_ARGS__); }} +#define dTrace(...) { if (dDebugFlag & DEBUG_TRACE) { taosPrintLog("DND ", DEBUG_TRACE, dDebugFlag, __VA_ARGS__); }} + +#define dGFatal(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dFatal(param ", gtid:%s", __VA_ARGS__, buf);} +#define dGError(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dError(param ", gtid:%s", __VA_ARGS__, buf);} +#define dGWarn(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dWarn (param ", gtid:%s", __VA_ARGS__, buf);} +#define dGInfo(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dInfo (param ", gtid:%s", __VA_ARGS__, buf);} +#define dGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dDebug(param ", gtid:%s", __VA_ARGS__, buf);} +#define dGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); dTrace(param ", gtid:%s", __VA_ARGS__, buf);} + +// clang-format on typedef enum { DNODE = 0, @@ -185,4 +195,3 @@ void dmSetMnodeEpSet(SDnodeData *pData, SEpSet *pEpSet); #endif #endif /*_TD_DM_INT_H_*/ -// clang-format on diff --git a/source/dnode/mgmt/node_util/src/dmFile.c b/source/dnode/mgmt/node_util/src/dmFile.c index 78e706f90814950287aed067103690f9c215e8e3..2185adc18ba114fd454500151ab2823624c0e856 100644 --- a/source/dnode/mgmt/node_util/src/dmFile.c +++ b/source/dnode/mgmt/node_util/src/dmFile.c @@ -127,14 +127,24 @@ TdFilePtr dmCheckRunning(const char *dataDir) { return NULL; } - int32_t ret = taosLockFile(pFile); - if (ret != 0) { + int32_t retryTimes = 0; + int32_t ret = 0; + do { + ret = taosLockFile(pFile); + if (ret == 0) break; + terrno = TAOS_SYSTEM_ERROR(errno); + taosMsleep(100); + retryTimes++; + dError("failed to lock file:%s since %s, retryTimes:%d", filepath, terrstr(), retryTimes); + } while (retryTimes < 120); + + if (ret < 0) { terrno = TAOS_SYSTEM_ERROR(errno); - dError("failed to lock file:%s since %s", filepath, terrstr()); taosCloseFile(&pFile); return NULL; } + terrno = 0; dDebug("lock file:%s to prevent repeated starts", filepath); return pFile; } diff --git a/source/dnode/mgmt/test/sut/src/sut.cpp b/source/dnode/mgmt/test/sut/src/sut.cpp index 6ef94481ea8edd66317f4d2a7b01bc3234ccd4d9..21d9351cebd8759e80095f18d98743d9bd103366 100644 --- a/source/dnode/mgmt/test/sut/src/sut.cpp +++ b/source/dnode/mgmt/test/sut/src/sut.cpp @@ -102,6 +102,7 @@ int32_t Testbase::SendShowReq(int8_t showType, const char* tb, const char* db) { ASSERT(pRsp->pCont != nullptr); if (pRsp->contLen == 0) return -1; + if (pRsp->code != 0) return -1; showRsp = (SRetrieveMetaTableRsp*)pRsp->pCont; showRsp->handle = htobe64(showRsp->handle); // show Id diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 987b01b96a202ca4e485fe45bf6684e839fc9e37..0605e3a69e8630c6a0c091110252775d8880547c 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -349,6 +349,7 @@ typedef struct { int32_t commentLen; int32_t ast1Len; int32_t ast2Len; + SArray* pFuncs; SSchema* pColumns; SSchema* pTags; char* comment; @@ -420,7 +421,8 @@ typedef struct { int64_t uid; int64_t dbUid; int32_t version; - int8_t subType; // column, db or stable + int8_t subType; // column, db or stable + int8_t withMeta; // TODO SRWLatch lock; int32_t sqlLen; int32_t astLen; @@ -487,6 +489,7 @@ typedef struct { int64_t dbUid; int32_t vgNum; int8_t subType; + int8_t withMeta; int64_t stbUid; SHashObj* consumerHash; // consumerId -> SMqConsumerEp SArray* unassignedVgs; // SArray diff --git a/source/dnode/mnode/impl/inc/mndInfoSchema.h b/source/dnode/mnode/impl/inc/mndInfoSchema.h index 43d934c43101161bc63a487c644a5f6f3b218348..b10d92ee3de1a0e06d801c9a8840751a9f52f37c 100644 --- a/source/dnode/mnode/impl/inc/mndInfoSchema.h +++ b/source/dnode/mnode/impl/inc/mndInfoSchema.h @@ -25,6 +25,7 @@ extern "C" { int32_t mndInitInfos(SMnode *pMnode); void mndCleanupInfos(SMnode *pMnode); int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); +int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index 5d508be2376d7276fdb905ee4cf9ccdaa0101586..58266724a5278d0baafcebd70f9d33423c6cd1cc 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -34,13 +34,21 @@ extern "C" { #endif // clang-format off -#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} -#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} -#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} -#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} -#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} -#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} -#define mGTrace(param, ...) do { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);} while(0) + + +#define mFatal(...) { if (mDebugFlag & DEBUG_FATAL) { taosPrintLog("MND FATAL ", DEBUG_FATAL, 255, __VA_ARGS__); }} +#define mError(...) { if (mDebugFlag & DEBUG_ERROR) { taosPrintLog("MND ERROR ", DEBUG_ERROR, 255, __VA_ARGS__); }} +#define mWarn(...) { if (mDebugFlag & DEBUG_WARN) { taosPrintLog("MND WARN ", DEBUG_WARN, 255, __VA_ARGS__); }} +#define mInfo(...) { if (mDebugFlag & DEBUG_INFO) { taosPrintLog("MND ", DEBUG_INFO, 255, __VA_ARGS__); }} +#define mDebug(...) { if (mDebugFlag & DEBUG_DEBUG) { taosPrintLog("MND ", DEBUG_DEBUG, mDebugFlag, __VA_ARGS__); }} +#define mTrace(...) { if (mDebugFlag & DEBUG_TRACE) { taosPrintLog("MND ", DEBUG_TRACE, mDebugFlag, __VA_ARGS__); }} + +#define mGFatal(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mFatal(param ", gtid:%s", __VA_ARGS__, buf);} +#define mGError(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mError(param ", gtid:%s", __VA_ARGS__, buf);} +#define mGWarn(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mWarn (param ", gtid:%s", __VA_ARGS__, buf);} +#define mGInfo(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mInfo (param ", gtid:%s", __VA_ARGS__, buf);} +#define mGDebug(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mDebug(param ", gtid:%s", __VA_ARGS__, buf);} +#define mGTrace(param, ...) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); mTrace(param ", gtid:%s", __VA_ARGS__, buf);} // clang-format on diff --git a/source/dnode/mnode/impl/inc/mndPerfSchema.h b/source/dnode/mnode/impl/inc/mndPerfSchema.h index 19f60229f926583f321ab350f6ac6043f6c9b096..87b4626e83fb400e45083ea5d3380dee0f543c75 100644 --- a/source/dnode/mnode/impl/inc/mndPerfSchema.h +++ b/source/dnode/mnode/impl/inc/mndPerfSchema.h @@ -23,6 +23,7 @@ extern "C" { #endif int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp); +int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp); int32_t mndInitPerfs(SMnode *pMnode); void mndCleanupPerfs(SMnode *pMnode); diff --git a/source/dnode/mnode/impl/inc/mndQnode.h b/source/dnode/mnode/impl/inc/mndQnode.h index 3e38565a4fe67b93d8ba8b9d30160ce54b13dee5..17615500c47abc2ea0dcad21b5a70ec8a1310892 100644 --- a/source/dnode/mnode/impl/inc/mndQnode.h +++ b/source/dnode/mnode/impl/inc/mndQnode.h @@ -24,12 +24,12 @@ extern "C" { #define QNODE_LOAD_VALUE(pQnode) (pQnode ? (pQnode->load.numOfQueryInQueue + pQnode->load.numOfFetchInQueue) : 0) -int32_t mndInitQnode(SMnode *pMnode); -void mndCleanupQnode(SMnode *pMnode); - +int32_t mndInitQnode(SMnode *pMnode); +void mndCleanupQnode(SMnode *pMnode); SQnodeObj *mndAcquireQnode(SMnode *pMnode, int32_t qnodeId); -void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj); -int32_t mndCreateQnodeList(SMnode *pMnode, SArray** pList, int32_t limit); +void mndReleaseQnode(SMnode *pMnode, SQnodeObj *pObj); +int32_t mndCreateQnodeList(SMnode *pMnode, SArray **pList, int32_t limit); +int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/inc/mndSnode.h b/source/dnode/mnode/impl/inc/mndSnode.h index 180f18a6ddc68a5cd93987714c3d2b42349a529f..ec7bb02ca7cd86ad5f677fa229f7a575ace6086a 100644 --- a/source/dnode/mnode/impl/inc/mndSnode.h +++ b/source/dnode/mnode/impl/inc/mndSnode.h @@ -22,9 +22,12 @@ extern "C" { #endif -int32_t mndInitSnode(SMnode *pMnode); -void mndCleanupSnode(SMnode *pMnode); -SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode); +int32_t mndInitSnode(SMnode *pMnode); +void mndCleanupSnode(SMnode *pMnode); +SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t qnodeId); +void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj); +SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode); +int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj); #ifdef __cplusplus } diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index 20ba71992eeaa9f847d991f0c2dcda5009284e29..dc7f08ebc2f2c8d05b4fbdc900939cda9e8ab02a 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -381,6 +381,7 @@ SMqSubscribeObj *tCloneSubscribeObj(const SMqSubscribeObj *pSub) { pSubNew->dbUid = pSub->dbUid; pSubNew->stbUid = pSub->stbUid; pSubNew->subType = pSub->subType; + pSubNew->withMeta = pSub->withMeta; pSubNew->vgNum = pSub->vgNum; pSubNew->consumerHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -414,6 +415,7 @@ int32_t tEncodeSubscribeObj(void **buf, const SMqSubscribeObj *pSub) { tlen += taosEncodeFixedI64(buf, pSub->dbUid); tlen += taosEncodeFixedI32(buf, pSub->vgNum); tlen += taosEncodeFixedI8(buf, pSub->subType); + tlen += taosEncodeFixedI8(buf, pSub->withMeta); tlen += taosEncodeFixedI64(buf, pSub->stbUid); void *pIter = NULL; @@ -440,6 +442,7 @@ void *tDecodeSubscribeObj(const void *buf, SMqSubscribeObj *pSub) { buf = taosDecodeFixedI64(buf, &pSub->dbUid); buf = taosDecodeFixedI32(buf, &pSub->vgNum); buf = taosDecodeFixedI8(buf, &pSub->subType); + buf = taosDecodeFixedI8(buf, &pSub->withMeta); buf = taosDecodeFixedI64(buf, &pSub->stbUid); int32_t sz; diff --git a/source/dnode/mnode/impl/src/mndDnode.c b/source/dnode/mnode/impl/src/mndDnode.c index 58c3570c3631ec0ceee76163d4afd50e047245ad..d3723326b99717ffa17f874a71cae76cee02f4aa 100644 --- a/source/dnode/mnode/impl/src/mndDnode.c +++ b/source/dnode/mnode/impl/src/mndDnode.c @@ -19,6 +19,7 @@ #include "mndMnode.h" #include "mndQnode.h" #include "mndShow.h" +#include "mndSnode.h" #include "mndTrans.h" #include "mndUser.h" #include "mndVgroup.h" @@ -46,6 +47,7 @@ static SSdbRow *mndDnodeActionDecode(SSdbRaw *pRaw); static int32_t mndDnodeActionInsert(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionDelete(SSdb *pSdb, SDnodeObj *pDnode); static int32_t mndDnodeActionUpdate(SSdb *pSdb, SDnodeObj *pOld, SDnodeObj *pNew); +static int32_t mndProcessDnodeListReq(SRpcMsg *pReq); static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq); static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq); @@ -75,6 +77,7 @@ int32_t mndInitDnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CONFIG_DNODE, mndProcessConfigDnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CONFIG_DNODE_RSP, mndProcessConfigDnodeRsp); mndSetMsgHandle(pMnode, TDMT_MND_STATUS, mndProcessStatusReq); + mndSetMsgHandle(pMnode, TDMT_MND_DNODE_LIST, mndProcessDnodeListReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndRetrieveConfigs); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_CONFIGS, mndCancelGetNextConfig); @@ -370,7 +373,8 @@ static int32_t mndProcessStatusReq(SRpcMsg *pReq) { SMnodeObj *pObj = mndAcquireMnode(pMnode, pDnode->id); if (pObj != NULL) { if (pObj->state != statusReq.mload.syncState) { - mInfo("dnode:%d, mnode syncstate from %s to %s", pObj->id, syncStr(pObj->state), syncStr(statusReq.mload.syncState)); + mInfo("dnode:%d, mnode syncstate from %s to %s", pObj->id, syncStr(pObj->state), + syncStr(statusReq.mload.syncState)); pObj->state = statusReq.mload.syncState; pObj->stateStartTime = taosGetTimestampMs(); } @@ -497,6 +501,59 @@ _OVER: return code; } +static int32_t mndProcessDnodeListReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SDnodeObj *pObj = NULL; + void *pIter = NULL; + SDnodeListRsp rsp = {0}; + int32_t code = -1; + + rsp.dnodeList = taosArrayInit(5, sizeof(SEpSet)); + if (NULL == rsp.dnodeList) { + mError("failed to alloc epSet while process dnode list req"); + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + while (1) { + pIter = sdbFetch(pSdb, SDB_DNODE, pIter, (void **)&pObj); + if (pIter == NULL) break; + + SEpSet epSet = {0}; + epSet.numOfEps = 1; + tstrncpy(epSet.eps[0].fqdn, pObj->fqdn, TSDB_FQDN_LEN); + epSet.eps[0].port = pObj->port; + + (void)taosArrayPush(rsp.dnodeList, &epSet); + + sdbRelease(pSdb, pObj); + } + + int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &rsp); + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSDnodeListRsp(pRsp, rspLen, &rsp); + + pReq->info.rspLen = rspLen; + pReq->info.rsp = pRsp; + code = 0; + +_OVER: + + if (code != 0) { + mError("failed to get dnode list since %s", terrstr()); + } + + tFreeSDnodeListRsp(&rsp); + + return code; +} + static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -508,7 +565,7 @@ static int32_t mndProcessCreateDnodeReq(SRpcMsg *pReq) { goto _OVER; } - mDebug("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); + mInfo("dnode:%s:%d, start to create", createReq.fqdn, createReq.port); if (createReq.fqdn[0] == 0 || createReq.port <= 0 || createReq.port > UINT16_MAX) { terrno = TSDB_CODE_MND_INVALID_DNODE_EP; @@ -538,7 +595,8 @@ _OVER: return code; } -static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj, int32_t numOfVnodes) { +static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMnodeObj *pMObj, SQnodeObj *pQObj, + SSnodeObj *pSObj, int32_t numOfVnodes) { int32_t code = -1; SSdbRaw *pRaw = NULL; STrans *pTrans = NULL; @@ -546,7 +604,7 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); - mDebug("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); + mInfo("trans:%d, used to drop dnode:%d", pTrans->id, pDnode->id); pRaw = mndDnodeActionEncode(pDnode); if (pRaw == NULL || mndTransAppendRedolog(pTrans, pRaw) != 0) goto _OVER; @@ -559,13 +617,25 @@ static int32_t mndDropDnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SM pRaw = NULL; if (pMObj != NULL) { - mDebug("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); + mInfo("trans:%d, mnode on dnode:%d will be dropped", pTrans->id, pDnode->id); if (mndSetDropMnodeInfoToTrans(pMnode, pTrans, pMObj) != 0) goto _OVER; } + + if (pQObj != NULL) { + mInfo("trans:%d, qnode on dnode:%d will be dropped", pTrans->id, pDnode->id); + if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pQObj) != 0) goto _OVER; + } + + if (pSObj != NULL) { + mInfo("trans:%d, snode on dnode:%d will be dropped", pTrans->id, pDnode->id); + if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pSObj) != 0) goto _OVER; + } + if (numOfVnodes > 0) { - mDebug("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); + mInfo("trans:%d, %d vnodes on dnode:%d will be dropped", pTrans->id, numOfVnodes, pDnode->id); if (mndSetMoveVgroupsInfoToTrans(pMnode, pTrans, pDnode->id) != 0) goto _OVER; } + if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -577,30 +647,34 @@ _OVER: } static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - int32_t code = -1; - SDnodeObj *pDnode = NULL; - SMnodeObj *pMObj = NULL; - SMDropMnodeReq dropReq = {0}; - - if (tDeserializeSCreateDropMQSBNodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + SDnodeObj *pDnode = NULL; + SMnodeObj *pMObj = NULL; + SQnodeObj *pQObj = NULL; + SSnodeObj *pSObj = NULL; + SDropDnodeReq dropReq = {0}; + + if (tDeserializeSDropDnodeReq(pReq->pCont, pReq->contLen, &dropReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; goto _OVER; } - mDebug("dnode:%d, start to drop", dropReq.dnodeId); - - if (dropReq.dnodeId <= 0) { - terrno = TSDB_CODE_MND_INVALID_DNODE_ID; - goto _OVER; - } + mInfo("dnode:%d, start to drop, ep:%s:%d", dropReq.dnodeId, dropReq.fqdn, dropReq.port); pDnode = mndAcquireDnode(pMnode, dropReq.dnodeId); if (pDnode == NULL) { - terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; - goto _OVER; + char ep[TSDB_EP_LEN + 1] = {0}; + snprintf(ep, sizeof(ep), dropReq.fqdn, dropReq.port); + pDnode = mndAcquireDnodeByEp(pMnode, ep); + if (pDnode == NULL) { + terrno = TSDB_CODE_MND_DNODE_NOT_EXIST; + goto _OVER; + } } + pQObj = mndAcquireQnode(pMnode, dropReq.dnodeId); + pSObj = mndAcquireSnode(pMnode, dropReq.dnodeId); pMObj = mndAcquireMnode(pMnode, dropReq.dnodeId); if (pMObj != NULL) { if (sdbGetSize(pMnode->pSdb, SDB_MNODE) <= 1) { @@ -627,7 +701,7 @@ static int32_t mndProcessDropDnodeReq(SRpcMsg *pReq) { goto _OVER; } - code = mndDropDnode(pMnode, pReq, pDnode, pMObj, numOfVnodes); + code = mndDropDnode(pMnode, pReq, pDnode, pMObj, pQObj, pSObj, numOfVnodes); if (code == 0) code = TSDB_CODE_ACTION_IN_PROGRESS; _OVER: @@ -637,6 +711,8 @@ _OVER: mndReleaseDnode(pMnode, pDnode); mndReleaseMnode(pMnode, pMObj); + mndReleaseQnode(pMnode, pQObj); + mndReleaseSnode(pMnode, pSObj); return code; } @@ -649,23 +725,46 @@ static int32_t mndProcessConfigDnodeReq(SRpcMsg *pReq) { return -1; } + mInfo("dnode:%d, start to config, option:%s, value:%s", cfgReq.dnodeId, cfgReq.config, cfgReq.value); SDnodeObj *pDnode = mndAcquireDnode(pMnode, cfgReq.dnodeId); if (pDnode == NULL) { mError("dnode:%d, failed to config since %s ", cfgReq.dnodeId, terrstr()); return -1; } - SEpSet epSet = mndGetDnodeEpset(pDnode); mndReleaseDnode(pMnode, pDnode); - int32_t bufLen = tSerializeSMCfgDnodeReq(NULL, 0, &cfgReq); + SDCfgDnodeReq dcfgReq = {0}; + if (strncasecmp(cfgReq.config, "debugFlag", 9) == 0) { + const char *value = cfgReq.value; + int32_t flag = atoi(value); + if (flag <= 0) { + flag = atoi(cfgReq.config + 10); + } + if (flag <= 0 || flag > 255) { + mError("dnode:%d, failed to config debugFlag since value:%d", cfgReq.dnodeId, flag); + terrno = TSDB_CODE_INVALID_CFG; + return -1; + } + + strcpy(dcfgReq.config, "debugFlag"); + snprintf(dcfgReq.value, TSDB_DNODE_VALUE_LEN, "%d", flag); + } else if (strcasecmp(cfgReq.config, "resetlog") == 0) { + strcpy(dcfgReq.config, "resetlog"); + } else { + terrno = TSDB_CODE_INVALID_CFG; + mError("dnode:%d, failed to config since %s", cfgReq.dnodeId, terrstr()); + return -1; + } + + int32_t bufLen = tSerializeSDCfgDnodeReq(NULL, 0, &dcfgReq); void *pBuf = rpcMallocCont(bufLen); if (pBuf == NULL) return -1; - tSerializeSMCfgDnodeReq(pBuf, bufLen, &cfgReq); + tSerializeSDCfgDnodeReq(pBuf, bufLen, &dcfgReq); mDebug("dnode:%d, send config req to dnode, app:%p", cfgReq.dnodeId, pReq->info.ahandle); - SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen, .info = pReq->info}; + SRpcMsg rpcMsg = {.msgType = TDMT_DND_CONFIG_DNODE, .pCont = pBuf, .contLen = bufLen}; return tmsgSendReq(&epSet, &rpcMsg); } @@ -679,28 +778,28 @@ static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p int32_t totalRows = 0; int32_t numOfRows = 0; char *cfgOpts[TSDB_CONFIG_NUMBER] = {0}; - char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONIIG_VALUE_LEN + 1] = {0}; + char cfgVals[TSDB_CONFIG_NUMBER][TSDB_CONFIG_VALUE_LEN + 1] = {0}; char *pWrite = NULL; int32_t cols = 0; cfgOpts[totalRows] = "statusInterval"; - snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%d", tsStatusInterval); + snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%d", tsStatusInterval); totalRows++; cfgOpts[totalRows] = "timezone"; - snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsTimezoneStr); + snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%s", tsTimezoneStr); totalRows++; cfgOpts[totalRows] = "locale"; - snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsLocale); + snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%s", tsLocale); totalRows++; cfgOpts[totalRows] = "charset"; - snprintf(cfgVals[totalRows], TSDB_CONIIG_VALUE_LEN, "%s", tsCharset); + snprintf(cfgVals[totalRows], TSDB_CONFIG_VALUE_LEN, "%s", tsCharset); totalRows++; char buf[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; - char bufVal[TSDB_CONIIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + char bufVal[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; for (int32_t i = 0; i < totalRows; i++) { cols = 0; @@ -709,7 +808,7 @@ static int32_t mndRetrieveConfigs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)buf, false); - STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONIIG_VALUE_LEN); + STR_WITH_MAXSIZE_TO_VARSTR(bufVal, cfgVals[i], TSDB_CONFIG_VALUE_LEN); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataAppend(pColInfo, numOfRows, (const char *)bufVal, false); diff --git a/source/dnode/mnode/impl/src/mndInfoSchema.c b/source/dnode/mnode/impl/src/mndInfoSchema.c index c9bccaf1d02117dc43a7c631605543243bd6bb3b..45fefaf87008ac8bc27935fc01bef0059265c8be 100644 --- a/source/dnode/mnode/impl/src/mndInfoSchema.c +++ b/source/dnode/mnode/impl/src/mndInfoSchema.c @@ -90,6 +90,38 @@ int32_t mndBuildInsTableSchema(SMnode *pMnode, const char *dbFName, const char * return 0; } +int32_t mndBuildInsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { + if (NULL == pMnode->infosMeta) { + terrno = TSDB_CODE_MND_NOT_READY; + return -1; + } + + STableMetaRsp *pMeta = taosHashGet(pMnode->infosMeta, tbName, strlen(tbName)); + if (NULL == pMeta) { + mError("invalid information schema table name:%s", tbName); + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; + return -1; + } + + strcpy(pRsp->tbName, pMeta->tbName); + strcpy(pRsp->stbName, pMeta->stbName); + strcpy(pRsp->dbFName, pMeta->dbFName); + pRsp->numOfTags = pMeta->numOfTags; + pRsp->numOfColumns = pMeta->numOfColumns; + pRsp->tableType = pMeta->tableType; + + pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pRsp->pSchemas = NULL; + return -1; + } + + memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); + return 0; +} + + int32_t mndInitInfos(SMnode *pMnode) { pMnode->infosMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_VARCHAR), false, HASH_NO_LOCK); if (pMnode->infosMeta == NULL) { diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index b50781e8a585da2be402c6994371978cd00392dc..dede1c45e6700ba9b78875b8a93cc8b143443c74 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -59,55 +59,34 @@ static void *mndBuildTimerMsg(int32_t *pContLen) { static void mndPullupTrans(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; + if (pReq != NULL) { + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TRANS_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); + } +} + +static void mndTtlTimer(SMnode *pMnode) { + int32_t contLen = 0; + void *pReq = mndBuildTimerMsg(&contLen); + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TTL_TIMER, .pCont = pReq, .contLen = contLen}; tmsgPutToQueue(&pMnode->msgCb, WRITE_QUEUE, &rpcMsg); } static void mndCalMqRebalance(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + if (pReq != NULL) { + SRpcMsg rpcMsg = {.msgType = TDMT_MND_MQ_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); + } } static void mndPullupTelem(SMnode *pMnode) { int32_t contLen = 0; void *pReq = mndBuildTimerMsg(&contLen); - SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; - tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); -} - -static void mndPushTtlTime(SMnode *pMnode) { - SSdb *pSdb = pMnode->pSdb; - SVgObj *pVgroup = NULL; - void *pIter = NULL; - - while (1) { - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t); - SMsgHead *pHead = rpcMallocCont(contLen); - if (pHead == NULL) { - mError("ttl time malloc err. contLen:%d", contLen); - sdbRelease(pSdb, pVgroup); - continue; - } - pHead->contLen = htonl(contLen); - pHead->vgId = htonl(pVgroup->vgId); - - int32_t t = taosGetTimestampSec(); - *(int32_t *)(POINTER_SHIFT(pHead, sizeof(SMsgHead))) = htonl(t); - - SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen}; - - SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); - int32_t code = tmsgSendReq(&epSet, &rpcMsg); - if (code != 0) { - mError("ttl time seed err. code:%d", code); - } - mError("ttl time seed succ. time:%d", t); - sdbRelease(pSdb, pVgroup); + if (pReq != NULL) { + SRpcMsg rpcMsg = {.msgType = TDMT_MND_TELEM_TIMER, .pCont = pReq, .contLen = contLen}; + tmsgPutToQueue(&pMnode->msgCb, READ_QUEUE, &rpcMsg); } } @@ -117,14 +96,14 @@ static void *mndThreadFp(void *param) { setThreadName("mnode-timer"); while (1) { - if (lastTime % (864000) == 0) { // sleep 1 day for ttl - mndPushTtlTime(pMnode); - } - lastTime++; taosMsleep(100); if (mndGetStop(pMnode)) break; + if (lastTime % (tsTransPullupInterval * 10) == 1) { + mndTtlTimer(pMnode); + } + if (lastTime % (tsTransPullupInterval * 10) == 0) { mndPullupTrans(pMnode); } @@ -549,25 +528,25 @@ int32_t mndProcessSyncMsg(SRpcMsg *pMsg) { static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (mndAcquireRpcRef(pMsg->info.node) == 0) return 0; + if (pMsg->msgType == TDMT_MND_MQ_TIMER || pMsg->msgType == TDMT_MND_TELEM_TIMER || + pMsg->msgType == TDMT_MND_TRANS_TIMER || TDMT_MND_TTL_TIMER) { + return -1; + } - if (IsReq(pMsg) && pMsg->msgType != TDMT_MND_MQ_TIMER && pMsg->msgType != TDMT_MND_TELEM_TIMER && - pMsg->msgType != TDMT_MND_TRANS_TIMER) { - mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); - - mndAbortPreprocessMsg(pMsg); + const STraceId *trace = &pMsg->info.traceId; + mError("msg:%p, failed to check mnode state since %s, type:%s", pMsg, terrstr(), TMSG_INFO(pMsg->msgType)); - SEpSet epSet = {0}; - mndGetMnodeEpSet(pMsg->info.node, &epSet); + SEpSet epSet = {0}; + mndGetMnodeEpSet(pMsg->info.node, &epSet); - int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); - pMsg->info.rsp = rpcMallocCont(contLen); - if (pMsg->info.rsp != NULL) { - tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); - pMsg->info.rspLen = contLen; - terrno = TSDB_CODE_RPC_REDIRECT; - } else { - terrno = TSDB_CODE_OUT_OF_MEMORY; - } + int32_t contLen = tSerializeSEpSet(NULL, 0, &epSet); + pMsg->info.rsp = rpcMallocCont(contLen); + if (pMsg->info.rsp != NULL) { + tSerializeSEpSet(pMsg->info.rsp, contLen, &epSet); + pMsg->info.rspLen = contLen; + terrno = TSDB_CODE_RPC_REDIRECT; + } else { + terrno = TSDB_CODE_OUT_OF_MEMORY; } return -1; @@ -576,18 +555,21 @@ static int32_t mndCheckMnodeState(SRpcMsg *pMsg) { static int32_t mndCheckMsgContent(SRpcMsg *pMsg) { if (!IsReq(pMsg)) return 0; if (pMsg->contLen != 0 && pMsg->pCont != NULL) return 0; - - mError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen, + + const STraceId *trace = &pMsg->info.traceId; + mGError("msg:%p, failed to check msg, cont:%p contLen:%d, app:%p type:%s", pMsg, pMsg->pCont, pMsg->contLen, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_INVALID_MSG_LEN; return -1; } int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { - SMnode *pMnode = pMsg->info.node; + SMnode *pMnode = pMsg->info.node; + const STraceId *trace = &pMsg->info.traceId; + MndMsgFp fp = pMnode->msgFp[TMSG_INDEX(pMsg->msgType)]; if (fp == NULL) { - mError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); + mGError("msg:%p, failed to get msg handle, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); terrno = TSDB_CODE_MSG_NOT_PROCESSED; return -1; } @@ -595,18 +577,17 @@ int32_t mndProcessRpcMsg(SRpcMsg *pMsg) { if (mndCheckMsgContent(pMsg) != 0) return -1; if (mndCheckMnodeState(pMsg) != 0) return -1; - STraceId *trace = &pMsg->info.traceId; mGTrace("msg:%p, start to process in mnode, app:%p type:%s", pMsg, pMsg->info.ahandle, TMSG_INFO(pMsg->msgType)); int32_t code = (*fp)(pMsg); mndReleaseRpcRef(pMnode); if (code == TSDB_CODE_ACTION_IN_PROGRESS) { - mTrace("msg:%p, won't response immediately since in progress", pMsg); + mGTrace("msg:%p, won't response immediately since in progress", pMsg); } else if (code == 0) { - mTrace("msg:%p, successfully processed", pMsg); + mGTrace("msg:%p, successfully processed", pMsg); } else { - mError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, - TMSG_INFO(pMsg->msgType)); + mGError("msg:%p, failed to process since %s, app:%p type:%s", pMsg, terrstr(), pMsg->info.ahandle, + TMSG_INFO(pMsg->msgType)); } return code; @@ -622,7 +603,6 @@ void mndSetMsgHandle(SMnode *pMnode, tmsg_t msgType, MndMsgFp fp) { // Note: uid 0 is reserved int64_t mndGenerateUid(char *name, int32_t len) { int32_t hashval = MurmurHash3_32(name, len); - do { int64_t us = taosGetTimestampUs(); int64_t x = (us & 0x000000FFFFFFFFFF) << 24; diff --git a/source/dnode/mnode/impl/src/mndPerfSchema.c b/source/dnode/mnode/impl/src/mndPerfSchema.c index d7b2e3ec24a89c0a4576753f1e6343cffff9ce02..7056337c8d6d84997f622eac898ffe8e4980898f 100644 --- a/source/dnode/mnode/impl/src/mndPerfSchema.c +++ b/source/dnode/mnode/impl/src/mndPerfSchema.c @@ -92,6 +92,37 @@ int32_t mndBuildPerfsTableSchema(SMnode *pMnode, const char *dbFName, const char return 0; } +int32_t mndBuildPerfsTableCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { + if (NULL == pMnode->perfsMeta) { + terrno = TSDB_CODE_MND_NOT_READY; + return -1; + } + + STableMetaRsp *pMeta = taosHashGet(pMnode->perfsMeta, tbName, strlen(tbName)); + if (NULL == pMeta) { + mError("invalid performance schema table name:%s", tbName); + terrno = TSDB_CODE_MND_INVALID_SYS_TABLENAME; + return -1; + } + + strcpy(pRsp->tbName, pMeta->tbName); + strcpy(pRsp->stbName, pMeta->stbName); + strcpy(pRsp->dbFName, pMeta->dbFName); + pRsp->numOfTags = pMeta->numOfTags; + pRsp->numOfColumns = pMeta->numOfColumns; + pRsp->tableType = pMeta->tableType; + + pRsp->pSchemas = taosMemoryCalloc(pMeta->numOfColumns, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + pRsp->pSchemas = NULL; + return -1; + } + + memcpy(pRsp->pSchemas, pMeta->pSchemas, pMeta->numOfColumns * sizeof(SSchema)); + return 0; +} + int32_t mndInitPerfs(SMnode *pMnode) { pMnode->perfsMeta = taosHashInit(20, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); if (pMnode->perfsMeta == NULL) { diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index fd80679316ad4df7dbcac44e636bb29f42c1800d..832e328d962479f739004ae765171ddfeb71aa2d 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -122,30 +122,33 @@ static SConnObj *mndCreateConn(SMnode *pMnode, const char *user, int8_t connType int32_t pid, const char *app, int64_t startTime) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; - char connStr[255] = {0}; - int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); + char connStr[255] = {0}; + int32_t len = snprintf(connStr, sizeof(connStr), "%s%d%d%d%s", user, ip, port, pid, app); uint32_t connId = mndGenerateUid(connStr, len); if (startTime == 0) startTime = taosGetTimestampMs(); - SConnObj connObj = {.id = connId, - .connType = connType, - .appStartTimeMs = startTime, - .pid = pid, - .ip = ip, - .port = port, - .killed = 0, - .loginTimeMs = taosGetTimestampMs(), - .lastAccessTimeMs = 0, - .killId = 0, - .numOfQueries = 0, - .pQueries = NULL}; + SConnObj connObj = { + .id = connId, + .connType = connType, + .appStartTimeMs = startTime, + .pid = pid, + .ip = ip, + .port = port, + .killed = 0, + .loginTimeMs = taosGetTimestampMs(), + .lastAccessTimeMs = 0, + .killId = 0, + .numOfQueries = 0, + .pQueries = NULL, + }; connObj.lastAccessTimeMs = connObj.loginTimeMs; tstrncpy(connObj.user, user, TSDB_USER_LEN); tstrncpy(connObj.app, app, TSDB_APP_NAME_LEN); int32_t keepTime = tsShellActivityTimer * 3; - SConnObj *pConn = taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), keepTime * 1000); + SConnObj *pConn = + taosCachePut(pMgmt->connCache, &connId, sizeof(uint32_t), &connObj, sizeof(connObj), keepTime * 1000); if (pConn == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("conn:%d, failed to put into cache since %s, user:%s", connId, user, terrstr()); @@ -174,7 +177,6 @@ static SConnObj *mndAcquireConn(SMnode *pMnode, uint32_t connId) { } pConn->lastAccessTimeMs = taosGetTimestampMs(); - mTrace("conn:%u, acquired from cache, data:%p", pConn->id, pConn); return pConn; } @@ -207,13 +209,14 @@ static void mndCancelGetNextConn(SMnode *pMnode, void *pIter) { } static int32_t mndProcessConnectReq(SRpcMsg *pReq) { - SMnode *pMnode = pReq->info.node; - SUserObj *pUser = NULL; - SDbObj *pDb = NULL; - SConnObj *pConn = NULL; - int32_t code = -1; - SConnectReq connReq = {0}; - char ip[30] = {0}; + SMnode *pMnode = pReq->info.node; + SUserObj *pUser = NULL; + SDbObj *pDb = NULL; + SConnObj *pConn = NULL; + int32_t code = -1; + SConnectReq connReq = {0}; + char ip[30] = {0}; + const STraceId *trace = &pReq->info.traceId; if (tDeserializeSConnectReq(pReq->pCont, pReq->contLen, &connReq) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -224,11 +227,11 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pUser = mndAcquireUser(pMnode, pReq->info.conn.user); if (pUser == NULL) { - mError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr()); + mGError("user:%s, failed to login while acquire user since %s", pReq->info.conn.user, terrstr()); goto CONN_OVER; } if (0 != strncmp(connReq.passwd, pUser->pass, TSDB_PASSWORD_LEN - 1)) { - mError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd); + mGError("user:%s, failed to auth while acquire user, input:%s", pReq->info.conn.user, connReq.passwd); code = TSDB_CODE_RPC_AUTH_FAILURE; goto CONN_OVER; } @@ -239,8 +242,8 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pDb = mndAcquireDb(pMnode, db); if (pDb == NULL) { terrno = TSDB_CODE_MND_INVALID_DB; - mError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db, - terrstr()); + mGError("user:%s, failed to login from %s while use db:%s since %s", pReq->info.conn.user, ip, connReq.db, + terrstr()); goto CONN_OVER; } } @@ -248,7 +251,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pConn = mndCreateConn(pMnode, pReq->info.conn.user, connReq.connType, pReq->info.conn.clientIp, pReq->info.conn.clientPort, connReq.pid, connReq.app, connReq.startTime); if (pConn == NULL) { - mError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr()); + mGError("user:%s, failed to login from %s while create connection since %s", pReq->info.conn.user, ip, terrstr()); goto CONN_OVER; } @@ -273,7 +276,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { pReq->info.rspLen = contLen; pReq->info.rsp = pRsp; - mDebug("user:%s, login from %s:%d, conn:%u, app:%s", pReq->info.conn.user, ip, pConn->port, pConn->id, connReq.app); + mGDebug("user:%s, login from %s:%d, conn:%u, app:%s", pReq->info.conn.user, ip, pConn->port, pConn->id, connReq.app); code = 0; @@ -302,7 +305,7 @@ static int32_t mndSaveQueryList(SConnObj *pConn, SQueryHbReqBasic *pBasic) { return TSDB_CODE_SUCCESS; } -static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq* pReq) { +static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq *pReq) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; SAppObj app; @@ -314,22 +317,19 @@ static SAppObj *mndCreateApp(SMnode *pMnode, uint32_t clientIp, SAppHbReq* pReq) memcpy(&app.summary, &pReq->summary, sizeof(pReq->summary)); app.lastAccessTimeMs = taosGetTimestampMs(); - int32_t keepTime = tsShellActivityTimer * 3; + const int32_t keepTime = tsShellActivityTimer * 3; SAppObj *pApp = taosCachePut(pMgmt->appCache, &pReq->appId, sizeof(pReq->appId), &app, sizeof(app), keepTime * 1000); if (pApp == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; mError("failed to app %" PRIx64 " into cache since %s", pReq->appId, terrstr()); return NULL; } - + mTrace("app %" PRIx64 " is put into cache", pReq->appId); return pApp; } -static void mndFreeApp(SAppObj *pApp) { - mTrace("app %" PRIx64 " is destroyed", pApp->appId); -} - +static void mndFreeApp(SAppObj *pApp) { mTrace("app %" PRIx64 " is destroyed", pApp->appId); } static SAppObj *mndAcquireApp(SMnode *pMnode, int64_t appId) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; @@ -356,7 +356,7 @@ static void mndReleaseApp(SMnode *pMnode, SAppObj *pApp) { void *mndGetNextApp(SMnode *pMnode, SCacheIter *pIter) { SAppObj *pApp = NULL; - bool hasNext = taosCacheIterNext(pIter); + bool hasNext = taosCacheIterNext(pIter); if (hasNext) { size_t dataLen = 0; pApp = taosCacheIterGetData(pIter, &dataLen); @@ -439,16 +439,16 @@ static SClientHbRsp *mndMqHbBuildRsp(SMnode *pMnode, SClientHbReq *pReq) { } static int32_t mndUpdateAppInfo(SMnode *pMnode, SClientHbReq *pHbReq, SRpcConnInfo *connInfo) { - SAppHbReq* pReq = &pHbReq->app; - SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId); + SAppHbReq *pReq = &pHbReq->app; + SAppObj *pApp = mndAcquireApp(pMnode, pReq->appId); if (pApp == NULL) { pApp = mndCreateApp(pMnode, connInfo->clientIp, pReq); if (pApp == NULL) { mError("failed to create new app %" PRIx64 " since %s", pReq->appId, terrstr()); return -1; } else { - mDebug("a new app %" PRIx64 "created", pReq->appId); - mndReleaseApp(pMnode, pApp); + mDebug("a new app %" PRIx64 " is created", pReq->appId); + mndReleaseApp(pMnode, pApp); return TSDB_CODE_SUCCESS; } } @@ -464,7 +464,7 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb SClientHbBatchRsp *pBatchRsp) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; SClientHbRsp hbRsp = {.connKey = pHbReq->connKey, .status = 0, .info = NULL, .query = NULL}; - SRpcConnInfo connInfo = pMsg->info.conn; + SRpcConnInfo connInfo = pMsg->info.conn; mndUpdateAppInfo(pMnode, pHbReq, &connInfo); @@ -637,9 +637,9 @@ static int32_t mndProcessKillQueryReq(SRpcMsg *pReq) { } mInfo("kill query msg is received, queryId:%s", killReq.queryStrId); - int32_t connId = 0; + int32_t connId = 0; uint64_t queryId = 0; - char* p = strchr(killReq.queryStrId, ':'); + char *p = strchr(killReq.queryStrId, ':'); if (NULL == p) { mError("invalid query id %s", killReq.queryStrId); terrno = TSDB_CODE_MND_INVALID_QUERY_ID; @@ -853,12 +853,12 @@ static int32_t mndRetrieveQueries(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *p } static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - int32_t numOfRows = 0; - int32_t cols = 0; - SAppObj *pApp = NULL; - + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + int32_t numOfRows = 0; + int32_t cols = 0; + SAppObj *pApp = NULL; + if (pShow->pIter == NULL) { SProfileMgmt *pMgmt = &pMnode->profileMgmt; pShow->pIter = taosCacheCreateIter(pMgmt->appCache); @@ -931,7 +931,6 @@ static int32_t mndRetrieveApps(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlo return numOfRows; } - static void mndCancelGetNextQuery(SMnode *pMnode, void *pIter) { if (pIter != NULL) { taosCacheDestroyIter(pIter); diff --git a/source/dnode/mnode/impl/src/mndQnode.c b/source/dnode/mnode/impl/src/mndQnode.c index f5625f32d5a73af783dc0172ba949cc6d4532462..0a6c97e63c8e9af36bd6d91b4d05445f77a82db3 100644 --- a/source/dnode/mnode/impl/src/mndQnode.c +++ b/source/dnode/mnode/impl/src/mndQnode.c @@ -354,6 +354,14 @@ static int32_t mndSetDropQnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SQn return 0; } +int32_t mndSetDropQnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SQnodeObj *pObj) { + if (pObj == NULL) return 0; + if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) return -1; + if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) return -1; + if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; + return 0; +} + static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { int32_t code = -1; @@ -361,9 +369,7 @@ static int32_t mndDropQnode(SMnode *pMnode, SRpcMsg *pReq, SQnodeObj *pObj) { if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop qnode:%d", pTrans->id, pObj->id); - if (mndSetDropQnodeRedoLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropQnodeCommitLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropQnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER; + if (mndSetDropQnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; diff --git a/source/dnode/mnode/impl/src/mndQuery.c b/source/dnode/mnode/impl/src/mndQuery.c index f32a3129dece8e55d37d68d61e39787e31fb8bdb..671152f9c6c4290e472e651aaff4e338798e19f3 100644 --- a/source/dnode/mnode/impl/src/mndQuery.c +++ b/source/dnode/mnode/impl/src/mndQuery.c @@ -18,16 +18,14 @@ #include "mndMnode.h" #include "qworker.h" -int32_t mndPreProcessMsg(SRpcMsg *pMsg) { +int32_t mndPreProcessQueryMsg(SRpcMsg *pMsg) { if (TDMT_VND_QUERY != pMsg->msgType) return 0; - SMnode *pMnode = pMsg->info.node; return qWorkerPreprocessQueryMsg(pMnode->pQuery, pMsg); } -void mndAbortPreprocessMsg(SRpcMsg *pMsg) { +void mndPostProcessQueryMsg(SRpcMsg *pMsg) { if (TDMT_VND_QUERY != pMsg->msgType) return; - SMnode *pMnode = pMsg->info.node; qWorkerAbortPreprocessQueryMsg(pMnode->pQuery, pMsg); } diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index 2b6258b10ab76772e69c7a6935c058a526e202f7..9d8ed3701ee9a7001c20540b3e6c97f87db40a29 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -37,7 +37,7 @@ extern bool tsSchedStreamToSnode; static int32_t mndAddTaskToTaskSet(SArray* pArray, SStreamTask* pTask) { int32_t childId = taosArrayGetSize(pArray); - pTask->childId = childId; + pTask->selfChildId = childId; taosArrayPush(pArray, &pTask); return 0; } @@ -271,7 +271,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, STrans* pTrans, SStreamOb pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); // source - pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + pTask->isDataScan = 0; // exec pTask->execType = TASK_EXEC__NONE; @@ -306,6 +306,8 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* } mndAddTaskToTaskSet(tasks, pTask); + ASSERT(pStream->fixedSinkVg.vgId == pStream->fixedSinkVgId); + pTask->nodeId = pStream->fixedSinkVgId; #if 0 SVgObj* pVgroup = mndAcquireVgroup(pMnode, pStream->fixedSinkVgId); @@ -315,8 +317,9 @@ int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pTask->epSet = mndGetVgroupEpset(pMnode, pVgroup); #endif pTask->epSet = mndGetVgroupEpset(pMnode, &pStream->fixedSinkVg); + // source - pTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + pTask->isDataScan = 0; // exec pTask->execType = TASK_EXEC__NONE; @@ -384,8 +387,11 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { pInnerTask = tNewSStreamTask(pStream->uid); mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); - // input - pInnerTask->inputType = TASK_INPUT_TYPE__DATA_BLOCK; + + pInnerTask->childEpInfo = taosArrayInit(0, sizeof(void*)); + + // source + pInnerTask->isDataScan = 0; // trigger pInnerTask->triggerParam = pStream->triggerParam; @@ -446,10 +452,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SStreamTask* pTask = tNewSStreamTask(pStream->uid); mndAddTaskToTaskSet(taskSourceLevel, pTask); - pTask->dataScan = 1; - - // input - pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; + // source + pTask->isDataScan = 1; // add fixed vg dispatch pTask->sinkType = TASK_SINK__NONE; @@ -467,6 +471,20 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { qDestroyQueryPlan(pPlan); return -1; } + + SStreamChildEpInfo* pEpInfo = taosMemoryMalloc(sizeof(SStreamChildEpInfo)); + if (pEpInfo == NULL) { + ASSERT(0); + terrno = TSDB_CODE_OUT_OF_MEMORY; + sdbRelease(pSdb, pVgroup); + qDestroyQueryPlan(pPlan); + return -1; + } + pEpInfo->childId = pTask->selfChildId; + pEpInfo->epSet = pTask->epSet; + pEpInfo->nodeId = pTask->nodeId; + pEpInfo->taskId = pTask->taskId; + taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); } } @@ -491,10 +509,8 @@ int32_t mndScheduleStream(SMnode* pMnode, STrans* pTrans, SStreamObj* pStream) { SStreamTask* pTask = tNewSStreamTask(pStream->uid); mndAddTaskToTaskSet(taskOneLevel, pTask); - pTask->dataScan = 1; - - // input - pTask->inputType = TASK_INPUT_TYPE__SUMBIT_BLOCK; + // source + pTask->isDataScan = 1; // trigger pTask->triggerParam = pStream->triggerParam; diff --git a/source/dnode/mnode/impl/src/mndShow.c b/source/dnode/mnode/impl/src/mndShow.c index fed6a5a721088fc39f24712f922ffe13e80e48d7..27de3883e92acbd79a58b54f0f8a4ade6e3d61b9 100644 --- a/source/dnode/mnode/impl/src/mndShow.c +++ b/source/dnode/mnode/impl/src/mndShow.c @@ -236,7 +236,6 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { int32_t numOfCols = pShow->pMeta->numOfColumns; SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData idata = {0}; @@ -271,7 +270,7 @@ static int32_t mndProcessRetrieveSysTableReq(SRpcMsg *pReq) { } size = sizeof(SRetrieveMetaTableRsp) + sizeof(int32_t) + sizeof(SSysTableSchema) * pShow->pMeta->numOfColumns + - blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(pBlock->info.numOfCols); + blockDataGetSize(pBlock) + blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)); SRetrieveMetaTableRsp *pRsp = rpcMallocCont(size); if (pRsp == NULL) { diff --git a/source/dnode/mnode/impl/src/mndSnode.c b/source/dnode/mnode/impl/src/mndSnode.c index 12188a3b3a08aee7bdae08b202c3563106ee35d1..df1330197a7b98ed85e047fbc7522c87dac5e19f 100644 --- a/source/dnode/mnode/impl/src/mndSnode.c +++ b/source/dnode/mnode/impl/src/mndSnode.c @@ -65,7 +65,7 @@ SEpSet mndAcquireEpFromSnode(SMnode *pMnode, const SSnodeObj *pSnode) { return epSet; } -static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { +SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { SSnodeObj *pObj = sdbAcquire(pMnode->pSdb, SDB_SNODE, &snodeId); if (pObj == NULL && terrno == TSDB_CODE_SDB_OBJ_NOT_THERE) { terrno = TSDB_CODE_MND_SNODE_NOT_EXIST; @@ -73,7 +73,7 @@ static SSnodeObj *mndAcquireSnode(SMnode *pMnode, int32_t snodeId) { return pObj; } -static void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) { +void mndReleaseSnode(SMnode *pMnode, SSnodeObj *pObj) { SSdb *pSdb = pMnode->pSdb; sdbRelease(pSdb, pObj); } @@ -361,6 +361,14 @@ static int32_t mndSetDropSnodeRedoActions(STrans *pTrans, SDnodeObj *pDnode, SSn return 0; } +int32_t mndSetDropSnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SSnodeObj *pObj) { + if (pObj == NULL) return 0; + if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) return -1; + if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) return -1; + if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) return -1; + return 0; +} + static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { int32_t code = -1; @@ -368,10 +376,7 @@ static int32_t mndDropSnode(SMnode *pMnode, SRpcMsg *pReq, SSnodeObj *pObj) { if (pTrans == NULL) goto _OVER; mDebug("trans:%d, used to drop snode:%d", pTrans->id, pObj->id); - - if (mndSetDropSnodeRedoLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropSnodeCommitLogs(pTrans, pObj) != 0) goto _OVER; - if (mndSetDropSnodeRedoActions(pTrans, pObj->pDnode, pObj) != 0) goto _OVER; + if (mndSetDropSnodeInfoToTrans(pMnode, pTrans, pObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index 3e50ea8262045795cc0e8b28c7e5aa967713aacb..f1bae14c0704948520d150aafca8a526c656b3ca 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -37,12 +37,14 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw); static int32_t mndStbActionInsert(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb); static int32_t mndStbActionUpdate(SSdb *pSdb, SStbObj *pOld, SStbObj *pNew); +static int32_t mndProcessTtlTimer(SRpcMsg *pReq); static int32_t mndProcessCreateStbReq(SRpcMsg *pReq); static int32_t mndProcessAlterStbReq(SRpcMsg *pReq); static int32_t mndProcessDropStbReq(SRpcMsg *pReq); static int32_t mndProcessTableMetaReq(SRpcMsg *pReq); static int32_t mndRetrieveStb(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); static void mndCancelGetNextStb(SMnode *pMnode, void *pIter); +static int32_t mndProcessTableCfgReq(SRpcMsg *pReq); int32_t mndInitStb(SMnode *pMnode) { SSdbTable table = { @@ -62,6 +64,8 @@ int32_t mndInitStb(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_ALTER_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DROP_STB_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_TABLE_META, mndProcessTableMetaReq); + mndSetMsgHandle(pMnode, TDMT_MND_TTL_TIMER, mndProcessTtlTimer); + mndSetMsgHandle(pMnode, TDMT_MND_TABLE_CFG, mndProcessTableCfgReq); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STB, mndRetrieveStb); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STB, mndCancelGetNextStb); @@ -75,7 +79,7 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { terrno = TSDB_CODE_OUT_OF_MEMORY; int32_t size = sizeof(SStbObj) + (pStb->numOfColumns + pStb->numOfTags) * sizeof(SSchema) + +pStb->commentLen + - pStb->ast1Len + pStb->ast2Len + STB_RESERVE_SIZE; + pStb->ast1Len + pStb->ast2Len + STB_RESERVE_SIZE + taosArrayGetSize(pStb->pFuncs) * TSDB_FUNC_NAME_LEN; SSdbRaw *pRaw = sdbAllocRaw(SDB_STB, STB_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -100,6 +104,13 @@ SSdbRaw *mndStbActionEncode(SStbObj *pStb) { SDB_SET_INT32(pRaw, dataPos, pStb->ast1Len, _OVER) SDB_SET_INT32(pRaw, dataPos, pStb->ast2Len, _OVER) + int32_t funcNum = taosArrayGetSize(pStb->pFuncs); + SDB_SET_INT32(pRaw, dataPos, funcNum, _OVER) + for (int32_t i = 0; i < funcNum; ++i) { + char* func = taosArrayGet(pStb->pFuncs, i); + SDB_SET_BINARY(pRaw, dataPos, func, TSDB_FUNC_NAME_LEN, _OVER) + } + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { SSchema *pSchema = &pStb->pColumns[i]; SDB_SET_INT8(pRaw, dataPos, pSchema->type, _OVER) @@ -181,6 +192,20 @@ static SSdbRow *mndStbActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pStb->ast1Len, _OVER) SDB_GET_INT32(pRaw, dataPos, &pStb->ast2Len, _OVER) + int32_t funcNum = 0; + SDB_GET_INT32(pRaw, dataPos, &funcNum, _OVER) + if (funcNum > 0) { + pStb->pFuncs = taosArrayInit(funcNum, TSDB_FUNC_NAME_LEN); + if (NULL == pStb->pFuncs) { + goto _OVER; + } + char funcName[TSDB_FUNC_NAME_LEN]; + for (int32_t i = 0; i < funcNum; ++i) { + SDB_GET_BINARY(pRaw, dataPos, funcName, TSDB_FUNC_NAME_LEN, _OVER) + taosArrayPush(pStb->pFuncs, funcName); + } + } + pStb->pColumns = taosMemoryCalloc(pStb->numOfColumns, sizeof(SSchema)); pStb->pTags = taosMemoryCalloc(pStb->numOfTags, sizeof(SSchema)); if (pStb->pColumns == NULL || pStb->pTags == NULL) { @@ -250,6 +275,7 @@ static int32_t mndStbActionDelete(SSdb *pSdb, SStbObj *pStb) { taosMemoryFreeClear(pStb->comment); taosMemoryFreeClear(pStb->pAst1); taosMemoryFreeClear(pStb->pAst2); + taosArrayDestroy(pStb->pFuncs); return 0; } @@ -680,6 +706,9 @@ int32_t mndBuildStbFromReq(SMnode *pMnode, SStbObj *pDst, SMCreateStbReq *pCreat pDst->numOfColumns = pCreate->numOfColumns; pDst->numOfTags = pCreate->numOfTags; pDst->commentLen = pCreate->commentLen; + pDst->pFuncs = pCreate->pFuncs; + pCreate->pFuncs = NULL; + if (pDst->commentLen > 0) { pDst->comment = taosMemoryCalloc(pDst->commentLen + 1, 1); if (pDst->comment == NULL) { @@ -772,6 +801,43 @@ int32_t mndAddStbToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SStbObj *p return 0; } +static int32_t mndProcessTtlTimer(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + SSdb *pSdb = pMnode->pSdb; + SVgObj *pVgroup = NULL; + void *pIter = NULL; + + while (1) { + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + if (pIter == NULL) break; + + int32_t contLen = sizeof(SMsgHead) + sizeof(int32_t); + SMsgHead *pHead = rpcMallocCont(contLen); + if (pHead == NULL) { + sdbCancelFetch(pSdb, pVgroup); + sdbRelease(pSdb, pVgroup); + continue; + } + pHead->contLen = htonl(contLen); + pHead->vgId = htonl(pVgroup->vgId); + + int32_t t = taosGetTimestampSec(); + *(int32_t *)((char *)pHead + sizeof(SMsgHead)) = htonl(t); + + SRpcMsg rpcMsg = {.msgType = TDMT_VND_DROP_TTL_TABLE, .pCont = pHead, .contLen = contLen}; + SEpSet epSet = mndGetVgroupEpset(pMnode, pVgroup); + int32_t code = tmsgSendReq(&epSet, &rpcMsg); + if (code != 0) { + mError("failed to send ttl time seed, code:0x%x", code); + } else { + mDebug("send ttl time seed success, time:%d", t); + } + sdbRelease(pSdb, pVgroup); + } + + return 0; +} + static int32_t mndProcessCreateStbReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; int32_t code = -1; @@ -1277,6 +1343,60 @@ static int32_t mndBuildStbSchemaImp(SDbObj *pDb, SStbObj *pStb, const char *tbNa return 0; } +static int32_t mndBuildStbCfgImp(SDbObj *pDb, SStbObj *pStb, const char *tbName, STableCfgRsp *pRsp) { + taosRLockLatch(&pStb->lock); + + int32_t totalCols = pStb->numOfColumns + pStb->numOfTags; + pRsp->pSchemas = taosMemoryCalloc(totalCols, sizeof(SSchema)); + if (pRsp->pSchemas == NULL) { + taosRUnLockLatch(&pStb->lock); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + strcpy(pRsp->dbFName, pStb->db); + strcpy(pRsp->tbName, tbName); + strcpy(pRsp->stbName, tbName); + pRsp->numOfTags = pStb->numOfTags; + pRsp->numOfColumns = pStb->numOfColumns; + pRsp->tableType = TSDB_SUPER_TABLE; + pRsp->delay1 = pStb->maxdelay[0]; + pRsp->delay2 = pStb->maxdelay[1]; + pRsp->watermark1 = pStb->watermark[0]; + pRsp->watermark2 = pStb->watermark[1]; + pRsp->ttl = pStb->ttl; + pRsp->commentLen = pStb->commentLen; + if (pStb->commentLen > 0) { + pRsp->pComment = strdup(pStb->comment); + } + + for (int32_t i = 0; i < pStb->numOfColumns; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i]; + SSchema *pSrcSchema = &pStb->pColumns[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = pSrcSchema->colId; + pSchema->bytes = pSrcSchema->bytes; + } + + for (int32_t i = 0; i < pStb->numOfTags; ++i) { + SSchema *pSchema = &pRsp->pSchemas[i + pStb->numOfColumns]; + SSchema *pSrcSchema = &pStb->pTags[i]; + memcpy(pSchema->name, pSrcSchema->name, TSDB_COL_NAME_LEN); + pSchema->type = pSrcSchema->type; + pSchema->colId = pSrcSchema->colId; + pSchema->bytes = pSrcSchema->bytes; + } + + if (pStb->pFuncs) { + pRsp->pFuncs = taosArrayDup(pStb->pFuncs); + } + + taosRUnLockLatch(&pStb->lock); + return 0; +} + + static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char *tbName, STableMetaRsp *pRsp, int32_t *smaVer) { char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; @@ -1305,6 +1425,32 @@ static int32_t mndBuildStbSchema(SMnode *pMnode, const char *dbFName, const char return code; } +static int32_t mndBuildStbCfg(SMnode *pMnode, const char *dbFName, const char *tbName, STableCfgRsp *pRsp) { + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", dbFName, tbName); + + SDbObj *pDb = mndAcquireDb(pMnode, dbFName); + if (pDb == NULL) { + terrno = TSDB_CODE_MND_DB_NOT_SELECTED; + return -1; + } + + SStbObj *pStb = mndAcquireStb(pMnode, tbFName); + if (pStb == NULL) { + mndReleaseDb(pMnode, pDb); + terrno = TSDB_CODE_PAR_TABLE_NOT_EXIST; + return -1; + } + + int32_t code = mndBuildStbCfgImp(pDb, pStb, tbName, pRsp); + + mndReleaseDb(pMnode, pDb); + mndReleaseStb(pMnode, pStb); + return code; +} + + + static int32_t mndBuildSMAlterStbRsp(SDbObj *pDb, const SMAlterStbReq *pAlter, SStbObj *pObj, void **pCont, int32_t *pLen) { int32_t ret; @@ -1664,6 +1810,63 @@ _OVER: return code; } +static int32_t mndProcessTableCfgReq(SRpcMsg *pReq) { + SMnode *pMnode = pReq->info.node; + int32_t code = -1; + STableCfgReq cfgReq = {0}; + STableCfgRsp cfgRsp = {0}; + + if (tDeserializeSTableCfgReq(pReq->pCont, pReq->contLen, &cfgReq) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + if (0 == strcmp(cfgReq.dbFName, TSDB_INFORMATION_SCHEMA_DB)) { + mDebug("information_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); + if (mndBuildInsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { + goto _OVER; + } + } else if (0 == strcmp(cfgReq.dbFName, TSDB_PERFORMANCE_SCHEMA_DB)) { + mDebug("performance_schema table:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); + if (mndBuildPerfsTableCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { + goto _OVER; + } + } else { + mDebug("stb:%s.%s, start to retrieve cfg", cfgReq.dbFName, cfgReq.tbName); + if (mndBuildStbCfg(pMnode, cfgReq.dbFName, cfgReq.tbName, &cfgRsp) != 0) { + goto _OVER; + } + } + + int32_t rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); + if (rspLen < 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + void *pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + + tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); + pReq->info.rsp = pRsp; + pReq->info.rspLen = rspLen; + code = 0; + + mTrace("%s.%s, cfg is retrieved", cfgReq.dbFName, cfgReq.tbName); + +_OVER: + if (code != 0) { + mError("stb:%s.%s, failed to retrieve cfg since %s", cfgReq.dbFName, cfgReq.tbName, terrstr()); + } + + tFreeSTableCfgRsp(&cfgRsp); + return code; +} + + int32_t mndValidateStbInfo(SMnode *pMnode, SSTableVersion *pStbVersions, int32_t numOfStbs, void **ppRsp, int32_t *pRspLen) { SSTbHbRsp hbRsp = {0}; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index fb92efecf618d779d63efb613ed5c80baa8188a0..7e9069d5fcf0fc62a349254d10b251d80ef4f0ec 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -54,9 +54,10 @@ int32_t mndInitStream(SMnode *pMnode) { }; mndSetMsgHandle(pMnode, TDMT_MND_CREATE_STREAM, mndProcessCreateStreamReq); - mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM, mndProcessDropStreamReq); - mndSetMsgHandle(pMnode, TDMT_MND_DROP_STREAM_RSP, mndTransProcessRsp); + + mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DEPLOY_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_STREAM_TASK_DROP_RSP, mndTransProcessRsp); mndAddShowRetrieveHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndRetrieveStream); mndAddShowFreeIterHandle(pMnode, TSDB_MGMT_TABLE_STREAMS, mndCancelGetNextStream); @@ -477,7 +478,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); action.pCont = pReq; action.contLen = sizeof(SVDropStreamTaskReq); - action.msgType = TDMT_VND_STREAM_TASK_DROP; + action.msgType = TDMT_STREAM_TASK_DROP; if (mndTransAppendRedoAction(pTrans, &action) != 0) { taosMemoryFree(pReq); return -1; @@ -670,20 +671,24 @@ _OVER: static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; - int32_t code = -1; SStreamObj *pStream = NULL; /*SDbObj *pDb = NULL;*/ /*SUserObj *pUser = NULL;*/ - SMDropStreamReq dropReq = *(SMDropStreamReq *)pReq->pCont; + SMDropStreamReq dropReq = {0}; + if (tDeserializeSMDropStreamReq(pReq->pCont, pReq->contLen, &dropReq) < 0) { + ASSERT(0); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } pStream = mndAcquireStream(pMnode, dropReq.name); if (pStream == NULL) { if (dropReq.igNotExists) { mDebug("stream:%s, not exist, ignore not exist is set", dropReq.name); - code = 0; - goto DROP_STREAM_OVER; + sdbRelease(pMnode->pSdb, pStream); + return -1; } else { terrno = TSDB_CODE_MND_STREAM_NOT_EXIST; return -1; @@ -701,14 +706,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq); if (pTrans == NULL) { mError("stream:%s, failed to drop since %s", dropReq.name, terrstr()); - return code; + sdbRelease(pMnode->pSdb, pStream); + return -1; } mDebug("trans:%d, used to drop stream:%s", pTrans->id, dropReq.name); // drop all tasks if (mndDropStreamTasks(pMnode, pTrans, pStream) < 0) { mError("stream:%s, failed to drop task since %s", dropReq.name, terrstr()); - return code; + sdbRelease(pMnode->pSdb, pStream); + return -1; } // drop stream @@ -717,8 +724,16 @@ static int32_t mndProcessDropStreamReq(SRpcMsg *pReq) { return -1; } -DROP_STREAM_OVER: - return code; + if (mndTransPrepare(pMnode, pTrans) != 0) { + mError("trans:%d, failed to prepare drop stream trans since %s", pTrans->id, terrstr()); + sdbRelease(pMnode->pSdb, pStream); + mndTransDrop(pTrans); + return -1; + } + + sdbRelease(pMnode->pSdb, pStream); + + return TSDB_CODE_ACTION_IN_PROGRESS; } int32_t mndDropStreamByDb(SMnode *pMnode, STrans *pTrans, SDbObj *pDb) { diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index d2b7a61e83ab5faa8ef412d1e4c9b638facfcf36..8dde3e92d8146235004a624d9f2e6545f9f9f3a1 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -96,6 +96,7 @@ static SMqSubscribeObj *mndCreateSub(SMnode *pMnode, const SMqTopicObj *pTopic, pSub->dbUid = pTopic->dbUid; pSub->stbUid = pTopic->stbUid; pSub->subType = pTopic->subType; + pSub->withMeta = pTopic->withMeta; ASSERT(pSub->unassignedVgs->size == 0); ASSERT(taosHashGetSize(pSub->consumerHash) == 0); @@ -120,6 +121,7 @@ static int32_t mndBuildSubChangeReq(void **pBuf, int32_t *pLen, const SMqSubscri req.vgId = pRebVg->pVgEp->vgId; req.qmsg = pRebVg->pVgEp->qmsg; req.subType = pSub->subType; + req.withMeta = pSub->withMeta; req.suid = pSub->stbUid; strncpy(req.subKey, pSub->key, TSDB_SUBSCRIBE_KEY_LEN); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 07f65b2a90a3f70498568091941fd76be6e644f8..e3358b41df07b75d3a1624bbd13f08645b1d66d7 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -75,12 +75,10 @@ void mndSyncCommitMsg(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SFsmCbMeta cbM } int32_t mndSyncGetSnapshot(struct SSyncFSM *pFsm, SSnapshot *pSnapshot, void *pReaderParam, void **ppReader) { - // TODO: - - // atomic operation - // step1. sdbGetCommitInfo - // step2. create ppReader with pReaderParam - + mDebug("start to read snapshot from sdb in atomic way"); + SMnode *pMnode = pFsm->data; + return sdbStartRead(pMnode->pSdb, (SSdbIter **)ppReader, &pSnapshot->lastApplyIndex, &pSnapshot->lastApplyTerm, + &pSnapshot->lastConfigIndex); return 0; } @@ -98,7 +96,7 @@ void mndRestoreFinish(struct SSyncFSM *pFsm) { mndTransPullup(pMnode); mndSetRestore(pMnode, true); } else { - mInfo("mnode sync restore finished, and will set ready after first deploy"); + mInfo("mnode sync restore finished"); } } @@ -106,14 +104,6 @@ void mndReConfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbM SMnode *pMnode = pFsm->data; SSyncMgmt *pMgmt = &pMnode->syncMgmt; -#if 0 -// send response - SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen, .conn.applyIndex = cbMeta.index}; - rpcMsg.pCont = rpcMallocCont(rpcMsg.contLen); - memcpy(rpcMsg.pCont, pMsg->pCont, pMsg->contLen); - syncGetAndDelRespRpc(pMnode->syncMgmt.sync, cbMeta.seqNum, &rpcMsg.info); -#endif - pMgmt->errCode = cbMeta.code; mInfo("trans:-1, sync reconfig is proposed, saved:%d code:0x%x, index:%" PRId64 " term:%" PRId64, pMgmt->transId, cbMeta.code, cbMeta.index, cbMeta.term); @@ -128,13 +118,13 @@ void mndReConfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReConfigCbMeta cbM } int32_t mndSnapshotStartRead(struct SSyncFSM *pFsm, void **ppReader) { - mInfo("start to read snapshot from sdb"); + mDebug("start to read snapshot from sdb"); SMnode *pMnode = pFsm->data; - return sdbStartRead(pMnode->pSdb, (SSdbIter **)ppReader); + return sdbStartRead(pMnode->pSdb, (SSdbIter **)ppReader, NULL, NULL, NULL); } int32_t mndSnapshotStopRead(struct SSyncFSM *pFsm, void *pReader) { - mInfo("stop to read snapshot from sdb"); + mDebug("stop to read snapshot from sdb"); SMnode *pMnode = pFsm->data; return sdbStopRead(pMnode->pSdb, pReader); } diff --git a/source/dnode/mnode/impl/src/mndTopic.c b/source/dnode/mnode/impl/src/mndTopic.c index 9632c04f4c4166d62ffaad64b8af572e3983aeec..a650ed29f1c4633f7f217d36494be623f51d9546 100644 --- a/source/dnode/mnode/impl/src/mndTopic.c +++ b/source/dnode/mnode/impl/src/mndTopic.c @@ -141,6 +141,7 @@ SSdbRaw *mndTopicActionEncode(SMqTopicObj *pTopic) { SDB_SET_INT64(pRaw, dataPos, pTopic->dbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->version, TOPIC_ENCODE_OVER); SDB_SET_INT8(pRaw, dataPos, pTopic->subType, TOPIC_ENCODE_OVER); + SDB_SET_INT8(pRaw, dataPos, pTopic->withMeta, TOPIC_ENCODE_OVER); SDB_SET_INT64(pRaw, dataPos, pTopic->stbUid, TOPIC_ENCODE_OVER); SDB_SET_INT32(pRaw, dataPos, pTopic->sqlLen, TOPIC_ENCODE_OVER); @@ -208,6 +209,7 @@ SSdbRow *mndTopicActionDecode(SSdbRaw *pRaw) { SDB_GET_INT64(pRaw, dataPos, &pTopic->dbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->version, TOPIC_DECODE_OVER); SDB_GET_INT8(pRaw, dataPos, &pTopic->subType, TOPIC_DECODE_OVER); + SDB_GET_INT8(pRaw, dataPos, &pTopic->withMeta, TOPIC_DECODE_OVER); SDB_GET_INT64(pRaw, dataPos, &pTopic->stbUid, TOPIC_DECODE_OVER); SDB_GET_INT32(pRaw, dataPos, &pTopic->sqlLen, TOPIC_DECODE_OVER); @@ -357,6 +359,10 @@ static int32_t mndCreateTopic(SMnode *pMnode, SRpcMsg *pReq, SCMCreateTopicReq * topicObj.sql = strdup(pCreate->sql); topicObj.sqlLen = strlen(pCreate->sql) + 1; topicObj.subType = pCreate->subType; + topicObj.withMeta = pCreate->withMeta; + if (topicObj.withMeta) { + ASSERT(topicObj.subType != TOPIC_SUB_TYPE__COLUMN); + } if (pCreate->subType == TOPIC_SUB_TYPE__COLUMN) { topicObj.ast = strdup(pCreate->ast); diff --git a/source/dnode/mnode/impl/src/mndTrans.c b/source/dnode/mnode/impl/src/mndTrans.c index de6a44d456c992defcdbae39d97b260b471e43a6..d1d88fdc9020dbdda72b1c70a2c9270f0b9e2ac8 100644 --- a/source/dnode/mnode/impl/src/mndTrans.c +++ b/source/dnode/mnode/impl/src/mndTrans.c @@ -56,6 +56,7 @@ static bool mndCannotExecuteTransAction(SMnode *pMnode) { return !pMnode->dep static void mndTransSendRpcRsp(SMnode *pMnode, STrans *pTrans); static int32_t mndProcessTransReq(SRpcMsg *pReq); +static int32_t mndProcessTtl(SRpcMsg *pReq); static int32_t mndProcessKillTransReq(SRpcMsg *pReq); static int32_t mndRetrieveTrans(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -432,11 +433,11 @@ static const char *mndTransStr(ETrnStage stage) { } static void mndTransTestStartFunc(SMnode *pMnode, void *param, int32_t paramLen) { - mInfo("test trans start, param:%s, len:%d", (char *)param, paramLen); + mDebug("test trans start, param:%s, len:%d", (char *)param, paramLen); } static void mndTransTestStopFunc(SMnode *pMnode, void *param, int32_t paramLen) { - mInfo("test trans stop, param:%s, len:%d", (char *)param, paramLen); + mDebug("test trans stop, param:%s, len:%d", (char *)param, paramLen); } static TransCbFp mndTransGetCbFp(ETrnFunc ftype) { diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 94ddbcd409c8ef4b1ad6bf8d97bda63edb63ea92..ae13987d259d10d4d001f5dd9cee6c17ec93018e 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -1017,6 +1017,11 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, if (mndAddDropVnodeAction(pMnode, pTrans, pDb, &newVg, &del, true) != 0) return -1; if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pDb, &newVg) != 0) return -1; + SSdbRaw *pRaw = mndVgroupActionEncode(&newVg); + if (pRaw == NULL || mndTransAppendCommitlog(pTrans, pRaw) != 0) return -1; + sdbSetRawStatus(pRaw, SDB_STATUS_READY); + pRaw = NULL; + mInfo("vgId:%d, vgroup info after move, replica:%d", newVg.vgId, newVg.replica); for (int32_t i = 0; i < newVg.replica; ++i) { mInfo("vgId:%d, vnode:%d dnode:%d", newVg.vgId, i, newVg.vnodeGid[i].dnodeId); @@ -1025,6 +1030,7 @@ int32_t mndSetMoveVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, } int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t delDnodeId) { + int32_t code = 0; SArray *pArray = mndBuildDnodesArray(pMnode, delDnodeId); if (pArray == NULL) return -1; @@ -1042,18 +1048,24 @@ int32_t mndSetMoveVgroupsInfoToTrans(SMnode *pMnode, STrans *pTrans, int32_t del } } + code = 0; if (vnIndex != -1) { mInfo("vgId:%d, vnode:%d will be removed from dnode:%d", pVgroup->vgId, vnIndex, delDnodeId); SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); - mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray); + code = mndSetMoveVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, vnIndex, pArray); mndReleaseDb(pMnode, pDb); } sdbRelease(pMnode->pSdb, pVgroup); + + if (code != 0) { + sdbCancelFetch(pMnode->pSdb, pIter); + break; + } } taosArrayDestroy(pArray); - return 0; + return code; } static int32_t mndAddIncVgroupReplicaToTrans(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, @@ -1541,10 +1553,11 @@ static int32_t mndSetBalanceVgroupInfoToTrans(SMnode *pMnode, STrans *pTrans, SD static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDnodeObj *pSrc, SDnodeObj *pDst) { void *pIter = NULL; int32_t code = -1; + SSdb *pSdb = pMnode->pSdb; while (1) { SVgObj *pVgroup = NULL; - pIter = sdbFetch(pMnode->pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); + pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); if (pIter == NULL) break; bool existInSrc = false; @@ -1556,13 +1569,15 @@ static int32_t mndBalanceVgroupBetweenDnode(SMnode *pMnode, STrans *pTrans, SDno } if (!existInSrc || existInDst) { - sdbRelease(pMnode->pSdb, pVgroup); + sdbRelease(pSdb, pVgroup); + continue; } SDbObj *pDb = mndAcquireDb(pMnode, pVgroup->dbName); code = mndSetBalanceVgroupInfoToTrans(pMnode, pTrans, pDb, pVgroup, pSrc, pDst); mndReleaseDb(pMnode, pDb); - sdbRelease(pMnode->pSdb, pVgroup); + sdbRelease(pSdb, pVgroup); + sdbCancelFetch(pSdb, pIter); break; } @@ -1581,15 +1596,25 @@ static int32_t mndBalanceVgroup(SMnode *pMnode, SRpcMsg *pReq, SArray *pArray) { while (1) { taosArraySort(pArray, (__compar_fn_t)mndCompareDnodeVnodes); - SDnodeObj *pSrc = taosArrayGet(pArray, 0); - SDnodeObj *pDst = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1); + for (int32_t i = 0; i < taosArrayGetSize(pArray); ++i) { + SDnodeObj *pDnode = taosArrayGet(pArray, i); + mDebug("dnode:%d, equivalent vnodes:%d support:%d, score:%f", pDnode->id, pDnode->numOfVnodes, + pDnode->numOfSupportVnodes, (float)pDnode->numOfVnodes / pDnode->numOfSupportVnodes); + } + + SDnodeObj *pSrc = taosArrayGet(pArray, taosArrayGetSize(pArray) - 1); + SDnodeObj *pDst = taosArrayGet(pArray, 0); float srcScore = (float)(pSrc->numOfVnodes - 1) / pSrc->numOfSupportVnodes; float dstScore = (float)(pDst->numOfVnodes + 1) / pDst->numOfSupportVnodes; - if (srcScore + 0.0001 < dstScore) { - mDebug("trans:%d, balance vgroup from dnode:%d to dnode:%d", pTrans->id, pSrc->id, pDst->id); + mDebug("trans:%d, after balance, src dnode:%d score:%f, dst dnode:%d score:%f", pTrans->id, pSrc->id, srcScore, + pDst->id, dstScore); + + if (srcScore > dstScore - 0.000001) { code = mndBalanceVgroupBetweenDnode(pMnode, pTrans, pSrc, pDst); if (code == 0) { + pSrc->numOfVnodes--; + pDst->numOfVnodes++; numOfVgroups++; continue; } else { @@ -1623,7 +1648,13 @@ static int32_t mndProcessBalanceVgroupMsg(SRpcMsg *pReq) { void *pIter = NULL; int64_t curMs = taosGetTimestampMs(); - mDebug("start to balance vgroup"); + SBalanceVgroupReq req = {0}; + if (tDeserializeSBalanceVgroupReq(pReq->pCont, pReq->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + goto _OVER; + } + + mInfo("start to balance vgroup"); if (mndCheckOperAuth(pMnode, pReq->info.conn.user, MND_OPER_BALANCE_VGROUP) != 0) goto _OVER; diff --git a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp index bc118ee26e5996e28e6ef3c1380a71a7f868dd3f..87895d3b4930ff78ab4b147f688f98ff9cd71229 100644 --- a/source/dnode/mnode/impl/test/sdb/sdbTest.cpp +++ b/source/dnode/mnode/impl/test/sdb/sdbTest.cpp @@ -910,7 +910,7 @@ TEST_F(MndTestSdb, 01_Read_Str) { int32_t len = 0; int32_t code = 0; - code = sdbStartRead(pSdb, &pReader); + code = sdbStartRead(pSdb, &pReader, NULL, NULL, NULL); ASSERT_EQ(code, 0); code = sdbStartWrite(pSdb, &pWritter); ASSERT_EQ(code, 0); diff --git a/source/dnode/mnode/sdb/inc/sdb.h b/source/dnode/mnode/sdb/inc/sdb.h index 3b1c4000a83957ea5c7360e40857e9fd4aee8aa2..1294f0cff3446dca7fb7dcdfc6d293d947d13852 100644 --- a/source/dnode/mnode/sdb/inc/sdb.h +++ b/source/dnode/mnode/sdb/inc/sdb.h @@ -388,7 +388,7 @@ SSdbRow *sdbAllocRow(int32_t objSize); void *sdbGetRowObj(SSdbRow *pRow); void sdbFreeRow(SSdb *pSdb, SSdbRow *pRow, bool callFunc); -int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter); +int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config); int32_t sdbStopRead(SSdb *pSdb, SSdbIter *pIter); int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len); diff --git a/source/dnode/mnode/sdb/src/sdbFile.c b/source/dnode/mnode/sdb/src/sdbFile.c index 0f4e1276c1612ad0f1707057d9e052eb307fe452..ff4a9e4ead5511b3c876522f2103b429f30da4ed 100644 --- a/source/dnode/mnode/sdb/src/sdbFile.c +++ b/source/dnode/mnode/sdb/src/sdbFile.c @@ -519,11 +519,11 @@ static void sdbCloseIter(SSdbIter *pIter) { pIter->name = NULL; } - mInfo("sdbiter:%p, is closed, total:%" PRId64, pIter, pIter->total); + mDebug("sdbiter:%p, is closed, total:%" PRId64, pIter, pIter->total); taosMemoryFree(pIter); } -int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { +int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter, int64_t *index, int64_t *term, int64_t *config) { SSdbIter *pIter = sdbCreateIter(pSdb); if (pIter == NULL) return -1; @@ -552,7 +552,11 @@ int32_t sdbStartRead(SSdb *pSdb, SSdbIter **ppIter) { } *ppIter = pIter; - mInfo("sdbiter:%p, is created to read snapshot, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", + if (index != NULL) *index = commitIndex; + if (term != NULL) *term = commitTerm; + if (config != NULL) *config = commitConfig; + + mDebug("sdbiter:%p, is created to read snapshot, commit index:%" PRId64 " term:%" PRId64 " config:%" PRId64 " file:%s", pIter, commitIndex, commitTerm, commitConfig, pIter->name); return 0; } @@ -579,14 +583,14 @@ int32_t sdbDoRead(SSdb *pSdb, SSdbIter *pIter, void **ppBuf, int32_t *len) { taosMemoryFree(pBuf); return -1; } else if (readlen == 0) { - mInfo("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total); + mDebug("sdbiter:%p, read snapshot to the end, total:%" PRId64, pIter, pIter->total); *ppBuf = NULL; *len = 0; taosMemoryFree(pBuf); return 0; } else { // (readlen <= maxlen) pIter->total += readlen; - mInfo("sdbiter:%p, read:%d bytes from snapshot, total:%" PRId64, pIter, readlen, pIter->total); + mDebug("sdbiter:%p, read:%d bytes from snapshot, total:%" PRId64, pIter, readlen, pIter->total); *ppBuf = pBuf; *len = readlen; return 0; @@ -605,7 +609,7 @@ int32_t sdbStartWrite(SSdb *pSdb, SSdbIter **ppIter) { } *ppIter = pIter; - mInfo("sdbiter:%p, is created to write snapshot, file:%s", pIter, pIter->name); + mDebug("sdbiter:%p, is created to write snapshot, file:%s", pIter, pIter->name); return 0; } @@ -614,7 +618,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply) { if (!isApply) { sdbCloseIter(pIter); - mInfo("sdbiter:%p, not apply to sdb", pIter); + mDebug("sdbiter:%p, not apply to sdb", pIter); return 0; } @@ -637,7 +641,7 @@ int32_t sdbStopWrite(SSdb *pSdb, SSdbIter *pIter, bool isApply) { return -1; } - mInfo("sdbiter:%p, successfully applyed to sdb", pIter); + mDebug("sdbiter:%p, successfully applyed to sdb", pIter); return 0; } @@ -650,6 +654,6 @@ int32_t sdbDoWrite(SSdb *pSdb, SSdbIter *pIter, void *pBuf, int32_t len) { } pIter->total += writelen; - mInfo("sdbiter:%p, write:%d bytes to snapshot, total:%" PRId64, pIter, writelen, pIter->total); + mDebug("sdbiter:%p, write:%d bytes to snapshot, total:%" PRId64, pIter, writelen, pIter->total); return 0; } \ No newline at end of file diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 84a66c680bb9bbd54114a19ed712b24cb8ebe7a9..3a92cba7731576658a8fa9edc9402c2c0e0ab068 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -92,7 +92,7 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) { } tDecoderClear(&decoder); - pTask->status = TASK_STATUS__IDLE; + pTask->execStatus = TASK_EXEC_STATUS__IDLE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -105,13 +105,14 @@ static int32_t sndProcessTaskDeployReq(SSnode *pNode, SRpcMsg *pMsg) { ASSERT(pTask->execType != TASK_EXEC__NONE); - ASSERT(pTask->dataScan == 0); + ASSERT(pTask->isDataScan == 0); pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, NULL); ASSERT(pTask->exec.executor); streamSetupTrigger(pTask); - qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId, pTask->childId); + qInfo("deploy stream: stream id %ld task id %d child id %d on snode", pTask->streamId, pTask->taskId, + pTask->selfChildId); taosHashPut(pMeta->pHash, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void *)); @@ -198,6 +199,34 @@ static int32_t sndProcessTaskDropReq(SSnode *pNode, SRpcMsg *pMsg) { return code; } +static int32_t sndProcessTaskRetrieveReq(SSnode *pNode, SRpcMsg *pMsg) { + SStreamMeta *pMeta = pNode->pMeta; + + char *msgStr = pMsg->pCont; + char *msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SStreamRetrieveReq req; + SDecoder decoder; + tDecoderInit(&decoder, msgBody, msgLen); + tDecodeStreamRetrieveReq(&decoder, &req); + int32_t taskId = req.dstTaskId; + SStreamTask *pTask = *(SStreamTask **)taosHashGet(pMeta->pHash, &taskId, sizeof(int32_t)); + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } + SRpcMsg rsp = { + .info = pMsg->info, + .code = 0, + }; + streamProcessRetrieveReq(pTask, &req, &rsp); + return 0; +} + +static int32_t sndProcessTaskRetrieveRsp(SSnode *pNode, SRpcMsg *pMsg) { + // + return 0; +} + int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { // stream deploy // stream stop/resume @@ -205,7 +234,7 @@ int32_t sndProcessUMsg(SSnode *pSnode, SRpcMsg *pMsg) { switch (pMsg->msgType) { case TDMT_STREAM_TASK_DEPLOY: return sndProcessTaskDeployReq(pSnode, pMsg); - case TDMT_VND_STREAM_TASK_DROP: + case TDMT_STREAM_TASK_DROP: return sndProcessTaskDropReq(pSnode, pMsg); default: ASSERT(0); @@ -221,10 +250,14 @@ int32_t sndProcessSMsg(SSnode *pSnode, SRpcMsg *pMsg) { return sndProcessTaskDispatchReq(pSnode, pMsg); case TDMT_STREAM_TASK_RECOVER: return sndProcessTaskRecoverReq(pSnode, pMsg); + case TDMT_STREAM_RETRIEVE: + return sndProcessTaskRecoverReq(pSnode, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return sndProcessTaskDispatchRsp(pSnode, pMsg); case TDMT_STREAM_TASK_RECOVER_RSP: return sndProcessTaskRecoverRsp(pSnode, pMsg); + case TDMT_STREAM_RETRIEVE_RSP: + return sndProcessTaskRecoverRsp(pSnode, pMsg); default: ASSERT(0); } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 15eb35c700f42edeb041b69b48385ab5df6851f6..b09ab3422429876e5ecd41d79318de07a05f2739 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -29,6 +29,7 @@ target_sources( # sma "src/sma/sma.c" "src/sma/smaEnv.c" + "src/sma/smaUtil.c" "src/sma/smaOpen.c" "src/sma/smaRollup.c" "src/sma/smaTimeRange.c" diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 70d89102e67dd933a6228402f647a212d953f63a..c7e8e8dc86f26d55c836fc7e5bc69ef9f87668be 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -149,8 +149,8 @@ int32_t tqReadHandleRemoveTbUidList(STqReadHandle *pHandle, const SArray *tbUidL int32_t tqReadHandleSetMsg(STqReadHandle *pHandle, SSubmitReq *pMsg, int64_t ver); bool tqNextDataBlock(STqReadHandle *pHandle); bool tqNextDataBlockFilterOut(STqReadHandle *pHandle, SHashObj *filterOutUids); -int32_t tqRetrieveDataBlock(SArray **ppCols, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, - int32_t *pNumOfRows, int16_t *pNumOfCols); +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle *pHandle, uint64_t *pGroupId, uint64_t *pUid, + int32_t *pNumOfRows); // sma int32_t smaGetTSmaDays(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); @@ -195,6 +195,7 @@ struct SVnodeCfg { typedef struct { TSKEY lastKey; uint64_t uid; + uint64_t groupId; } STableKeyInfo; struct SMetaEntry { diff --git a/source/dnode/vnode/src/inc/sma.h b/source/dnode/vnode/src/inc/sma.h index 902e5e1bcc291fbe4826b41d8702f716fd6b6a58..1489b02492575791547e873d6414a932bcc2d3cf 100644 --- a/source/dnode/vnode/src/inc/sma.h +++ b/source/dnode/vnode/src/inc/sma.h @@ -34,7 +34,8 @@ extern "C" { typedef struct SSmaEnv SSmaEnv; typedef struct SSmaStat SSmaStat; -typedef struct SSmaStatItem SSmaStatItem; +typedef struct STSmaStat STSmaStat; +typedef struct SRSmaStat SRSmaStat; typedef struct SSmaKey SSmaKey; typedef struct SRSmaInfo SRSmaInfo; typedef struct SRSmaInfoItem SRSmaInfoItem; @@ -45,26 +46,38 @@ struct SSmaEnv { SSmaStat *pStat; }; -#define SMA_ENV_LOCK(env) ((env)->lock) -#define SMA_ENV_TYPE(env) ((env)->type) -#define SMA_ENV_STAT(env) ((env)->pStat) -#define SMA_ENV_STAT_ITEM(env) ((env)->pStat->tsmaStatItem) +#define SMA_ENV_LOCK(env) ((env)->lock) +#define SMA_ENV_TYPE(env) ((env)->type) +#define SMA_ENV_STAT(env) ((env)->pStat) -struct SSmaStatItem { +struct STSmaStat { int8_t state; // ETsdbSmaStat STSma *pTSma; // cache schema STSchema *pTSchema; }; +struct SRSmaStat { + SSma *pSma; + void *tmrHandle; + tmr_h tmrId; + int8_t tmrStat; + int32_t tmrSeconds; + SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; +}; + struct SSmaStat { union { - SSmaStatItem tsmaStatItem; - SHashObj *rsmaInfoHash; // key: stbUid, value: SRSmaInfo; + STSmaStat tsmaStat; // time-range-wise sma + SRSmaStat rsmaStat; // rollup sma }; T_REF_DECLARE() }; -#define SMA_STAT_ITEM(s) ((s)->tsmaStatItem) -#define SMA_STAT_INFO_HASH(s) ((s)->rsmaInfoHash) +#define SMA_TSMA_STAT(s) (&(s)->tsmaStat) +#define SMA_RSMA_STAT(s) (&(s)->rsmaStat) +#define SMA_RSMA_INFO_HASH(s) ((s)->rsmaStat.rsmaInfoHash) +#define SMA_RSMA_TMR_HANDLE(s) ((s)->rsmaStat.tmrHandle) +#define SMA_RSMA_TMR_STAT(s) ((s)->rsmaStat.tmrStat) +#define RSMA_INFO_HASH(r) ((r)->rsmaInfoHash) void tdDestroySmaEnv(SSmaEnv *pSmaEnv); void *tdFreeSmaEnv(SSmaEnv *pSmaEnv); @@ -107,53 +120,51 @@ static FORCE_INLINE int32_t tdUnLockSmaEnv(SSmaEnv *pEnv) { return 0; } -static FORCE_INLINE int8_t tdSmaStat(SSmaStatItem *pStatItem) { - if (pStatItem) { - return atomic_load_8(&pStatItem->state); +static FORCE_INLINE int8_t tdSmaStat(STSmaStat *pTStat) { + if (pTStat) { + return atomic_load_8(&pTStat->state); } return TSDB_SMA_STAT_UNKNOWN; } -static FORCE_INLINE bool tdSmaStatIsOK(SSmaStatItem *pStatItem, int8_t *state) { - if (!pStatItem) { +static FORCE_INLINE bool tdSmaStatIsOK(STSmaStat *pTStat, int8_t *state) { + if (!pTStat) { return false; } if (state) { - *state = atomic_load_8(&pStatItem->state); + *state = atomic_load_8(&pTStat->state); return *state == TSDB_SMA_STAT_OK; } - return atomic_load_8(&pStatItem->state) == TSDB_SMA_STAT_OK; + return atomic_load_8(&pTStat->state) == TSDB_SMA_STAT_OK; } -static FORCE_INLINE bool tdSmaStatIsExpired(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_EXPIRED) : true; +static FORCE_INLINE bool tdSmaStatIsExpired(STSmaStat *pTStat) { + return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_EXPIRED) : true; } -static FORCE_INLINE bool tdSmaStatIsDropped(SSmaStatItem *pStatItem) { - return pStatItem ? (atomic_load_8(&pStatItem->state) & TSDB_SMA_STAT_DROPPED) : true; +static FORCE_INLINE bool tdSmaStatIsDropped(STSmaStat *pTStat) { + return pTStat ? (atomic_load_8(&pTStat->state) & TSDB_SMA_STAT_DROPPED) : true; } -static FORCE_INLINE void tdSmaStatSetOK(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_store_8(&pStatItem->state, TSDB_SMA_STAT_OK); +static FORCE_INLINE void tdSmaStatSetOK(STSmaStat *pTStat) { + if (pTStat) { + atomic_store_8(&pTStat->state, TSDB_SMA_STAT_OK); } } -static FORCE_INLINE void tdSmaStatSetExpired(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_EXPIRED); +static FORCE_INLINE void tdSmaStatSetExpired(STSmaStat *pTStat) { + if (pTStat) { + atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_EXPIRED); } } -static FORCE_INLINE void tdSmaStatSetDropped(SSmaStatItem *pStatItem) { - if (pStatItem) { - atomic_or_fetch_8(&pStatItem->state, TSDB_SMA_STAT_DROPPED); +static FORCE_INLINE void tdSmaStatSetDropped(STSmaStat *pTStat) { + if (pTStat) { + atomic_or_fetch_8(&pTStat->state, TSDB_SMA_STAT_DROPPED); } } -static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); -void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem); static int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType); void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType); @@ -163,6 +174,51 @@ int32_t tdProcessTSmaCreateImpl(SSma *pSma, int64_t version, const char *pMsg); int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg); int32_t tdProcessTSmaGetDaysImpl(SVnodeCfg *pCfg, void *pCont, uint32_t contLen, int32_t *days); +typedef struct STFInfo STFInfo; +typedef struct STFile STFile; + +struct STFInfo { + uint32_t magic; + uint32_t ftype; + uint32_t fver; + uint64_t fsize; +}; + +struct STFile { + STFInfo info; + STfsFile f; + TdFilePtr pFile; + uint8_t state; +}; + +#define TD_FILE_F(tf) (&((tf)->f)) +#define TD_FILE_PFILE(tf) ((tf)->pFile) +#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) +#define TD_FILE_FULL_NAME(tf) (TD_FILE_F(tf)->aname) +#define TD_FILE_REL_NAME(tf) (TD_FILE_F(tf)->rname) +#define TD_FILE_OPENED(tf) (TD_FILE_PFILE(tf) != NULL) +#define TD_FILE_CLOSED(tf) (!TD_FILE_OPENED(tf)) +#define TD_FILE_SET_CLOSED(f) (TD_FILE_PFILE(f) = NULL) +#define TD_FILE_STATE(tf) ((tf)->state) +#define TD_FILE_SET_STATE(tf, s) ((tf)->state = (s)) +#define TD_FILE_DID(tf) (TD_FILE_F(tf)->did) +#define TD_FILE_IS_OK(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_OK) +#define TD_FILE_IS_BAD(tf) (TD_FILE_STATE(tf) == TD_FILE_STATE_BAD) + +int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname); +int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType); +int32_t tdOpenTFile(STFile *pTFile, int flags); +int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte); +int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence); +int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte); +int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset); +int32_t tdRemoveTFile(STFile *pTFile); +int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo); +int32_t tdUpdateTFileHeader(STFile *pTFile); +void tdUpdateTFileMagic(STFile *pTFile, void *pCksm); +void tdCloseTFile(STFile *pTFile); +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName); + #ifdef __cplusplus } #endif diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 2ee0673ce5f04b34ad07cd4b8edc6cc74ca943ed..59c3e95b9cf1b3f207e53a767b1f6e8877bc7976 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -114,6 +114,7 @@ typedef struct { char subKey[TSDB_SUBSCRIBE_KEY_LEN]; int64_t consumerId; int32_t epoch; + int8_t fetchMeta; // reader SWalReadHandle* pWalReader; @@ -149,6 +150,7 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* // tqExec int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId); +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId); int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp); // tqMeta diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 9b1c2e456d024b2fdd1dafb2d3b6ad4da53aa348..32be479116c30e6e4023baa1bccaafdd1efdf22b 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -39,6 +39,7 @@ extern "C" { #define vGWarn(param, ...) do { if (vDebugFlag & DEBUG_WARN) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vWarn(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGInfo(param, ...) do { if (vDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vInfo(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define vGDebug(param, ...) do { if (vDebugFlag & DEBUG_DEBUG) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); vDebug(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) + // clang-format on // vnodeCfg.c @@ -77,6 +78,7 @@ void vnodeBufPoolReset(SVBufPool* pPool); int32_t vnodeQueryOpen(SVnode* pVnode); void vnodeQueryClose(SVnode* pVnode); int32_t vnodeGetTableMeta(SVnode* pVnode, SRpcMsg* pMsg); +int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index c0dfebb08f566e6a9e28d233c7dc7c8dd1a991ea..f7965f090240844230aeeef7a98e3b0e4cc6900e 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -149,6 +149,8 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg); int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg); SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pSchema, bool createTb, int64_t suid, const char* stbFullName, int32_t vgId); @@ -245,7 +247,6 @@ struct SVnode { struct STbUidStore { tb_uid_t suid; - tb_uid_t uid; // TODO: just for debugging, remove when uid provided in SSDataBlock SArray* tbUids; SHashObj* uidHash; }; diff --git a/source/dnode/vnode/src/meta/metaQuery.c b/source/dnode/vnode/src/meta/metaQuery.c index a5ca90e55fcba3e20c250e5ffa58d4e71cc5ca07..910f4bba518ddf3ea192f731eba998684f5626f0 100644 --- a/source/dnode/vnode/src/meta/metaQuery.c +++ b/source/dnode/vnode/src/meta/metaQuery.c @@ -219,11 +219,9 @@ _err: } int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList){ - metaRLock(pMeta); TBC * pCur; int ret = tdbTbcOpen(pMeta->pTtlIdx, &pCur, NULL); if (ret < 0) { - metaULock(pMeta); return ret; } @@ -249,6 +247,7 @@ int metaTtlSmaller(SMeta *pMeta, uint64_t ttl, SArray *uidList){ tdbTbcClose(pCur); tdbFree(pKey); + return 0; } @@ -265,8 +264,8 @@ struct SMCtbCursor { SMCtbCursor *metaOpenCtbCursor(SMeta *pMeta, tb_uid_t uid) { SMCtbCursor *pCtbCur = NULL; SCtbIdxKey ctbIdxKey; - int ret; - int c; + int ret = 0; + int c = 0; pCtbCur = (SMCtbCursor *)taosMemoryCalloc(1, sizeof(*pCtbCur)); if (pCtbCur == NULL) { @@ -613,9 +612,6 @@ const void *metaGetTableTagVal(SMetaEntry *pEntry, int16_t type, STagVal *val) { ASSERT(pEntry->type == TSDB_CHILD_TABLE); STag *tag = (STag *)pEntry->ctbEntry.pTags; if (type == TSDB_DATA_TYPE_JSON) { - if (tag->nTag == 0) { - return NULL; - } return tag; } bool find = tTagGet(tag, val); diff --git a/source/dnode/vnode/src/meta/metaTable.c b/source/dnode/vnode/src/meta/metaTable.c index bf5d5912f93df374010043c10ef0cdd48368573c..ab512f7774f066245c2837a19087d09471b98253 100644 --- a/source/dnode/vnode/src/meta/metaTable.c +++ b/source/dnode/vnode/src/meta/metaTable.c @@ -375,6 +375,7 @@ int metaTtlDropTable(SMeta *pMeta, int64_t ttl, SArray *tbUids) { metaWLock(pMeta); int ret = metaTtlSmaller(pMeta, ttl, tbUids); if(ret != 0){ + metaULock(pMeta); return ret; } for (int i = 0; i < taosArrayGetSize(tbUids); ++i) { @@ -400,8 +401,7 @@ static void metaBuildTtlIdxKey(STtlIdxKey *ttlKey, const SMetaEntry *pME){ if (ttlDays <= 0) return; - ttlKey->dtime = ctime / 1000 + ttlDays * 24 * 60 * 60; -// ttlKey->dtime = ctime / 1000 + ttlDays; + ttlKey->dtime = ctime / 1000 + ttlDays * tsTtlUnit; ttlKey->uid = pME->uid; } diff --git a/source/dnode/vnode/src/sma/smaEnv.c b/source/dnode/vnode/src/sma/smaEnv.c index a80af2b20255fa51d114d9568afadac9318765cf..4e907e93ca05d99fae85a91f97be15e195f927c1 100644 --- a/source/dnode/vnode/src/sma/smaEnv.c +++ b/source/dnode/vnode/src/sma/smaEnv.c @@ -21,9 +21,10 @@ typedef struct SSmaStat SSmaStat; // declaration of static functions -static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType); +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma); static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path); static int32_t tdInitSmaEnv(SSma *pSma, int8_t smaType, const char *path, SSmaEnv **pEnv); +static void *tdFreeTSmaStat(STSmaStat *pStat); // implementation @@ -45,7 +46,7 @@ static SSmaEnv *tdNewSmaEnv(const SSma *pSma, int8_t smaType, const char *path) return NULL; } - if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType) != TSDB_CODE_SUCCESS) { + if (tdInitSmaStat(&SMA_ENV_STAT(pEnv), smaType, pSma) != TSDB_CODE_SUCCESS) { tdFreeSmaEnv(pEnv); return NULL; } @@ -105,7 +106,7 @@ int32_t tdUnRefSmaStat(SSma *pSma, SSmaStat *pStat) { return 0; } -static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { +static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType, const SSma *pSma) { ASSERT(pSmaStat != NULL); if (*pSmaStat) { // no lock @@ -125,10 +126,23 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { } if (smaType == TSDB_SMA_TYPE_ROLLUP) { - SMA_STAT_INFO_HASH(*pSmaStat) = taosHashInit( - RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + SMA_RSMA_STAT(*pSmaStat)->pSma = (SSma*)pSma; + // init timer + SMA_RSMA_TMR_HANDLE(*pSmaStat) = taosTmrInit(10000, 100, 10000, "RSMA_G"); + if (!SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + taosMemoryFreeClear(*pSmaStat); + return TSDB_CODE_FAILED; + } + + atomic_store_8(&SMA_RSMA_TMR_STAT(*pSmaStat), TASK_TRIGGER_STATUS__ACTIVE); - if (!SMA_STAT_INFO_HASH(*pSmaStat)) { + // init hash + SMA_RSMA_INFO_HASH(*pSmaStat) = taosHashInit( + RSMA_TASK_INFO_HASH_SLOT, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); + if (!SMA_RSMA_INFO_HASH(*pSmaStat)) { + if (SMA_RSMA_TMR_HANDLE(*pSmaStat)) { + taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(*pSmaStat)); + } taosMemoryFreeClear(*pSmaStat); return TSDB_CODE_FAILED; } @@ -141,16 +155,16 @@ static int32_t tdInitSmaStat(SSmaStat **pSmaStat, int8_t smaType) { return TSDB_CODE_SUCCESS; } -void *tdFreeSmaStatItem(SSmaStatItem *pSmaStatItem) { - if (pSmaStatItem) { - tDestroyTSma(pSmaStatItem->pTSma); - taosMemoryFreeClear(pSmaStatItem->pTSma); - taosMemoryFreeClear(pSmaStatItem); +static void *tdFreeTSmaStat(STSmaStat *pStat) { + if (pStat) { + tDestroyTSma(pStat->pTSma); + taosMemoryFreeClear(pStat->pTSma); + taosMemoryFreeClear(pStat); } return NULL; } -void* tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { +void *tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { tdDestroySmaState(pSmaStat, smaType); taosMemoryFreeClear(pSmaStat); return NULL; @@ -165,16 +179,19 @@ void* tdFreeSmaState(SSmaStat *pSmaStat, int8_t smaType) { int32_t tdDestroySmaState(SSmaStat *pSmaStat, int8_t smaType) { if (pSmaStat) { if (smaType == TSDB_SMA_TYPE_TIME_RANGE) { - tdFreeSmaStatItem(&pSmaStat->tsmaStatItem); + tdFreeTSmaStat(&pSmaStat->tsmaStat); } else if (smaType == TSDB_SMA_TYPE_ROLLUP) { + if (SMA_RSMA_TMR_HANDLE(pSmaStat)) { + taosTmrCleanUp(SMA_RSMA_TMR_HANDLE(pSmaStat)); + } // TODO: use taosHashSetFreeFp when taosHashSetFreeFp is ready. - void *infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), NULL); + void *infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), NULL); while (infoHash) { SRSmaInfo *pInfoHash = *(SRSmaInfo **)infoHash; tdFreeRSmaInfo(pInfoHash); - infoHash = taosHashIterate(SMA_STAT_INFO_HASH(pSmaStat), infoHash); + infoHash = taosHashIterate(SMA_RSMA_INFO_HASH(pSmaStat), infoHash); } - taosHashCleanup(SMA_STAT_INFO_HASH(pSmaStat)); + taosHashCleanup(SMA_RSMA_INFO_HASH(pSmaStat)); } else { ASSERT(0); } @@ -273,4 +290,4 @@ void smaTimerCleanUp(void *timer, int8_t *initFlag) { taosTmrCleanUp(timer); atomic_store_8(initFlag, 0); } -} +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaOpen.c b/source/dnode/vnode/src/sma/smaOpen.c index a1c47a96c0d2d12a6652dc66019a109fc698a5af..383c264e7b755870708955b96cfbb88006c85184 100644 --- a/source/dnode/vnode/src/sma/smaOpen.c +++ b/source/dnode/vnode/src/sma/smaOpen.c @@ -137,4 +137,17 @@ int32_t smaClose(SSma *pSma) { taosMemoryFreeClear(pSma); } return 0; +} + +/** + * @brief rsma env restore + * + * @param pSma + * @return int32_t + */ +int32_t smaRestore(SSma *pSma) { + if (!pSma) return 0; + // iterate all stables to restore the rsma env + + return TSDB_CODE_SUCCESS; } \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index 1f18f6cb8777452dde659bbdec8a22d9f84e30de..3c2f710fa720fc91c1d94309bad2d27d5da68ca4 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -16,35 +16,17 @@ #include "sma.h" #include "tstream.h" -static FORCE_INLINE int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); -static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); -static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, - tb_uid_t suid, int8_t level); - -#define SET_RSMA_INFO_ITEM_PARAMS(__idx, __level) \ - if (param->qmsg[__idx]) { \ - pRSmaInfo->items[__idx].pRsmaInfo = pRSmaInfo; \ - pRSmaInfo->items[__idx].taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], &handle); \ - if (!pRSmaInfo->items[__idx].taskInfo) { \ - goto _err; \ - } \ - pRSmaInfo->items[__idx].triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; \ - if (param->maxdelay[__idx] < 1) { \ - int64_t msInterval = \ - convertTimeFromPrecisionToUnit(pRetention[__level].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); \ - pRSmaInfo->items[__idx].maxDelay = msInterval; \ - } else { \ - pRSmaInfo->items[__idx].maxDelay = param->maxdelay[__idx]; \ - } \ - if (pRSmaInfo->items[__idx].maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { \ - pRSmaInfo->items[__idx].maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; \ - } \ - pRSmaInfo->items[__idx].level = TSDB_RETENTION_L##__level; \ - pRSmaInfo->items[__idx].tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); \ - if (!pRSmaInfo->items[__idx].tmrHandle) { \ - goto _err; \ - } \ - } +typedef enum { TD_QTASK_TMP_FILE = 0, TD_QTASK_CUR_FILE } TD_QTASK_FILE_T; +static const char *tdQTaskInfoFname[] = {"qtaskinfo.t", "qtaskinfo"}; + +static int32_t tdUidStorePut(STbUidStore *pStore, tb_uid_t suid, tb_uid_t *uid); +static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids); +static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *handle, + int8_t idx); +static int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *rsmaItem, + tb_uid_t suid, int8_t level); +static void tdRSmaFetchTrigger(void *param, void *tmrId); +static void tdRSmaPersistTrigger(void *param, void *tmrId); struct SRSmaInfoItem { SRSmaInfo *pRsmaInfo; @@ -56,14 +38,6 @@ struct SRSmaInfoItem { int8_t triggerStatus; // TASK_TRIGGER_STATUS__IN_ACTIVE/TASK_TRIGGER_STATUS__ACTIVE int32_t maxDelay; }; - -typedef struct { - int64_t suid; - SRSmaInfoItem *pItem; - SSma *pSma; - STSchema *pTSchema; -} SRSmaTriggerParam; - struct SRSmaInfo { STSchema *pTSchema; SSma *pSma; @@ -81,7 +55,7 @@ static FORCE_INLINE void tdFreeTaskHandle(qTaskInfo_t *taskHandle) { void *tdFreeRSmaInfo(SRSmaInfo *pInfo) { if (pInfo) { - for (int32_t i = 0; i < TSDB_RETENTION_MAX; ++i) { + for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { SRSmaInfoItem *pItem = &pInfo->items[i]; if (pItem->taskInfo) { tdFreeTaskHandle(pItem->taskInfo); @@ -118,7 +92,7 @@ static FORCE_INLINE int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SA return TSDB_CODE_FAILED; } - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaError("vgId:%d, failed to get rsma info for uid:%" PRIi64, SMA_VID(pSma), *suid); terrno = TSDB_CODE_RSMA_INVALID_STAT; @@ -187,7 +161,7 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui SSmaStat *pStat = SMA_ENV_STAT(pEnv); SHashObj *infoHash = NULL; - if (!pStat || !(infoHash = SMA_STAT_INFO_HASH(pStat))) { + if (!pStat || !(infoHash = SMA_RSMA_INFO_HASH(pStat))) { terrno = TSDB_CODE_RSMA_INVALID_STAT; return TSDB_CODE_FAILED; } @@ -213,6 +187,40 @@ int32_t tdFetchTbUidList(SSma *pSma, STbUidStore **ppStore, tb_uid_t suid, tb_ui return TSDB_CODE_SUCCESS; } +static int32_t tdSetRSmaInfoItemParams(SSma *pSma, SRSmaParam *param, SRSmaInfo *pRSmaInfo, SReadHandle *pReadHandle, + int8_t idx) { + SRetention *pRetention = SMA_RETENTION(pSma); + STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); + + if (param->qmsg[idx]) { + SRSmaInfoItem *pItem = &(pRSmaInfo->items[idx]); + pItem->pRsmaInfo = pRSmaInfo; + pItem->taskInfo = qCreateStreamExecTaskInfo(param->qmsg[0], pReadHandle); + if (!pItem->taskInfo) { + goto _err; + } + pItem->triggerStatus = TASK_TRIGGER_STATUS__IN_ACTIVE; + if (param->maxdelay[idx] < TSDB_MIN_ROLLUP_MAX_DELAY) { + int64_t msInterval = + convertTimeFromPrecisionToUnit(pRetention[idx + 1].freq, pTsdbCfg->precision, TIME_UNIT_MILLISECOND); + pItem->maxDelay = (int32_t)msInterval; + } else { + pItem->maxDelay = (int32_t)param->maxdelay[idx]; + } + if (pItem->maxDelay > TSDB_MAX_ROLLUP_MAX_DELAY) { + pItem->maxDelay = TSDB_MAX_ROLLUP_MAX_DELAY; + } + pItem->level = (idx == 0 ? TSDB_RETENTION_L1 : TSDB_RETENTION_L2); + pItem->tmrHandle = taosTmrInit(10000, 100, 10000, "RSMA"); + if (!pItem->tmrHandle) { + goto _err; + } + } + return TSDB_CODE_SUCCESS; +_err: + return TSDB_CODE_FAILED; +} + /** * @brief Check and init qTaskInfo_t, only applicable to stable with SRSmaParam. * @@ -246,7 +254,7 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t)); if (pRSmaInfo) { ASSERT(0); // TODO: free original pRSmaInfo is exists abnormally smaWarn("vgId:%d, rsma info already exists for stb: %s, %" PRIi64, SMA_VID(pSma), pReq->name, pReq->suid); @@ -282,14 +290,14 @@ int32_t tdProcessRSmaCreate(SVnode *pVnode, SVCreateStbReq *pReq) { pRSmaInfo->pSma = pSma; pRSmaInfo->suid = pReq->suid; - SRetention *pRetention = SMA_RETENTION(pSma); - STsdbCfg *pTsdbCfg = SMA_TSDB_CFG(pSma); - - SET_RSMA_INFO_ITEM_PARAMS(0, 1); - SET_RSMA_INFO_ITEM_PARAMS(1, 2); + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 0) < 0) { + goto _err; + } + if (tdSetRSmaInfoItemParams(pSma, param, pRSmaInfo, &handle, 1) < 0) { + goto _err; + } - if (taosHashPut(SMA_STAT_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) != - TSDB_CODE_SUCCESS) { + if (taosHashPut(SMA_RSMA_INFO_HASH(pStat), &pReq->suid, sizeof(tb_uid_t), &pRSmaInfo, sizeof(pRSmaInfo)) < 0) { goto _err; } else { smaDebug("vgId:%d, register rsma info succeed for suid:%" PRIi64, SMA_VID(pSma), pReq->suid); @@ -418,7 +426,6 @@ static int32_t tdFetchSubmitReqSuids(SSubmitReq *pMsg, STbUidStore *pStore) { if (!pBlock) break; tdUidStorePut(pStore, msgIter.suid, NULL); - pStore->uid = msgIter.uid; // TODO: remove, just for debugging } if (terrno != TSDB_CODE_SUCCESS) return -1; @@ -439,8 +446,9 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) if (!output) { break; } + if (!pResult) { - pResult = taosArrayInit(0, sizeof(SSDataBlock)); + pResult = taosArrayInit(1, sizeof(SSDataBlock)); if (!pResult) { terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_FAILED; @@ -451,7 +459,7 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } if (taosArrayGetSize(pResult) > 0) { -#if 0 +#if 1 char flag[10] = {0}; snprintf(flag, 10, "level %" PRIi8, pItem->level); blockDebugShowData(pResult, flag); @@ -459,14 +467,12 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) STsdb *sinkTsdb = (pItem->level == TSDB_RETENTION_L1 ? pSma->pRSmaTsdb1 : pSma->pRSmaTsdb2); SSubmitReq *pReq = NULL; if (buildSubmitReqFromDataBlock(&pReq, pResult, pRSmaInfo->pTSchema, SMA_VID(pSma), pRSmaInfo->suid) < 0) { - taosArrayDestroy(pResult); - return TSDB_CODE_FAILED; + goto _err; } if (pReq && tdProcessSubmitReq(sinkTsdb, INT64_MAX, pReq) < 0) { - taosArrayDestroy(pResult); taosMemoryFreeClear(pReq); - return TSDB_CODE_FAILED; + goto _err; } taosMemoryFreeClear(pReq); @@ -479,7 +485,10 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) } taosArrayDestroy(pResult); - return 0; + return TSDB_CODE_SUCCESS; +_err: + taosArrayDestroy(pResult); + return TSDB_CODE_FAILED; } /** @@ -488,13 +497,12 @@ static int32_t tdFetchAndSubmitRSmaResult(SRSmaInfoItem *pItem, int8_t blkType) * @param param * @param tmrId */ -static void rsmaTriggerByTimer(void *param, void *tmrId) { - // SRSmaTriggerParam *pParam = (SRSmaTriggerParam *)param; - // SRSmaInfoItem *pItem = pParam->pItem; +static void tdRSmaFetchTrigger(void *param, void *tmrId) { SRSmaInfoItem *pItem = param; if (atomic_load_8(&pItem->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { - smaTrace("level %" PRIi8 " status is active for tb suid:%" PRIi64, pItem->level, pItem->pRsmaInfo->suid); + smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is active for tb suid:%" PRIi64, __func__, __LINE__, + taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); SSDataBlock dataBlock = {.info.type = STREAM_GET_ALL}; atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__IN_ACTIVE); @@ -502,10 +510,11 @@ static void rsmaTriggerByTimer(void *param, void *tmrId) { tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SSDATA_BLOCK); } else { - smaTrace("level %" PRIi8 " status is inactive for tb suid:%" PRIi64, pItem->level, pItem->pRsmaInfo->suid); + smaWarn("%s:%d THREAD:%" PRIi64 " level %" PRIi8 " status is inactive for tb suid:%" PRIi64, __func__, __LINE__, + taosGetSelfPthreadId(), pItem->level, pItem->pRsmaInfo->suid); } - // taosTmrReset(rsmaTriggerByTimer, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + // taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); } static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int32_t inputType, SRSmaInfoItem *pItem, @@ -518,16 +527,20 @@ static FORCE_INLINE int32_t tdExecuteRSmaImpl(SSma *pSma, const void *pMsg, int3 smaDebug("vgId:%d, execute rsma %" PRIi8 " task for qTaskInfo:%p suid:%" PRIu64, SMA_VID(pSma), level, pItem->taskInfo, suid); - // inputType = STREAM_DATA_TYPE_SUBMIT_BLOCK(1) - if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { + if (qSetStreamInput(pItem->taskInfo, pMsg, inputType, true) < 0) { // STREAM_DATA_TYPE_SUBMIT_BLOCK smaError("vgId:%d, rsma % " PRIi8 " qSetStreamInput failed since %s", SMA_VID(pSma), level, tstrerror(terrno)); return TSDB_CODE_FAILED; } - // SRSmaTriggerParam triggerParam = {.suid = suid, .pItem = pItem, .pSma = pSma, .pTSchema = pTSchema}; tdFetchAndSubmitRSmaResult(pItem, STREAM_DATA_TYPE_SUBMIT_BLOCK); atomic_store_8(&pItem->triggerStatus, TASK_TRIGGER_STATUS__ACTIVE); - taosTmrReset(rsmaTriggerByTimer, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); + smaWarn("%s:%d THREAD:%" PRIi64 " process rsma insert", __func__, __LINE__, taosGetSelfPthreadId()); + + SSmaEnv *pEnv = SMA_RSMA_ENV(pSma); + SRSmaStat *pStat = SMA_RSMA_STAT(pEnv->pStat); + + taosTmrStart(tdRSmaPersistTrigger, 5000, pStat, pStat->tmrHandle); + taosTmrReset(tdRSmaFetchTrigger, pItem->maxDelay, pItem, pItem->tmrHandle, &pItem->tmrId); return TSDB_CODE_SUCCESS; } @@ -542,7 +555,7 @@ static int32_t tdExecuteRSma(SSma *pSma, const void *pMsg, int32_t inputType, tb SSmaStat *pStat = SMA_ENV_STAT(pEnv); SRSmaInfo *pRSmaInfo = NULL; - pRSmaInfo = taosHashGet(SMA_STAT_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); + pRSmaInfo = taosHashGet(SMA_RSMA_INFO_HASH(pStat), &suid, sizeof(tb_uid_t)); if (!pRSmaInfo || !(pRSmaInfo = *(SRSmaInfo **)pRSmaInfo)) { smaDebug("vgId:%d, return as no rsma info for suid:%" PRIu64, SMA_VID(pSma), suid); @@ -594,3 +607,106 @@ int32_t tdProcessRSmaSubmit(SSma *pSma, void *pMsg, int32_t inputType) { } return TSDB_CODE_SUCCESS; } + +void tdRSmaQTaskGetFName(int32_t vid, int8_t ftype, char* outputName) { + tdGetVndFileName(vid, "rsma", tdQTaskInfoFname[ftype], outputName); +} + +static void *tdRSmaPersistExec(void *param) { + setThreadName("rsma-task-persist"); + SRSmaStat *pRSmaStat = param; + SSma *pSma = pRSmaStat->pSma; + STfs *pTfs = pSma->pVnode->pTfs; + int64_t toffset = 0; + + void *infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), NULL); + if (!infoHash) { + goto _end; + } + + STFile tFile = {0}; + int32_t vid = 2; + char qTaskInfoFName[TSDB_FILENAME_LEN]; + tdRSmaQTaskGetFName(vid, TD_QTASK_TMP_FILE, qTaskInfoFName); + tdInitTFile(&tFile, pTfs, qTaskInfoFName); + tdCreateTFile(&tFile, pTfs, true, -1); + + while (infoHash) { + SRSmaInfo *pRSmaInfo = *(SRSmaInfo **)infoHash; + char *pOutput = NULL; + int32_t len = 0; + if (qSerializeTaskStatus(pRSmaInfo->items[0].taskInfo, &pOutput, &len) < 0) { + smaError("serialize rsma task for table %" PRIi64 " failed since %s", pRSmaInfo->items[0].pRsmaInfo->suid, + terrstr(terrno)); + } else { + smaWarn("serialize rsma task for table %" PRIi64 " success and len is %d", pRSmaInfo->items[0].pRsmaInfo->suid, + len); + } + tdAppendTFile(&tFile, &len, sizeof(len), &toffset); + tdAppendTFile(&tFile, pOutput, len, &toffset); + + taosMemoryFree(pOutput); + infoHash = taosHashIterate(RSMA_INFO_HASH(pRSmaStat), infoHash); + } +_end: + + if (tdUpdateTFileHeader(&tFile) < 0) { + smaError("vgId:%d, failed to update tfile %s header since %s", vid, TD_FILE_FULL_NAME(&tFile), tstrerror(terrno)); + tdCloseTFile(&tFile); + tdRemoveTFile(&tFile); + return NULL; + } + + tdCloseTFile(&tFile); + + char newFName[TSDB_FILENAME_LEN]; + strncpy(newFName, TD_FILE_FULL_NAME(&tFile), TSDB_FILENAME_LEN); + char *pos = strstr(newFName, tdQTaskInfoFname[TD_QTASK_TMP_FILE]); + strncpy(pos, tdQTaskInfoFname[TD_QTASK_CUR_FILE], TSDB_FILENAME_LEN - POINTER_DISTANCE(pos, newFName)); + taosRenameFile(TD_FILE_FULL_NAME(&tFile), newFName); + + atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); + return NULL; +_err: + atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__ACTIVE); + // remove the .tmp file + return NULL; +} + +static void tdRSmaPersistTask(SRSmaStat *pRSmaStat) { + smaWarn("%s:%d entry ", __func__, __LINE__); + TdThread threadId; + TdThreadAttr thAttr; + taosThreadAttrInit(&thAttr); + taosThreadAttrSetDetachState(&thAttr, PTHREAD_CREATE_DETACHED); + + if (taosThreadCreate(&threadId, &thAttr, tdRSmaPersistExec, pRSmaStat) != 0) { + smaError("failed to create thread to persist rsma qTaskInfo since %s", strerror(errno)); + } + + taosThreadAttrDestroy(&thAttr); + smaWarn("%s:%d end ", __func__, __LINE__); +} + +/** + * @brief trigger to persist rsma qTaskInfo + * + * @param param + * @param tmrId + */ +static void tdRSmaPersistTrigger(void *param, void *tmrId) { + SRSmaStat *pRSmaStat = param; + + if (atomic_load_8(&pRSmaStat->tmrStat) == TASK_TRIGGER_STATUS__ACTIVE) { + smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence start since active", __func__, __LINE__, taosGetSelfPthreadId()); + atomic_store_8(&pRSmaStat->tmrStat, TASK_TRIGGER_STATUS__IN_ACTIVE); + + // execution + tdRSmaPersistTask(pRSmaStat); + } else { + smaWarn("%s:%d THREAD:%" PRIi64 " rsma persistence not start since inactive", __func__, __LINE__, + taosGetSelfPthreadId()); + } + + taosTmrReset(tdRSmaPersistTrigger, 3600000, pRSmaStat, pRSmaStat->tmrHandle, &pRSmaStat->tmrId); +} \ No newline at end of file diff --git a/source/dnode/vnode/src/sma/smaTimeRange.c b/source/dnode/vnode/src/sma/smaTimeRange.c index 4352c466c5526cc4a00f10c629e2e99b1015cf7d..2244b91c28136b7d4f076259888ef8906fab011b 100644 --- a/source/dnode/vnode/src/sma/smaTimeRange.c +++ b/source/dnode/vnode/src/sma/smaTimeRange.c @@ -129,7 +129,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { SSmaEnv *pEnv = SMA_TSMA_ENV(pSma); SSmaStat *pStat = NULL; - SSmaStatItem *pItem = NULL; + STSmaStat *pItem = NULL; if (!pEnv || !(pStat = SMA_ENV_STAT(pEnv))) { terrno = TSDB_CODE_TSMA_INVALID_STAT; @@ -137,7 +137,7 @@ int32_t tdProcessTSmaInsertImpl(SSma *pSma, int64_t indexUid, const char *msg) { } tdRefSmaStat(pSma, pStat); - pItem = &pStat->tsmaStatItem; + pItem = &pStat->tsmaStat; ASSERT(pItem); diff --git a/source/dnode/vnode/src/sma/smaUtil.c b/source/dnode/vnode/src/sma/smaUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..76690a3bde0215fcc8ffc16a992abad8375ceb11 --- /dev/null +++ b/source/dnode/vnode/src/sma/smaUtil.c @@ -0,0 +1,238 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "sma.h" + +#define TD_FILE_HEAD_SIZE 512 + +#define TD_FILE_STATE_OK 0 +#define TD_FILE_STATE_BAD 1 + +#define TD_FILE_INIT_MAGIC 0xFFFFFFFF + + +static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo); +static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo); + +static int32_t tdEncodeTFInfo(void **buf, STFInfo *pInfo) { + int32_t tlen = 0; + + tlen += taosEncodeFixedU32(buf, pInfo->magic); + tlen += taosEncodeFixedU32(buf, pInfo->ftype); + tlen += taosEncodeFixedU32(buf, pInfo->fver); + tlen += taosEncodeFixedU64(buf, pInfo->fsize); + + return tlen; +} + +static void *tdDecodeTFInfo(void *buf, STFInfo *pInfo) { + buf = taosDecodeFixedU32(buf, &(pInfo->magic)); + buf = taosDecodeFixedU32(buf, &(pInfo->ftype)); + buf = taosDecodeFixedU32(buf, &(pInfo->fver)); + buf = taosDecodeFixedU64(buf, &(pInfo->fsize)); + return buf; +} + +int64_t tdWriteTFile(STFile *pTFile, void *buf, int64_t nbyte) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t nwrite = taosWriteFile(pTFile->pFile, buf, nbyte); + if (nwrite < nbyte) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nwrite; +} + +int64_t tdSeekTFile(STFile *pTFile, int64_t offset, int whence) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t loffset = taosLSeekFile(TD_FILE_PFILE(pTFile), offset, whence); + if (loffset < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return loffset; +} + +int64_t tdReadTFile(STFile *pTFile, void *buf, int64_t nbyte) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t nread = taosReadFile(pTFile->pFile, buf, nbyte); + if (nread < 0) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return nread; +} + +int32_t tdUpdateTFileHeader(STFile *pTFile) { + char buf[TD_FILE_HEAD_SIZE] = "\0"; + + if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { + return -1; + } + + void *ptr = buf; + tdEncodeTFInfo(&ptr, &(pTFile->info)); + + taosCalcChecksumAppend(0, (uint8_t *)buf, TD_FILE_HEAD_SIZE); + if (tdWriteTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { + return -1; + } + + return 0; +} + +int32_t tdLoadTFileHeader(STFile *pTFile, STFInfo *pInfo) { + char buf[TD_FILE_HEAD_SIZE] = "\0"; + uint32_t _version; + + ASSERT(TD_FILE_OPENED(pTFile)); + + if (tdSeekTFile(pTFile, 0, SEEK_SET) < 0) { + return -1; + } + + if (tdReadTFile(pTFile, buf, TD_FILE_HEAD_SIZE) < 0) { + return -1; + } + + if (!taosCheckChecksumWhole((uint8_t *)buf, TD_FILE_HEAD_SIZE)) { + terrno = TSDB_CODE_FILE_CORRUPTED; + return -1; + } + + void *pBuf = buf; + pBuf = tdDecodeTFInfo(pBuf, pInfo); + return 0; +} + +void tdUpdateTFileMagic(STFile *pTFile, void *pCksm) { + pTFile->info.magic = taosCalcChecksum(pTFile->info.magic, (uint8_t *)(pCksm), sizeof(TSCKSUM)); +} + +int64_t tdAppendTFile(STFile *pTFile, void *buf, int64_t nbyte, int64_t *offset) { + ASSERT(TD_FILE_OPENED(pTFile)); + + int64_t toffset; + + if ((toffset = tdSeekTFile(pTFile, 0, SEEK_END)) < 0) { + return -1; + } + + ASSERT(pTFile->info.fsize == toffset); + + if (offset) { + *offset = toffset; + } + + if (tdWriteTFile(pTFile, buf, nbyte) < 0) { + return -1; + } + + pTFile->info.fsize += nbyte; + + return nbyte; +} + +int32_t tdOpenTFile(STFile *pTFile, int flags) { + ASSERT(!TD_FILE_OPENED(pTFile)); + + pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), flags); + if (pTFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + + return 0; +} + +void tdCloseTFile(STFile *pTFile) { + if (TD_FILE_OPENED(pTFile)) { + taosCloseFile(&pTFile->pFile); + TD_FILE_SET_CLOSED(pTFile); + } +} + +void tdGetVndFileName(int32_t vid, const char *dname, const char *fname, char *outputName) { + snprintf(outputName, TSDB_FILENAME_LEN, "vnode/vnode%d/%s/%s", vid, dname, fname); +} + +int32_t tdInitTFile(STFile *pTFile, STfs *pTfs, const char *fname) { + char fullname[TSDB_FILENAME_LEN]; + SDiskID did = {0}; + + TD_FILE_SET_STATE(pTFile, TD_FILE_STATE_OK); + TD_FILE_SET_CLOSED(pTFile); + + memset(&(pTFile->info), 0, sizeof(pTFile->info)); + pTFile->info.magic = TD_FILE_INIT_MAGIC; + + if (tfsAllocDisk(pTfs, 0, &did) < 0) { + terrno = TSDB_CODE_NO_AVAIL_DISK; + return -1; + } + + tfsInitFile(pTfs, &(pTFile->f), did, fname); + + return 0; +} + +int32_t tdCreateTFile(STFile *pTFile, STfs *pTfs, bool updateHeader, int8_t fType) { + ASSERT(pTFile->info.fsize == 0 && pTFile->info.magic == TD_FILE_INIT_MAGIC); + + pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pTFile->pFile == NULL) { + if (errno == ENOENT) { + // Try to create directory recursively + char *s = strdup(TD_FILE_REL_NAME(pTFile)); + if (tfsMkdirRecurAt(pTfs, taosDirName(s), TD_FILE_DID(pTFile)) < 0) { + taosMemoryFreeClear(s); + return -1; + } + taosMemoryFreeClear(s); + + pTFile->pFile = taosOpenFile(TD_FILE_FULL_NAME(pTFile), TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC); + if (pTFile->pFile == NULL) { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } else { + terrno = TAOS_SYSTEM_ERROR(errno); + return -1; + } + } + + if (!updateHeader) { + return 0; + } + + pTFile->info.fsize += TD_FILE_HEAD_SIZE; + pTFile->info.fver = 0; + + if (tdUpdateTFileHeader(pTFile) < 0) { + tdCloseTFile(pTFile); + tdRemoveTFile(pTFile); + return -1; + } + + return 0; +} + +int32_t tdRemoveTFile(STFile *pTFile) { return tfsRemoveFile(TD_FILE_F(pTFile)); } \ No newline at end of file diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index ece4b7e2a45685bf1106d10e5fe00350d7183c55..cca3a5858814e54d32fb8965326b3786e8f3f642 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -85,6 +85,34 @@ void tqClose(STQ* pTq) { } } +int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) { + int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqMetaRsp(NULL, pRsp); + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP; + ((SMqRspHead*)buf)->epoch = pReq->epoch; + ((SMqRspHead*)buf)->consumerId = pReq->consumerId; + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + tEncodeSMqMetaRsp(&abuf, pRsp); + + SRpcMsg resp = { + .info = pMsg->info, + .pCont = buf, + .contLen = tlen, + .code = 0, + }; + tmsgSendRsp(&resp); + + tqDebug("vg %d from consumer %ld (epoch %d) send rsp, res msg type %d, reqOffset: %ld, rspOffset: %ld", + TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->reqOffset, pRsp->rspOffset); + + return 0; +} + int32_t tqSendPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataBlkRsp* pRsp) { int32_t tlen = sizeof(SMqRspHead) + tEncodeSMqDataBlkRsp(NULL, pRsp); void* buf = rpcMallocCont(tlen); @@ -199,19 +227,16 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { consumerEpoch = atomic_val_compare_exchange_32(&pHandle->epoch, consumerEpoch, reqEpoch); } - SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); - if (pHeadWithCkSum == NULL) { - return -1; - } - - walSetReaderCapacity(pHandle->pWalReader, 2048); - SMqDataBlkRsp rsp = {0}; rsp.reqOffset = pReq->currentOffset; rsp.blockData = taosArrayInit(0, sizeof(void*)); rsp.blockDataLen = taosArrayInit(0, sizeof(int32_t)); + if (rsp.blockData == NULL || rsp.blockDataLen == NULL) { + return -1; + } + rsp.withTbName = pReq->withTbName; if (rsp.withTbName) { rsp.blockTbName = taosArrayInit(0, sizeof(void*)); @@ -225,6 +250,32 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { rsp.blockSchema = taosArrayInit(0, sizeof(void*)); } +#if 1 + if (pReq->useSnapshot) { + // TODO set ver into snapshot + int64_t lastVer = walGetCommittedVer(pTq->pWal); + if (rsp.reqOffset < lastVer) { + tqInfo("retrieve using snapshot req offset %ld last ver %ld", rsp.reqOffset, lastVer); + tqScanSnapshot(pTq, &pHandle->execHandle, &rsp, workerId); + + if (rsp.blockNum != 0) { + rsp.withTbName = false; + rsp.rspOffset = lastVer; + tqInfo("direct send by snapshot req offset %ld rsp offset %ld", rsp.reqOffset, rsp.rspOffset); + fetchOffset = lastVer; + goto SEND_RSP; + } + } + } +#endif + + SWalHead* pHeadWithCkSum = taosMemoryMalloc(sizeof(SWalHead) + 2048); + if (pHeadWithCkSum == NULL) { + return -1; + } + + walSetReaderCapacity(pHandle->pWalReader, 2048); + while (1) { consumerEpoch = atomic_load_32(&pHandle->epoch); if (consumerEpoch > reqEpoch) { @@ -250,8 +301,24 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { /*ASSERT(0);*/ } } else { - // TODO - ASSERT(0); + ASSERT(pHandle->fetchMeta); + ASSERT(pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB || + pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || + pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || + pHead->msgType == TDMT_VND_DROP_TTL_TABLE); + tqInfo("fetch meta msg, ver: %ld, type: %d", pHead->version, pHead->msgType); + SMqMetaRsp metaRsp = {0}; + metaRsp.reqOffset = pReq->currentOffset; + metaRsp.rspOffset = fetchOffset; + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + if (tqSendMetaPollRsp(pTq, pMsg, pReq, &metaRsp) < 0) { + code = -1; + goto OVER; + } + code = 0; + goto OVER; } // TODO batch optimization: @@ -265,6 +332,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { taosMemoryFree(pHeadWithCkSum); +SEND_RSP: ASSERT(taosArrayGetSize(rsp.blockData) == rsp.blockNum); ASSERT(taosArrayGetSize(rsp.blockDataLen) == rsp.blockNum); if (rsp.withSchema) { @@ -276,7 +344,7 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg, int32_t workerId) { if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) { code = -1; } - +OVER: // TODO wrap in destroy func taosArrayDestroy(rsp.blockDataLen); taosArrayDestroyP(rsp.blockData, (FDelete)taosMemoryFree); @@ -321,6 +389,7 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { pHandle->epoch = -1; pHandle->execHandle.subType = req.subType; + pHandle->fetchMeta = req.withMeta; pHandle->pWalReader = walOpenReadHandle(pTq->pVnode->pWal); for (int32_t i = 0; i < 5; i++) { @@ -333,6 +402,8 @@ int32_t tqProcessVgChangeReq(STQ* pTq, char* msg, int32_t msgLen) { SReadHandle handle = { .reader = pHandle->execHandle.pExecReader[i], .meta = pTq->pVnode->pMeta, + .vnode = pTq->pVnode, + .initTsdbReader = 1, }; pHandle->execHandle.execCol.task[i] = qCreateStreamExecTaskInfo(pHandle->execHandle.execCol.qmsg, &handle); ASSERT(pHandle->execHandle.execCol.task[i]); @@ -383,8 +454,9 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { ASSERT(0); } tDecoderClear(&decoder); + ASSERT(pTask->isDataScan == 0 || pTask->isDataScan == 1); - pTask->status = TASK_STATUS__IDLE; + pTask->execStatus = TASK_EXEC_STATUS__IDLE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -398,12 +470,13 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { // exec if (pTask->execType != TASK_EXEC__NONE) { // expand runners - if (pTask->dataScan) { + if (pTask->isDataScan) { STqReadHandle* pStreamReader = tqInitSubmitMsgScanner(pTq->pVnode->pMeta); SReadHandle handle = { .reader = pStreamReader, .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, + .initTsdbReader = 1, }; /*pTask->exec.inputHandle = pStreamReader;*/ pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle); @@ -433,7 +506,7 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, char* msg, int32_t msgLen) { streamSetupTrigger(pTask); - tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->childId, TD_VID(pTq->pVnode)); + tqInfo("deploy stream task id %d child id %d on vg %d", pTask->taskId, pTask->selfChildId, TD_VID(pTq->pVnode)); taosHashPut(pTq->pStreamTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); @@ -459,7 +532,10 @@ int32_t tqProcessStreamTrigger(STQ* pTq, SSubmitReq* pReq) { pIter = taosHashIterate(pTq->pStreamTasks, pIter); if (pIter == NULL) break; SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->inputType != STREAM_INPUT__DATA_SUBMIT) continue; + if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + continue; + } + if (!pTask->isDataScan) continue; if (!failed) { if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { @@ -487,6 +563,9 @@ int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } streamProcessRunReq(pTask); return 0; } @@ -501,9 +580,12 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg) { tDecodeStreamDispatchReq(&decoder, &req); int32_t taskId = req.taskId; SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } + SRpcMsg rsp = { + .info = pMsg->info, + .code = 0, }; streamProcessDispatchReq(pTask, &req, &rsp); return 0; @@ -513,6 +595,9 @@ int32_t tqProcessTaskRecoverReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverReq* pReq = pMsg->pCont; int32_t taskId = pReq->taskId; SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } streamProcessRecoverReq(pTask, pReq, pMsg); return 0; } @@ -521,6 +606,9 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = pRsp->taskId; SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } streamProcessDispatchRsp(pTask, pRsp); return 0; } @@ -529,16 +617,58 @@ int32_t tqProcessTaskRecoverRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRecoverRsp* pRsp = pMsg->pCont; int32_t taskId = pRsp->taskId; SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } streamProcessRecoverRsp(pTask, pRsp); return 0; } int32_t tqProcessTaskDropReq(STQ* pTq, char* msg, int32_t msgLen) { SVDropStreamTaskReq* pReq = (SVDropStreamTaskReq*)msg; + + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); + atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); + // todo + // clear queue + // push drop req into queue + // launch exec to free memory + // remove from hash + return 0; + +#if 0 int32_t code = taosHashRemove(pTq->pStreamTasks, &pReq->taskId, sizeof(int32_t)); + // set status dropping ASSERT(code == 0); if (code == 0) { // sendrsp } return code; +#endif +} + +int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { + char* msgStr = pMsg->pCont; + char* msgBody = POINTER_SHIFT(msgStr, sizeof(SMsgHead)); + int32_t msgLen = pMsg->contLen - sizeof(SMsgHead); + SStreamRetrieveReq req; + SDecoder decoder; + tDecoderInit(&decoder, msgBody, msgLen); + tDecodeStreamRetrieveReq(&decoder, &req); + int32_t taskId = req.dstTaskId; + SStreamTask* pTask = *(SStreamTask**)taosHashGet(pTq->pStreamTasks, &taskId, sizeof(int32_t)); + if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { + return 0; + } + SRpcMsg rsp = { + .info = pMsg->info, + .code = 0, + }; + streamProcessRetrieveReq(pTask, &req, &rsp); + return 0; +} + +int32_t tqProcessTaskRetrieveRsp(STQ* pTq, SRpcMsg* pMsg) { + // + return 0; } diff --git a/source/dnode/vnode/src/tq/tqExec.c b/source/dnode/vnode/src/tq/tqExec.c index 7c75d88a83d5813ceb5c34eb6f1e0cb779335d5f..da052ee3eb7723ab3d23ffc348d09384801ed578 100644 --- a/source/dnode/vnode/src/tq/tqExec.c +++ b/source/dnode/vnode/src/tq/tqExec.c @@ -29,7 +29,7 @@ static int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataBlkRsp* pRs // TODO enable compress int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, taosArrayGetSize(pBlock->pDataBlock), false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pRsp->blockDataLen, &actualLen); @@ -60,6 +60,30 @@ static int32_t tqAddTbNameToRsp(const STQ* pTq, const STqExecHandle* pExec, SMqD return 0; } +int32_t tqScanSnapshot(STQ* pTq, const STqExecHandle* pExec, SMqDataBlkRsp* pRsp, int32_t workerId) { + ASSERT(pExec->subType == TOPIC_SUB_TYPE__COLUMN); + qTaskInfo_t task = pExec->execCol.task[workerId]; + if (qStreamScanSnapshot(task) < 0) { + ASSERT(0); + } + while (1) { + SSDataBlock* pDataBlock = NULL; + uint64_t ts = 0; + if (qExecTask(task, &pDataBlock, &ts) < 0) { + ASSERT(0); + } + if (pDataBlock == NULL) break; + + ASSERT(pDataBlock->info.rows != 0); + ASSERT(taosArrayGetSize(pDataBlock->pDataBlock) != 0); + + tqAddBlockDataToRsp(pDataBlock, pRsp); + pRsp->blockNum++; + } + + return 0; +} + int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkRsp* pRsp, int32_t workerId) { if (pExec->subType == TOPIC_SUB_TYPE__COLUMN) { qTaskInfo_t task = pExec->execCol.task[workerId]; @@ -74,7 +98,6 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR if (pDataBlock == NULL) break; ASSERT(pDataBlock->info.rows != 0); - ASSERT(pDataBlock->info.numOfCols != 0); tqAddBlockDataToRsp(pDataBlock, pRsp); if (pRsp->withTbName) { @@ -88,8 +111,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlock(pReader)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } @@ -106,8 +128,7 @@ int32_t tqDataExec(STQ* pTq, STqExecHandle* pExec, SSubmitReq* pReq, SMqDataBlkR tqReadHandleSetMsg(pReader, pReq, 0); while (tqNextDataBlockFilterOut(pReader, pExec->execDb.pFilterOutTbUid)) { SSDataBlock block = {0}; - if (tqRetrieveDataBlock(&block.pDataBlock, pReader, &block.info.groupId, &block.info.uid, &block.info.rows, - &block.info.numOfCols) < 0) { + if (tqRetrieveDataBlock(&block, pReader, &block.info.groupId, &block.info.uid, &block.info.rows) < 0) { if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; ASSERT(0); } diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index 3af8901b2b20f97f66ea78209ddb3d9936fc93c1..2ec627bd5c92f351549ccc95328fa5b40a7b6577 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -45,9 +45,9 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { SMqDataBlkRsp rsp = {0}; // 1. guard and set status executing - int8_t execStatus = - atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - if (execStatus == TASK_STATUS__IDLE) { + int8_t execStatus = atomic_val_compare_exchange_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE, + TASK_EXEC_STATUS__EXECUTING); + if (execStatus == TASK_EXEC_STATUS__IDLE) { SStreamDataSubmit* pSubmit = NULL; // 2. check processedVer // 2.1. if not missed, get msg from queue @@ -68,18 +68,18 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { goto SEND_RSP; } // set exec status closing - atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__CLOSING); + atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__CLOSING); // second run if (tqLoopExecFromQueue(pTq, pHandle, &pSubmit, &rsp) == 0) { goto SEND_RSP; } // set exec status idle - atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); + atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE); } SEND_RSP: // 4. if get result // 4.1 set exec input status blocked and exec status idle - atomic_store_8(&pHandle->pushHandle.execStatus, TASK_STATUS__IDLE); + atomic_store_8(&pHandle->pushHandle.execStatus, TASK_EXEC_STATUS__IDLE); // 4.2 rpc send rsp.rspOffset = pHandle->pushHandle.processedVer; /*if (tqSendPollRsp(pTq, pMsg, pReq, &rsp) < 0) {*/ @@ -150,7 +150,7 @@ int32_t tqEnqueueAll(STQ* pTq, SSubmitReq* pReq) { continue; } int8_t execStatus = atomic_load_8(&pHandle->pushHandle.execStatus); - if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { + if (execStatus == TASK_EXEC_STATUS__IDLE || execStatus == TASK_EXEC_STATUS__CLOSING) { tqSendExecReq(pTq, pHandle); } } diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 0c38d6442b623ff56b0d67c771c800b83b570bb8..5e5a4883a9d91f63601dca9db9da7252563a3164 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -42,6 +42,25 @@ int64_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalHead* code = 0; goto END; } else { + if (pHandle->fetchMeta) { + SWalReadHead* pHead = &((*ppHeadWithCkSum)->head); + if (pHead->msgType == TDMT_VND_CREATE_STB || pHead->msgType == TDMT_VND_ALTER_STB || + pHead->msgType == TDMT_VND_DROP_STB || pHead->msgType == TDMT_VND_CREATE_TABLE || + pHead->msgType == TDMT_VND_ALTER_TABLE || pHead->msgType == TDMT_VND_DROP_TABLE || + pHead->msgType == TDMT_VND_DROP_TTL_TABLE) { + code = walFetchBody(pHandle->pWalReader, ppHeadWithCkSum); + + if (code < 0) { + ASSERT(0); + *fetchOffset = offset; + code = -1; + goto END; + } + *fetchOffset = offset; + code = 0; + goto END; + } + } code = walSkipFetchBody(pHandle->pWalReader, *ppHeadWithCkSum); if (code < 0) { ASSERT(0); @@ -123,8 +142,8 @@ bool tqNextDataBlockFilterOut(STqReadHandle* pHandle, SHashObj* filterOutUids) { return false; } -int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, - int32_t* pNumOfRows, int16_t* pNumOfCols) { +int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReadHandle* pHandle, uint64_t* pGroupId, uint64_t* pUid, + int32_t* pNumOfRows) { *pUid = 0; // TODO set to real sversion @@ -161,23 +180,14 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p int32_t colNumNeed = taosArrayGetSize(pHandle->pColIdList); if (colNumNeed == 0) { - *ppCols = taosArrayInit(pSchemaWrapper->nCols, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; while (colMeta < pSchemaWrapper->nCols) { SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; } } else { @@ -185,11 +195,6 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p colNumNeed = pSchemaWrapper->nCols; } - *ppCols = taosArrayInit(colNumNeed, sizeof(SColumnInfoData)); - if (*ppCols == NULL) { - return -1; - } - int32_t colMeta = 0; int32_t colNeed = 0; while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { @@ -201,23 +206,22 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p } else if (colIdSchema > colIdNeed) { colNeed++; } else { - SColumnInfoData colInfo = {0}; - colInfo.info.bytes = pColSchema->bytes; - colInfo.info.colId = pColSchema->colId; - colInfo.info.type = pColSchema->type; - - if (colInfoDataEnsureCapacity(&colInfo, 0, *pNumOfRows) < 0) { + SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); + int32_t code = blockDataAppendColInfo(pBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS) { goto FAIL; } - taosArrayPush(*ppCols, &colInfo); colMeta++; colNeed++; } } } - int32_t colActual = taosArrayGetSize(*ppCols); - *pNumOfCols = colActual; + if (blockDataEnsureCapacity(pBlock, *pNumOfRows) < 0) { + goto FAIL; + } + + int32_t colActual = blockDataGetNumOfCols(pBlock); // TODO in stream shuffle case, fetch groupId *pGroupId = 0; @@ -234,7 +238,7 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p tdSTSRowIterReset(&iter, row); // get all wanted col of that block for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(*ppCols, i); + SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); SCellVal sVal = {0}; if (!tdSTSRowIterNext(&iter, pColData->info.colId, pColData->info.type, &sVal)) { break; @@ -246,8 +250,9 @@ int32_t tqRetrieveDataBlock(SArray** ppCols, STqReadHandle* pHandle, uint64_t* p curRow++; } return 0; -FAIL: - if (*ppCols) taosArrayDestroy(*ppCols); + +FAIL: // todo refactor here +// if (*ppCols) taosArrayDestroy(*ppCols); return -1; } @@ -326,8 +331,8 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { while (1) { pIter = taosHashIterate(pTq->pStreamTasks, pIter); if (pIter == NULL) break; - SStreamTask* pTask = (SStreamTask*)pIter; - if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->isDataScan) { int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); ASSERT(code == 0); } diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index b628f0dde5347b2a482bb618c77017c1753a60bf..ef3b205b3e2481af6bc755ada64b7caaec6cf642 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -16,7 +16,7 @@ #include "tq.h" SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, bool createTb, int64_t suid, - const char* stbFullName, int32_t vgId) { + const char* stbFullName, int32_t vgId) { SSubmitReq* ret = NULL; SArray* tagArray = taosArrayInit(1, sizeof(STagVal)); if (!tagArray) { @@ -44,7 +44,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; @@ -80,11 +80,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo ret->length = sizeof(SSubmitReq); ret->numOfBlocks = htonl(sz); - void* submitBlk = POINTER_SHIFT(ret, sizeof(SSubmitReq)); + SSubmitBlk* blkHead = POINTER_SHIFT(ret, sizeof(SSubmitReq)); for (int32_t i = 0; i < sz; i++) { SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); - SSubmitBlk* blkHead = submitBlk; blkHead->numOfRows = htons(pDataBlock->info.rows); blkHead->sversion = htonl(pTSchema->version); // TODO @@ -93,11 +92,10 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo blkHead->uid = 0; int32_t rows = pDataBlock->info.rows; - /*int32_t maxLen = TD_ROW_MAX_BYTES_FROM_SCHEMA(pTSchema);*/ - /*blkHead->dataLen = htonl(rows * maxLen);*/ - blkHead->dataLen = 0; - void* blockData = POINTER_SHIFT(submitBlk, sizeof(SSubmitBlk)); + int32_t dataLen = 0; + + void* blkSchema = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk)); int32_t schemaLen = 0; if (createTb) { @@ -109,7 +107,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo createTbReq.ctb.suid = suid; STagVal tagVal = { - .cid = pDataBlock->info.numOfCols + 1, + .cid = taosArrayGetSize(pDataBlock->pDataBlock) + 1, .type = TSDB_DATA_TYPE_UBIGINT, .i64 = (int64_t)pDataBlock->info.groupId, }; @@ -135,7 +133,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } SEncoder encoder = {0}; - tEncoderInit(&encoder, blockData, schemaLen); + tEncoderInit(&encoder, blkSchema, schemaLen); code = tEncodeSVCreateTbReq(&encoder, &createTbReq); tEncoderClear(&encoder); tdDestroySVCreateTbReq(&createTbReq); @@ -148,7 +146,7 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } blkHead->schemaLen = htonl(schemaLen); - STSRow* rowData = POINTER_SHIFT(blockData, schemaLen); + STSRow* rowData = POINTER_SHIFT(blkSchema, schemaLen); for (int32_t j = 0; j < rows; j++) { SRowBuilder rb = {0}; @@ -168,14 +166,12 @@ SSubmitReq* tdBlockToSubmit(const SArray* pBlocks, const STSchema* pTSchema, boo } int32_t rowLen = TD_ROW_LEN(rowData); rowData = POINTER_SHIFT(rowData, rowLen); - blkHead->dataLen += rowLen; + dataLen += rowLen; } - int32_t dataLen = blkHead->dataLen; blkHead->dataLen = htonl(dataLen); ret->length += sizeof(SSubmitBlk) + schemaLen + dataLen; - blkHead = POINTER_SHIFT(blkHead, schemaLen + dataLen); - /*submitBlk = blkHead;*/ + blkHead = POINTER_SHIFT(blkHead, sizeof(SSubmitBlk) + schemaLen + dataLen); } ret->length = htonl(ret->length); diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index ce73246e51adcb3906b16a56084d44cf2fb0bed4..e16360a58b403937db0ca0e3759ec50e5ddccb86 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -16,13 +16,11 @@ #include "tsdb.h" #include "vnode.h" -#define EXTRA_BYTES 2 #define ASCENDING_TRAVERSE(o) (o == TSDB_ORDER_ASC) -#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pColumns))) +#define QH_GET_NUM_OF_COLS(handle) ((size_t)(taosArrayGetSize((handle)->pResBlock->pDataBlock))) #define GET_FILE_DATA_BLOCK_INFO(_checkInfo, _block) \ ((SDataBlockInfo){.window = {.skey = (_block)->minKey.ts, .ekey = (_block)->maxKey.ts}, \ - .numOfCols = (_block)->numOfCols, \ .rows = (_block)->numOfRows, \ .uid = (_checkInfo)->tableId}) @@ -115,7 +113,8 @@ typedef struct STsdbReadHandle { // SColumnDataAgg* statis; // query level statistics, only one table block statistics info exists at any time // SColumnDataAgg** pstatis;// the ptr array list to return to caller int32_t numOfBlocks; - SArray* pColumns; // column list, SColumnInfoData array list + SSDataBlock* pResBlock; +// SArray* pColumns; // column list, SColumnInfoData array list bool locateStart; int32_t outputCapacity; int32_t realNumOfRows; @@ -177,7 +176,7 @@ static SArray* getColumnIdList(STsdbReadHandle* pTsdbReadHandle) { SArray* pIdList = taosArrayInit(numOfCols, sizeof(int16_t)); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); taosArrayPush(pIdList, &pCol->info.colId); } @@ -425,23 +424,21 @@ static STsdbReadHandle* tsdbQueryTablesImpl(SVnode* pVnode, SQueryTableDataCond* } // todo: use list instead of array? - pReadHandle->pColumns = taosArrayInit(pCond->numOfCols, sizeof(SColumnInfoData)); - if (pReadHandle->pColumns == NULL) { + pReadHandle->pResBlock = createDataBlock(); + if (pReadHandle->pResBlock == NULL) { goto _end; } for (int32_t i = 0; i < pCond->numOfCols; ++i) { - SColumnInfoData colInfo = {{0}, 0}; - colInfo.info = pCond->colList[i]; - - int32_t code = colInfoDataEnsureCapacity(&colInfo, 0, pReadHandle->outputCapacity); - if (code != TSDB_CODE_SUCCESS) { + SColumnInfoData colInfo = {.info = pCond->colList[i], 0}; + int32_t code = blockDataAppendColInfo(pReadHandle->pResBlock, &colInfo); + if (code != TSDB_CODE_SUCCESS){ goto _end; } - - taosArrayPush(pReadHandle->pColumns, &colInfo); } + blockDataEnsureCapacity(pReadHandle->pResBlock, pReadHandle->outputCapacity); + pReadHandle->suppInfo.defaultLoadColumn = getDefaultLoadColumns(pReadHandle, true); size_t size = taosArrayGetSize(pReadHandle->suppInfo.defaultLoadColumn); @@ -1066,11 +1063,11 @@ static int32_t getFileIdFromKey(TSKEY key, int32_t daysPerFile, int32_t precisio } int64_t fid = (int64_t)(key / (daysPerFile * tsTickPerMin[precision])); // set the starting fileId - if (fid < 0L && llabs(fid) > INT32_MAX) { // data value overflow for INT32 + if (fid < 0LL && llabs(fid) > INT32_MAX) { // data value overflow for INT32 fid = INT32_MIN; } - if (fid > 0L && fid > INT32_MAX) { + if (fid > 0LL && fid > INT32_MAX) { fid = INT32_MAX; } @@ -1536,12 +1533,12 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t int32_t trueEnd = ascScan ? end : start; int32_t step = ascScan ? 1 : -1; - int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t requiredNumOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); // data in buffer has greater timestamp, copy data in file block int32_t i = 0, j = 0; while (i < requiredNumOfCols && j < pCols->numOfCols) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SDataCol* src = &pCols->cols[j]; if (src->colId < pColInfo->info.colId) { @@ -1592,7 +1589,7 @@ static int32_t doCopyRowsFromFileBlock(STsdbReadHandle* pTsdbReadHandle, int32_t } while (i < requiredNumOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppendNNULL(pColInfo, numOfRows, num); i++; } @@ -1674,7 +1671,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa int32_t i = 0, j = 0, k = 0; while (i < numOfCols && (j < numOfColsOfRow1 || k < numOfColsOfRow2)) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); int32_t colIdOfRow1; if (j >= numOfColsOfRow1) { @@ -1805,7 +1802,7 @@ static int32_t mergeTwoRowFromMem(STsdbReadHandle* pTsdbReadHandle, int32_t capa if (mergeOption == 1) { while (i < numOfCols) { // the remain columns are all null data - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colDataAppend(pColInfo, *curRow, NULL, true); ++i; } @@ -1861,7 +1858,7 @@ static void doCheckGeneratedBlockRange(STsdbReadHandle* pTsdbReadHandle) { assert(cur->win.skey >= pTsdbReadHandle->window.ekey && cur->win.ekey <= pTsdbReadHandle->window.skey); } - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(cur->win.skey == ((TSKEY*)pColInfoData->pData)[0] && cur->win.ekey == ((TSKEY*)pColInfoData->pData)[cur->rows - 1]); } else { @@ -2771,7 +2768,7 @@ static int tsdbReadRowsFromCache(STableCheckInfo* pCheckInfo, TSKEY maxKey, int STsdbReadHandle* pTsdbReadHandle) { int numOfRows = 0; int curRows = 0; - int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pColumns); + int32_t numOfCols = (int32_t)taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); STsdbCfg* pCfg = REPO_CFG(pTsdbReadHandle->pTsdb); win->skey = TSKEY_INITIAL_VAL; @@ -2845,7 +2842,7 @@ int32_t tsdbGetAllTableList(SMeta* pMeta, uint64_t uid, SArray* list) { break; } - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, uid = id, .groupId = 0}; taosArrayPush(list, &info); } @@ -2901,7 +2898,7 @@ static bool loadBlockOfActiveTable(STsdbReadHandle* pTsdbReadHandle) { if (exists) { tsdbRetrieveDataBlock((tsdbReaderT*)pTsdbReadHandle, NULL); if (pTsdbReadHandle->currentLoadExternalRows && pTsdbReadHandle->window.skey == pTsdbReadHandle->window.ekey) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, 0); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, 0); assert(*(int64_t*)pColInfo->pData == pTsdbReadHandle->window.skey); } @@ -3003,7 +3000,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // int32_t i = 0, j = 0; // while(i < tgNumOfCols && j < numOfCols) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // if (pTable->lastCols[j].colId < pColInfo->info.colId) { // j++; // continue; @@ -3066,7 +3063,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // if (pColInfo->info.colId == PRIMARYKEY_TIMESTAMP_COL_ID) { @@ -3091,7 +3088,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // // // leave the real ts column as the last row, because last function only (not stable) use the last row as res // if (priKey != TSKEY_INITIAL_VAL) { -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, priIdx); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, priIdx); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes; // // *(TSKEY *)pData = priKey; @@ -3101,7 +3098,7 @@ static bool loadCachedLastRow(STsdbReadHandle* pTsdbReadHandle) { // continue; // } // -// pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, n); +// pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, n); // pData = (char*)pColInfo->pData + numOfRows * pColInfo->info.bytes;; // // assert (pColInfo->info.colId != PRIMARYKEY_TIMESTAMP_COL_ID); @@ -3161,9 +3158,9 @@ static bool loadDataBlockFromTableSeq(STsdbReadHandle* pTsdbReadHandle) { bool tsdbNextDataBlock(tsdbReaderT pHandle) { STsdbReadHandle* pTsdbReadHandle = (STsdbReadHandle*)pHandle; - size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pColumns); + size_t numOfCols = taosArrayGetSize(pTsdbReadHandle->pResBlock->pDataBlock); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfo = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); colInfoDataCleanup(pColInfo, pTsdbReadHandle->outputCapacity); } @@ -3250,7 +3247,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // SArray* row = (type == TSDB_PREV_ROW)? pTsdbReadHandle->prev : pTsdbReadHandle->next; // // for (int32_t i = 0; i < numOfCols; ++i) { -// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pCol = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // // SColumnInfoData colInfo = {{0}, 0}; // colInfo.info = pCol->info; @@ -3280,7 +3277,7 @@ bool tsdbNextDataBlock(tsdbReaderT pHandle) { // } // // for (int32_t i = 0; i < cond.numOfCols; ++i) { -// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); +// SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); // memcpy(&cond.colList[i], &pColInfoData->info, sizeof(SColumnInfo)); // } // @@ -3342,7 +3339,7 @@ bool tsdbGetExternalRow(tsdbReaderT pHandle) { int32_t numOfCols = (int32_t)QH_GET_NUM_OF_COLS(pTsdbReadHandle); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pColumns, i); + SColumnInfoData* pColInfoData = taosArrayGet(pTsdbReadHandle->pResBlock->pDataBlock, i); SColumnInfoData* first = taosArrayGet(pTsdbReadHandle->prev, i); memcpy(pColInfoData->pData, first->pData, pColInfoData->info.bytes); @@ -3617,13 +3614,13 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { */ STsdbReadHandle* pHandle = (STsdbReadHandle*)pTsdbReadHandle; if (pHandle->cur.fid == INT32_MIN) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { STableBlockInfo* pBlockInfo = &pHandle->pDataBlockInfo[pHandle->cur.slot]; STableCheckInfo* pCheckInfo = pBlockInfo->pTableCheckInfo; if (pHandle->cur.mixBlock) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { SDataBlockInfo binfo = GET_FILE_DATA_BLOCK_INFO(pCheckInfo, pBlockInfo->compBlock); assert(pHandle->realNumOfRows <= binfo.rows); @@ -3633,7 +3630,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { if (pBlockLoadInfo->slot == pHandle->cur.slot && pBlockLoadInfo->fileGroup->fid == pHandle->cur.fid && pBlockLoadInfo->uid == pCheckInfo->tableId) { - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } else { // only load the file block SBlock* pBlock = pBlockInfo->compBlock; if (doLoadFileDataBlock(pHandle, pBlock, pCheckInfo, pHandle->cur.slot) != TSDB_CODE_SUCCESS) { @@ -3641,7 +3638,7 @@ SArray* tsdbRetrieveDataBlock(tsdbReaderT* pTsdbReadHandle, SArray* pIdList) { } int32_t numOfRows = doCopyRowsFromFileBlock(pHandle, pHandle->outputCapacity, 0, 0, pBlock->numOfRows - 1); - return pHandle->pColumns; + return pHandle->pResBlock->pDataBlock; } } } @@ -3692,7 +3689,7 @@ void tsdbCleanupReadHandle(tsdbReaderT queryHandle) { return; } - pTsdbReadHandle->pColumns = doFreeColumnInfoData(pTsdbReadHandle->pColumns); + pTsdbReadHandle->pResBlock->pDataBlock = doFreeColumnInfoData(pTsdbReadHandle->pResBlock->pDataBlock); taosArrayDestroy(pTsdbReadHandle->suppInfo.defaultLoadColumn); taosMemoryFreeClear(pTsdbReadHandle->pDataBlockInfo); diff --git a/source/dnode/vnode/src/vnd/vnodeQuery.c b/source/dnode/vnode/src/vnd/vnodeQuery.c index 2333a93ce094e000b725e4df917d7b088f4cc4e3..32d183ca0ad44438d28d40a234fd7dcd5cf8d0c3 100644 --- a/source/dnode/vnode/src/vnd/vnodeQuery.c +++ b/source/dnode/vnode/src/vnd/vnodeQuery.c @@ -124,6 +124,115 @@ _exit: return TSDB_CODE_SUCCESS; } +int vnodeGetTableCfg(SVnode *pVnode, SRpcMsg *pMsg) { + STableCfgReq cfgReq = {0}; + STableCfgRsp cfgRsp = {0}; + SMetaReader mer1 = {0}; + SMetaReader mer2 = {0}; + char tableFName[TSDB_TABLE_FNAME_LEN]; + SRpcMsg rpcMsg; + int32_t code = 0; + int32_t rspLen = 0; + void *pRsp = NULL; + SSchemaWrapper schema = {0}; + SSchemaWrapper schemaTag = {0}; + + // decode req + if (tDeserializeSTableCfgReq(pMsg->pCont, pMsg->contLen, &cfgReq) != 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + strcpy(cfgRsp.tbName, cfgReq.tbName); + memcpy(cfgRsp.dbFName, cfgReq.dbFName, sizeof(cfgRsp.dbFName)); + + sprintf(tableFName, "%s.%s", cfgReq.dbFName, cfgReq.tbName); + code = vnodeValidateTableHash(pVnode, tableFName); + if (code) { + goto _exit; + } + + // query meta + metaReaderInit(&mer1, pVnode->pMeta, 0); + + if (metaGetTableEntryByName(&mer1, cfgReq.tbName) < 0) { + code = terrno; + goto _exit; + } + + cfgRsp.tableType = mer1.me.type; + + if (mer1.me.type == TSDB_SUPER_TABLE) { + code = TSDB_CODE_VND_HASH_MISMATCH; + goto _exit; + } else if (mer1.me.type == TSDB_CHILD_TABLE) { + metaReaderInit(&mer2, pVnode->pMeta, 0); + if (metaGetTableEntryByUid(&mer2, mer1.me.ctbEntry.suid) < 0) goto _exit; + + strcpy(cfgRsp.stbName, mer2.me.name); + schema = mer2.me.stbEntry.schemaRow; + schemaTag = mer2.me.stbEntry.schemaTag; + cfgRsp.ttl = mer1.me.ctbEntry.ttlDays; + cfgRsp.commentLen = mer1.me.ctbEntry.commentLen; + if (mer1.me.ctbEntry.commentLen > 0) { + cfgRsp.pComment = strdup(mer1.me.ctbEntry.comment); + } + STag *pTag = (STag *)mer1.me.ctbEntry.pTags; + cfgRsp.tagsLen = pTag->len; + cfgRsp.pTags = taosMemoryMalloc(cfgRsp.tagsLen); + memcpy(cfgRsp.pTags, pTag, cfgRsp.tagsLen); + } else if (mer1.me.type == TSDB_NORMAL_TABLE) { + schema = mer1.me.ntbEntry.schemaRow; + cfgRsp.ttl = mer1.me.ntbEntry.ttlDays; + cfgRsp.commentLen = mer1.me.ntbEntry.commentLen; + if (mer1.me.ntbEntry.commentLen > 0) { + cfgRsp.pComment = strdup(mer1.me.ntbEntry.comment); + } + } else { + ASSERT(0); + } + + cfgRsp.numOfTags = schemaTag.nCols; + cfgRsp.numOfColumns = schema.nCols; + cfgRsp.pSchemas = (SSchema *)taosMemoryMalloc(sizeof(SSchema) * (cfgRsp.numOfColumns + cfgRsp.numOfTags)); + + memcpy(cfgRsp.pSchemas, schema.pSchema, sizeof(SSchema) * schema.nCols); + if (schemaTag.nCols) { + memcpy(cfgRsp.pSchemas + schema.nCols, schemaTag.pSchema, sizeof(SSchema) * schemaTag.nCols); + } + + // encode and send response + rspLen = tSerializeSTableCfgRsp(NULL, 0, &cfgRsp); + if (rspLen < 0) { + code = TSDB_CODE_INVALID_MSG; + goto _exit; + } + + pRsp = rpcMallocCont(rspLen); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _exit; + } + tSerializeSTableCfgRsp(pRsp, rspLen, &cfgRsp); + +_exit: + rpcMsg.info = pMsg->info; + rpcMsg.pCont = pRsp; + rpcMsg.contLen = rspLen; + rpcMsg.code = code; + + if (code) { + qError("get table %s cfg failed cause of %s", cfgReq.tbName, tstrerror(code)); + } + + tmsgSendRsp(&rpcMsg); + + tFreeSTableCfgRsp(&cfgRsp); + metaReaderClear(&mer2); + metaReaderClear(&mer1); + return TSDB_CODE_SUCCESS; +} + int32_t vnodeGetLoad(SVnode *pVnode, SVnodeLoad *pLoad) { pLoad->vgId = TD_VID(pVnode); pLoad->syncState = syncGetMyRole(pVnode->sync); @@ -157,4 +266,4 @@ tsdbReaderT tsdbQueryCacheLast(SVnode *pVnode, SQueryTableDataCond *pCond, STabl return tsdbQueryCacheLastT(pVnode->pTsdb, pCond, groupList, qId, pMemRef); #endif return 0; -} \ No newline at end of file +} diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index f3595ebfb0972b0d6c1ad9e03fd3902d200fb41d..88bdea3ae7722cc65727ff4e697a7597a0f7852b 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -172,7 +172,7 @@ int32_t vnodeProcessWriteReq(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } } break; - case TDMT_VND_STREAM_TASK_DROP: { + case TDMT_STREAM_TASK_DROP: { if (tqProcessTaskDropReq(pVnode->pTq, pMsg->pCont, pMsg->contLen) < 0) { goto _err; } @@ -255,6 +255,8 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return qWorkerProcessHbMsg(pVnode, pVnode->pQuery, pMsg, 0); case TDMT_VND_TABLE_META: return vnodeGetTableMeta(pVnode, pMsg); + case TDMT_VND_TABLE_CFG: + return vnodeGetTableCfg(pVnode, pMsg); case TDMT_VND_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg, pInfo->workerId); case TDMT_STREAM_TASK_RUN: @@ -263,10 +265,14 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return tqProcessTaskDispatchReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RECOVER: return tqProcessTaskRecoverReq(pVnode->pTq, pMsg); + case TDMT_STREAM_RETRIEVE: + return tqProcessTaskRetrieveReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: return tqProcessTaskDispatchRsp(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RECOVER_RSP: return tqProcessTaskRecoverRsp(pVnode->pTq, pMsg); + case TDMT_STREAM_RETRIEVE_RSP: + return tqProcessTaskRetrieveRsp(pVnode->pTq, pMsg); default: vError("unknown msg type:%d in fetch queue", pMsg->msgType); return TSDB_CODE_VND_APP_ERROR; @@ -309,7 +315,7 @@ static int32_t vnodeProcessDropTtlTbReq(SVnode *pVnode, int64_t version, void *p if (tbUids == NULL) return TSDB_CODE_OUT_OF_MEMORY; int32_t t = ntohl(*(int32_t *)pReq); - vError("rec ttl time:%d", t); + vDebug("vgId:%d, recv ttl msg, time:%d", pVnode->config.vgId, t); int32_t ret = metaTtlDropTable(pVnode->pMeta, t, tbUids); if (ret != 0) { goto end; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index f1c43512cec80c744f9a89f983ecc3c985e83d15..32090b774ecced3b0cb3b6e4f98d45ff72e93c69 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -336,7 +336,7 @@ static void vnodeSyncReconfig(struct SSyncFSM *pFsm, const SRpcMsg *pMsg, SReCon SVnode *pVnode = pFsm->data; SRpcMsg rpcMsg = {.msgType = pMsg->msgType, .contLen = pMsg->contLen}; - syncGetAndDelRespRpc(pVnode->sync, cbMeta.seqNum, &rpcMsg.info); + syncGetAndDelRespRpc(pVnode->sync, cbMeta.newCfgSeqNum, &rpcMsg.info); rpcMsg.info.conn.applyIndex = cbMeta.index; STraceId *trace = (STraceId *)&pMsg->info.traceId; diff --git a/source/dnode/vnode/test/tsdbSmaTest.cpp b/source/dnode/vnode/test/tsdbSmaTest.cpp index 2c1e6fbbbd55968d5bbf4d6057a09d814460aca9..2a8c7a583ccb4f5aa720f2fd83db85a19f76b93b 100644 --- a/source/dnode/vnode/test/tsdbSmaTest.cpp +++ b/source/dnode/vnode/test/tsdbSmaTest.cpp @@ -435,7 +435,7 @@ TEST(testCase, tSma_Data_Insert_Query_Test) { SSDataBlock *pDataBlock = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); EXPECT_NE(pDataBlock, nullptr); pDataBlock->pBlockAgg = NULL; - pDataBlock->info.numOfCols = tSmaNumOfCols; + taosArrayGetSize(pDataBlock->pDataBlock) = tSmaNumOfCols; pDataBlock->info.rows = tSmaNumOfRows; pDataBlock->info.groupId = tSmaGroupId + g; diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index 7c90c3538cc3428d6c6456c8c67f9acb4818e82c..384c3f19e5ed76333d8bcebcac403e7c7c4c1358 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -65,17 +65,25 @@ enum { typedef enum { CTG_TASK_GET_QNODE = 0, + CTG_TASK_GET_DNODE, CTG_TASK_GET_DB_VGROUP, CTG_TASK_GET_DB_CFG, CTG_TASK_GET_DB_INFO, CTG_TASK_GET_TB_META, CTG_TASK_GET_TB_HASH, CTG_TASK_GET_TB_INDEX, + CTG_TASK_GET_TB_CFG, CTG_TASK_GET_INDEX, CTG_TASK_GET_UDF, CTG_TASK_GET_USER, } CTG_TASK_TYPE; +typedef enum { + CTG_TASK_LAUNCHED = 1, + CTG_TASK_DONE, +} CTG_TASK_STATUS; + + typedef struct SCtgDebug { bool lockEnable; bool cacheEnable; @@ -102,6 +110,12 @@ typedef struct SCtgTbIndexCtx { SName* pName; } SCtgTbIndexCtx; +typedef struct SCtgTbCfgCtx { + SName* pName; + int32_t tbType; + SVgroupInfo* pVgInfo; +} SCtgTbCfgCtx; + typedef struct SCtgDbVgCtx { char dbFName[TSDB_DB_FNAME_LEN]; } SCtgDbVgCtx; @@ -190,7 +204,9 @@ typedef struct SCtgJob { SArray* pTasks; int32_t taskDone; SMetaData jobRes; - + int32_t taskIdx; + SRWLatch taskLock; + uint64_t queryId; SCatalog* pCtg; SRequestConnInfo conn; @@ -201,11 +217,13 @@ typedef struct SCtgJob { int32_t dbVgNum; int32_t udfNum; int32_t qnodeNum; + int32_t dnodeNum; int32_t dbCfgNum; int32_t indexNum; int32_t userNum; int32_t dbInfoNum; int32_t tbIndexNum; + int32_t tbCfgNum; } SCtgJob; typedef struct SCtgMsgCtx { @@ -215,24 +233,44 @@ typedef struct SCtgMsgCtx { char* target; } SCtgMsgCtx; +typedef struct SCtgTask SCtgTask; +typedef int32_t (*ctgSubTaskCbFp)(SCtgTask*); + +typedef struct SCtgSubRes { + CTG_TASK_TYPE type; + int32_t code; + void* res; + ctgSubTaskCbFp fp; +} SCtgSubRes; + typedef struct SCtgTask { - CTG_TASK_TYPE type; - int32_t taskId; - SCtgJob* pJob; - void* taskCtx; - SCtgMsgCtx msgCtx; - int32_t code; - void* res; + CTG_TASK_TYPE type; + int32_t taskId; + SCtgJob* pJob; + void* taskCtx; + SCtgMsgCtx msgCtx; + int32_t code; + void* res; + CTG_TASK_STATUS status; + SRWLatch lock; + SArray* pParents; + SCtgSubRes subRes; } SCtgTask; +typedef int32_t (*ctgInitTaskFp)(SCtgJob*, int32_t, void*); typedef int32_t (*ctgLanchTaskFp)(SCtgTask*); typedef int32_t (*ctgHandleTaskMsgRspFp)(SCtgTask*, int32_t, const SDataBuf *, int32_t); typedef int32_t (*ctgDumpTaskResFp)(SCtgTask*); +typedef int32_t (*ctgCloneTaskResFp)(SCtgTask*, void**); +typedef int32_t (*ctgCompTaskFp)(SCtgTask*, void*, bool*); typedef struct SCtgAsyncFps { - ctgLanchTaskFp launchFp; + ctgInitTaskFp initFp; + ctgLanchTaskFp launchFp; ctgHandleTaskMsgRspFp handleRspFp; - ctgDumpTaskResFp dumpResFp; + ctgDumpTaskResFp dumpResFp; + ctgCompTaskFp compFp; + ctgCloneTaskResFp cloneFp; } SCtgAsyncFps; typedef struct SCtgApiStat { @@ -520,6 +558,8 @@ int32_t ctgDropTbIndexEnqueue(SCatalog* pCtg, SName* pName, bool syncOp); int32_t ctgOpDropTbIndex(SCtgCacheOperation *operation); int32_t ctgOpUpdateTbIndex(SCtgCacheOperation *operation); int32_t ctgOpClearCache(SCtgCacheOperation *operation); +int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType); +int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVgroupInfo **pVgroup); @@ -527,6 +567,7 @@ int32_t ctgOpClearCache(SCtgCacheOperation *operation); int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, int32_t rspCode, char* target); int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask); int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask); +int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray **out, SCtgTask* pTask); int32_t ctgGetDBCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *dbFName, SDbCfgInfo *out, SCtgTask* pTask); int32_t ctgGetIndexInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const char *indexName, SIndexInfo *out, SCtgTask* pTask); int32_t ctgGetTbIndexFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SName *name, STableIndex* out, SCtgTask* pTask); @@ -535,10 +576,14 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask); int32_t ctgGetTbMetaFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableMetaOutput* out, SCtgTask* pTask); int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableMetaOutput* out, SCtgTask* pTask); +int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask); +int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask); int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum); int32_t ctgLaunchJob(SCtgJob *pJob); int32_t ctgMakeAsyncRes(SCtgJob *pJob); +int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param); +int32_t ctgGetTbCfgCb(SCtgTask *pTask); int32_t ctgCloneVgInfo(SDBVgInfo *src, SDBVgInfo **dst); int32_t ctgCloneMetaOutput(STableMetaOutput *output, STableMetaOutput **pOutput); @@ -559,6 +604,7 @@ char * ctgTaskTypeStr(CTG_TASK_TYPE type); int32_t ctgUpdateSendTargetInfo(SMsgSendInfo *pMsgSendInfo, int32_t msgType, SCtgTask* pTask); int32_t ctgCloneTableIndex(SArray* pIndex, SArray** pRes); void ctgFreeSTableIndex(void *info); +void ctgClearSubTaskRes(SCtgSubRes *pRes); extern SCatalogMgmt gCtgMgmt; diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index bb02895569c180ea442992f60818e541a842eb8d..931a944adfbe4db5370ffb6591c4d33403dc0afb 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -22,36 +22,6 @@ SCatalogMgmt gCtgMgmt = {0}; -int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { - int32_t code = 0; - STableMeta* tblMeta = NULL; - SCtgTbMetaCtx tbCtx = {0}; - tbCtx.flag = CTG_FLAG_UNKNOWN_STB; - tbCtx.pName = pTableName; - - CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta)); - - if (NULL == tblMeta) { - ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); - return TSDB_CODE_SUCCESS; - } - - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTableName, dbFName); - - if (TSDB_SUPER_TABLE == tblMeta->tableType) { - CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); - } else { - CTG_ERR_JRET(ctgDropTbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq)); - } - -_return: - - taosMemoryFreeClear(tblMeta); - - CTG_RET(code); -} - int32_t ctgGetDBVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* dbFName, SCtgDBCache** dbCache, SDBVgInfo **pInfo) { int32_t code = 0; @@ -212,29 +182,6 @@ _return: CTG_RET(code); } -int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { - if (CTG_IS_SYS_DBNAME(ctx->pName->dbname)) { - CTG_FLAG_SET_SYS_DB(ctx->flag); - } - - CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, ctx, pTableMeta)); - - if (*pTableMeta) { - if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && - ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) { - return TSDB_CODE_SUCCESS; - } - - taosMemoryFreeClear(*pTableMeta); - } - - if (CTG_FLAG_IS_UNKNOWN_STB(ctx->flag)) { - CTG_FLAG_SET_STB(ctx->flag, ctx->tbInfo.tbType); - } - - return TSDB_CODE_SUCCESS; -} - int32_t ctgGetTbMeta(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { int32_t code = 0; STableMetaOutput *output = NULL; @@ -381,6 +328,23 @@ _return: return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, int32_t *tbType) { + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + CTG_ERR_RET(ctgReadTbTypeFromCache(pCtg, dbFName, pTableName->tname, tbType)); + if (*tbType > 0) { + return TSDB_CODE_SUCCESS; + } + + STableMeta* pMeta = NULL; + CTG_ERR_RET(catalogGetTableMeta(pCtg, pConn, pTableName, &pMeta)); + + *tbType = pMeta->tableType; + taosMemoryFree(pMeta); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetTbIndex(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pRes) { CTG_ERR_RET(ctgReadTbIndexFromCache(pCtg, pTableName, pRes)); if (*pRes) { @@ -419,6 +383,20 @@ _return: CTG_RET(code); } +int32_t ctgGetTbCfg(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, STableCfg** pCfg) { + int32_t tbType = 0; + CTG_ERR_RET(ctgGetTbType(pCtg, pConn, pTableName, &tbType)); + + if (TSDB_SUPER_TABLE == tbType) { + CTG_ERR_RET(ctgGetTableCfgFromMnode(pCtg, pConn, pTableName, pCfg, NULL)); + } else { + SVgroupInfo vgroupInfo = {0}; + CTG_ERR_RET(catalogGetTableHashVgroup(pCtg, pConn, pTableName, &vgroupInfo)); + CTG_ERR_RET(ctgGetTableCfgFromVnode(pCtg, pConn, pTableName, &vgroupInfo, pCfg, NULL)); + } + + CTG_RET(TSDB_CODE_SUCCESS); +} int32_t ctgGetTbDistVgInfo(SCatalog* pCtg, SRequestConnInfo *pConn, SName* pTableName, SArray** pVgList) { STableMeta *tbMeta = NULL; @@ -1121,6 +1099,21 @@ _return: CTG_API_LEAVE(TSDB_CODE_SUCCESS); } +int32_t catalogGetDnodeList(SCatalog* pCtg, SRequestConnInfo* pConn, SArray** pDnodeList) { + CTG_API_ENTER(); + + int32_t code = 0; + if (NULL == pCtg || NULL == pConn || NULL == pDnodeList) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + CTG_ERR_JRET(ctgGetDnodeListFromMnode(pCtg, pConn, pDnodeList, NULL)); + +_return: + + CTG_API_LEAVE(TSDB_CODE_SUCCESS); +} + int32_t catalogGetExpiredSTables(SCatalog* pCtg, SSTableVersion **stables, uint32_t *num) { CTG_API_ENTER(); @@ -1207,6 +1200,23 @@ _return: CTG_API_LEAVE(code); } +int32_t catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg** pCfg) { + CTG_API_ENTER(); + + if (NULL == pCtg || NULL == pConn || NULL == pTableName || NULL == pCfg) { + CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); + } + + int32_t code = 0; + CTG_ERR_JRET(catalogRemoveTableMeta(pCtg, (SName*)pTableName)); + + CTG_ERR_JRET(ctgGetTbCfg(pCtg, pConn, (SName*)pTableName, pCfg)); + +_return: + + CTG_API_LEAVE(code); +} + int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo *pConn, const char* funcName, SFuncInfo* pInfo) { CTG_API_ENTER(); diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 6adadf50455950e5d49bf0a5a9e178673c6fcf6b..8928a7e028bd4a6cdc9f07ab0de29447dc5b09d8 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -20,7 +20,8 @@ #include "systable.h" #include "tref.h" -int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { +int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_META; @@ -44,12 +45,13 @@ int32_t ctgInitGetTbMetaTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { +int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + char *dbFName = (char*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_DB_VGROUP; @@ -67,12 +69,13 @@ int32_t ctgInitGetDbVgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { +int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + char *dbFName = (char*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_DB_CFG; @@ -90,12 +93,13 @@ int32_t ctgInitGetDbCfgTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetDbInfoTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { +int32_t ctgInitGetDbInfoTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + char *dbFName = (char*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_DB_INFO; @@ -113,13 +117,14 @@ int32_t ctgInitGetDbInfoTask(SCtgJob *pJob, int32_t taskIdx, char *dbFName) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, dbFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), dbFName); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { +int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_HASH; @@ -143,12 +148,12 @@ int32_t ctgInitGetTbHashTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tableName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) { +int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { SCtgTask task = {0}; task.type = CTG_TASK_GET_QNODE; @@ -163,7 +168,23 @@ int32_t ctgInitGetQnodeTask(SCtgJob *pJob, int32_t taskIdx) { return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) { +int32_t ctgInitGetDnodeTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_DNODE; + task.taskId = taskIdx; + task.pJob = pJob; + task.taskCtx = NULL; + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %d task type %s initialized", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type)); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + char *name = (char*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_INDEX; @@ -181,12 +202,13 @@ int32_t ctgInitGetIndexTask(SCtgJob *pJob, int32_t taskIdx, char *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, indexFName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) { +int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + char *name = (char*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_UDF; @@ -204,12 +226,13 @@ int32_t ctgInitGetUdfTask(SCtgJob *pJob, int32_t taskIdx, char *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, udfName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) { +int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SUserAuthInfo *user = (SUserAuthInfo*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_USER; @@ -227,12 +250,13 @@ int32_t ctgInitGetUserTask(SCtgJob *pJob, int32_t taskIdx, SUserAuthInfo *user) taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, user:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), user->user); return TSDB_CODE_SUCCESS; } -int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { +int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; SCtgTask task = {0}; task.type = CTG_TASK_GET_TB_INDEX; @@ -255,11 +279,41 @@ int32_t ctgInitGetTbIndexTask(SCtgJob *pJob, int32_t taskIdx, SName *name) { taosArrayPush(pJob->pTasks, &task); - qDebug("QID:0x%" PRIx64 " the %d task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); return TSDB_CODE_SUCCESS; } +int32_t ctgInitGetTbCfgTask(SCtgJob *pJob, int32_t taskIdx, void* param) { + SName *name = (SName*)param; + SCtgTask task = {0}; + + task.type = CTG_TASK_GET_TB_CFG; + task.taskId = taskIdx; + task.pJob = pJob; + + task.taskCtx = taosMemoryCalloc(1, sizeof(SCtgTbCfgCtx)); + if (NULL == task.taskCtx) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + SCtgTbCfgCtx* ctx = task.taskCtx; + ctx->pName = taosMemoryMalloc(sizeof(*name)); + if (NULL == ctx->pName) { + taosMemoryFree(task.taskCtx); + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + memcpy(ctx->pName, name, sizeof(*name)); + + taosArrayPush(pJob->pTasks, &task); + + qDebug("QID:0x%" PRIx64 " the %dth task type %s initialized, tbName:%s", pJob->queryId, taskIdx, ctgTaskTypeStr(task.type), name->tname); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, const SCatalogReq* pReq) { SHashObj* pDb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); @@ -296,6 +350,13 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); } + for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { + SName* name = taosArrayGet(pReq->pTableCfg, i); + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(name, dbFName); + taosHashPut(pDb, dbFName, strlen(dbFName), dbFName, TSDB_DB_FNAME_LEN); + } + char* dbFName = taosHashIterate(pDb, NULL); while (dbFName) { ctgDropDbVgroupEnqueue(pCtg, dbFName, true); @@ -304,40 +365,32 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con taosHashCleanup(pDb); - int32_t tbNum = pJob->tbMetaNum + pJob->tbHashNum; - if (tbNum > 0) { - if (tbNum > pJob->tbMetaNum && tbNum > pJob->tbHashNum) { - SHashObj* pTb = taosHashInit(tbNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); - } + // REFRESH TABLE META + SHashObj* pTb = taosHashInit(taskNum, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), false, HASH_NO_LOCK); + for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { + SName* name = taosArrayGet(pReq->pTableMeta, i); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } + + for (int32_t i = 0; i < pJob->tbHashNum; ++i) { + SName* name = taosArrayGet(pReq->pTableHash, i); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } - SName* name = taosHashIterate(pTb, NULL); - while (name) { - catalogRemoveTableMeta(pCtg, name); - name = taosHashIterate(pTb, name); - } + for (int32_t i = 0; i < pJob->tbCfgNum; ++i) { + SName* name = taosArrayGet(pReq->pTableCfg, i); + taosHashPut(pTb, name, sizeof(SName), name, sizeof(SName)); + } - taosHashCleanup(pTb); - } else { - for (int32_t i = 0; i < pJob->tbMetaNum; ++i) { - SName* name = taosArrayGet(pReq->pTableMeta, i); - catalogRemoveTableMeta(pCtg, name); - } - - for (int32_t i = 0; i < pJob->tbHashNum; ++i) { - SName* name = taosArrayGet(pReq->pTableHash, i); - catalogRemoveTableMeta(pCtg, name); - } - } + SName* name = taosHashIterate(pTb, NULL); + while (name) { + catalogRemoveTableMeta(pCtg, name); + name = taosHashIterate(pTb, name); } + taosHashCleanup(pTb); + + for (int32_t i = 0; i < pJob->tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); ctgDropTbIndexEnqueue(pCtg, name, true); @@ -346,6 +399,20 @@ int32_t ctgHandleForceUpdate(SCatalog* pCtg, int32_t taskNum, SCtgJob *pJob, con return TSDB_CODE_SUCCESS; } +int32_t ctgInitTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t *taskId) { + int32_t tid = atomic_fetch_add_32(&pJob->taskIdx, 1); + + CTG_LOCK(CTG_WRITE, &pJob->taskLock); + CTG_ERR_RET((*gCtgAsyncFps[type].initFp)(pJob, tid, param)); + CTG_UNLOCK(CTG_WRITE, &pJob->taskLock); + + if (taskId) { + *taskId = tid; + } + + return TSDB_CODE_SUCCESS; +} + int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint64_t reqId, const SCatalogReq* pReq, catalogCallback fp, void* param, int32_t* taskNum) { int32_t code = 0; int32_t tbMetaNum = (int32_t)taosArrayGetSize(pReq->pTableMeta); @@ -353,13 +420,15 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 int32_t tbHashNum = (int32_t)taosArrayGetSize(pReq->pTableHash); int32_t udfNum = (int32_t)taosArrayGetSize(pReq->pUdf); int32_t qnodeNum = pReq->qNodeRequired ? 1 : 0; + int32_t dnodeNum = pReq->dNodeRequired ? 1 : 0; int32_t dbCfgNum = (int32_t)taosArrayGetSize(pReq->pDbCfg); int32_t indexNum = (int32_t)taosArrayGetSize(pReq->pIndex); int32_t userNum = (int32_t)taosArrayGetSize(pReq->pUser); int32_t dbInfoNum = (int32_t)taosArrayGetSize(pReq->pDbInfo); int32_t tbIndexNum = (int32_t)taosArrayGetSize(pReq->pTableIndex); + int32_t tbCfgNum = (int32_t)taosArrayGetSize(pReq->pTableCfg); - *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum; + *taskNum = tbMetaNum + dbVgNum + udfNum + tbHashNum + qnodeNum + dnodeNum + dbCfgNum + indexNum + userNum + dbInfoNum + tbIndexNum + tbCfgNum; if (*taskNum <= 0) { ctgDebug("Empty input for job, no need to retrieve meta, reqId:0x%" PRIx64, reqId); return TSDB_CODE_SUCCESS; @@ -382,6 +451,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 pJob->tbMetaNum = tbMetaNum; pJob->tbHashNum = tbHashNum; pJob->qnodeNum = qnodeNum; + pJob->dnodeNum = dnodeNum; pJob->dbVgNum = dbVgNum; pJob->udfNum = udfNum; pJob->dbCfgNum = dbCfgNum; @@ -389,6 +459,7 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 pJob->userNum = userNum; pJob->dbInfoNum = dbInfoNum; pJob->tbIndexNum = tbIndexNum; + pJob->tbCfgNum = tbCfgNum; pJob->pTasks = taosArrayInit(*taskNum, sizeof(SCtgTask)); @@ -401,54 +472,62 @@ int32_t ctgInitJob(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgJob** job, uint6 CTG_ERR_JRET(ctgHandleForceUpdate(pCtg, *taskNum, pJob, pReq)); } - int32_t taskIdx = 0; for (int32_t i = 0; i < dbVgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbVgroup, i); - CTG_ERR_JRET(ctgInitGetDbVgTask(pJob, taskIdx++, dbFName)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_VGROUP, dbFName, NULL)); } for (int32_t i = 0; i < dbCfgNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbCfg, i); - CTG_ERR_JRET(ctgInitGetDbCfgTask(pJob, taskIdx++, dbFName)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_CFG, dbFName, NULL)); } for (int32_t i = 0; i < dbInfoNum; ++i) { char* dbFName = taosArrayGet(pReq->pDbInfo, i); - CTG_ERR_JRET(ctgInitGetDbInfoTask(pJob, taskIdx++, dbFName)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DB_INFO, dbFName, NULL)); } for (int32_t i = 0; i < tbMetaNum; ++i) { SName* name = taosArrayGet(pReq->pTableMeta, i); - CTG_ERR_JRET(ctgInitGetTbMetaTask(pJob, taskIdx++, name)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_META, name, NULL)); } for (int32_t i = 0; i < tbHashNum; ++i) { SName* name = taosArrayGet(pReq->pTableHash, i); - CTG_ERR_JRET(ctgInitGetTbHashTask(pJob, taskIdx++, name)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_HASH, name, NULL)); } for (int32_t i = 0; i < tbIndexNum; ++i) { SName* name = taosArrayGet(pReq->pTableIndex, i); - CTG_ERR_JRET(ctgInitGetTbIndexTask(pJob, taskIdx++, name)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_INDEX, name, NULL)); + } + + for (int32_t i = 0; i < tbCfgNum; ++i) { + SName* name = taosArrayGet(pReq->pTableCfg, i); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_TB_CFG, name, NULL)); } for (int32_t i = 0; i < indexNum; ++i) { char* indexName = taosArrayGet(pReq->pIndex, i); - CTG_ERR_JRET(ctgInitGetIndexTask(pJob, taskIdx++, indexName)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_INDEX, indexName, NULL)); } for (int32_t i = 0; i < udfNum; ++i) { char* udfName = taosArrayGet(pReq->pUdf, i); - CTG_ERR_JRET(ctgInitGetUdfTask(pJob, taskIdx++, udfName)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_UDF, udfName, NULL)); } for (int32_t i = 0; i < userNum; ++i) { SUserAuthInfo* user = taosArrayGet(pReq->pUser, i); - CTG_ERR_JRET(ctgInitGetUserTask(pJob, taskIdx++, user)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_USER, user, NULL)); } if (qnodeNum) { - CTG_ERR_JRET(ctgInitGetQnodeTask(pJob, taskIdx++)); + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_QNODE, NULL, NULL)); + } + + if (dnodeNum) { + CTG_ERR_JRET(ctgInitTask(pJob, CTG_TASK_GET_DNODE, NULL, NULL)); } pJob->refId = taosAddRef(gCtgMgmt.jobPool, pJob); @@ -528,6 +607,21 @@ int32_t ctgDumpTbIndexRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpTbCfgRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pTableCfg) { + pJob->jobRes.pTableCfg = taosArrayInit(pJob->tbCfgNum, sizeof(SMetaRes)); + if (NULL == pJob->jobRes.pTableCfg) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; + taosArrayPush(pJob->jobRes.pTableCfg, &res); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgDumpIndexRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pIndex) { @@ -558,6 +652,22 @@ int32_t ctgDumpQnodeRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgDumpDnodeRes(SCtgTask* pTask) { + SCtgJob* pJob = pTask->pJob; + if (NULL == pJob->jobRes.pDnodeList) { + pJob->jobRes.pDnodeList = taosArrayInit(1, sizeof(SMetaRes)); + if (NULL == pJob->jobRes.pDnodeList) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + } + + SMetaRes res = {.code = pTask->code, .pRes = pTask->res}; + taosArrayPush(pJob->jobRes.pDnodeList, &res); + + return TSDB_CODE_SUCCESS; +} + + int32_t ctgDumpDbCfgRes(SCtgTask* pTask) { SCtgJob* pJob = pTask->pJob; if (NULL == pJob->jobRes.pDbCfg) { @@ -618,13 +728,48 @@ int32_t ctgDumpUserRes(SCtgTask* pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgInvokeSubCb(SCtgTask *pTask) { + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &pTask->lock); + + int32_t parentNum = taosArrayGetSize(pTask->pParents); + for (int32_t i = 0; i < parentNum; ++i) { + SCtgTask* pParent = taosArrayGetP(pTask->pParents, i); + + pParent->subRes.code = pTask->code; + if (TSDB_CODE_SUCCESS == pTask->code) { + code = (*gCtgAsyncFps[pTask->type].cloneFp)(pTask, &pParent->subRes.res); + if (code) { + pParent->subRes.code = code; + } + } + + CTG_ERR_JRET(pParent->subRes.fp(pParent)); + } + +_return: + + CTG_UNLOCK(CTG_WRITE, &pTask->lock); + + CTG_RET(code); +} + + int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { SCtgJob* pJob = pTask->pJob; int32_t code = 0; + if (CTG_TASK_DONE == pTask->status) { + return TSDB_CODE_SUCCESS; + } + qDebug("QID:0x%" PRIx64 " task %d end with res %s", pJob->queryId, pTask->taskId, tstrerror(rspCode)); pTask->code = rspCode; + pTask->status = CTG_TASK_DONE; + + ctgInvokeSubCb(pTask); int32_t taskDone = atomic_add_fetch_32(&pJob->taskDone, 1); if (taskDone < taosArrayGetSize(pJob->pTasks)) { @@ -636,7 +781,7 @@ int32_t ctgHandleTaskEnd(SCtgTask* pTask, int32_t rspCode) { _return: - qDebug("QID:0x%" PRIx64 " user callback with rsp %s", pJob->queryId, tstrerror(code)); + qDebug("QID:0x%" PRIx64 " ctg call user callback with rsp %s", pJob->queryId, tstrerror(code)); (*pJob->userFp)(&pJob->jobRes, pJob->userParam, code); @@ -802,11 +947,12 @@ int32_t ctgHandleGetDbVgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pM switch (reqType) { case TDMT_MND_USE_DB: { SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; - - CTG_ERR_JRET(ctgGenerateVgList(pCtg, pOut->dbVgroup->vgHash, (SArray**)&pTask->res)); + SDBVgInfo* pDb = NULL; - CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, ctx->dbFName, pOut->dbId, pOut->dbVgroup, false)); - pOut->dbVgroup = NULL; + CTG_ERR_JRET(ctgGenerateVgList(pCtg, pOut->dbVgroup->vgHash, (SArray**)&pTask->res)); + + CTG_ERR_JRET(cloneDbVgInfo(pOut->dbVgroup, &pDb)); + CTG_ERR_JRET(ctgUpdateVgroupEnqueue(pCtg, ctx->dbFName, pOut->dbId, pDb, false)); break; } @@ -874,6 +1020,7 @@ int32_t ctgHandleGetTbIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf CTG_ERR_JRET(ctgUpdateTbIndexEnqueue(pTask->pJob->pCtg, (STableIndex**)&pTask->msgCtx.out, false)); _return: + if (TSDB_CODE_MND_DB_INDEX_NOT_EXIST == code) { code = TSDB_CODE_SUCCESS; } @@ -882,6 +1029,18 @@ _return: CTG_RET(code); } +int32_t ctgHandleGetTbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} int32_t ctgHandleGetDbCfgRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; @@ -914,6 +1073,19 @@ _return: CTG_RET(code); } +int32_t ctgHandleGetDnodeRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { + int32_t code = 0; + CTG_ERR_JRET(ctgProcessRspMsg(&pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); + + TSWAP(pTask->res, pTask->msgCtx.out); + +_return: + + ctgHandleTaskEnd(pTask, code); + + CTG_RET(code); +} + int32_t ctgHandleGetIndexRsp(SCtgTask* pTask, int32_t reqType, const SDataBuf *pMsg, int32_t rspCode) { int32_t code = 0; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); @@ -1138,6 +1310,48 @@ int32_t ctgLaunchGetTbIndexTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgLaunchGetTbCfgTask(SCtgTask *pTask) { + int32_t code = 0; + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + SCtgTbCfgCtx* pCtx = (SCtgTbCfgCtx*)pTask->taskCtx; + SArray* pRes = NULL; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pCtx->pName, dbFName); + + if (pCtx->tbType <= 0) { + CTG_ERR_JRET(ctgReadTbTypeFromCache(pCtg, dbFName, pCtx->pName->tname, &pCtx->tbType)); + if (pCtx->tbType <= 0) { + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, pCtx->pName)); + return TSDB_CODE_SUCCESS; + } + } + + if (TSDB_SUPER_TABLE == pCtx->tbType) { + CTG_ERR_JRET(ctgGetTableCfgFromMnode(pCtg, pConn, pCtx->pName, NULL, pTask)); + } else { + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(ctgGetTbHashVgroupFromCache(pCtg, pCtx->pName, &pCtx->pVgInfo)); + if (NULL == pCtx->pVgInfo) { + CTG_ERR_JRET(ctgLaunchSubTask(pTask, CTG_TASK_GET_DB_VGROUP, ctgGetTbCfgCb, dbFName)); + return TSDB_CODE_SUCCESS; + } + } + + CTG_ERR_JRET(ctgGetTableCfgFromVnode(pCtg, pConn, pCtx->pName, pCtx->pVgInfo, NULL, pTask)); + } + + return TSDB_CODE_SUCCESS; + +_return: + + if (CTG_TASK_LAUNCHED == pTask->status) { + ctgHandleTaskEnd(pTask, code); + } + + CTG_RET(code); +} + int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; @@ -1147,6 +1361,15 @@ int32_t ctgLaunchGetQnodeTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgLaunchGetDnodeTask(SCtgTask *pTask) { + SCatalog* pCtg = pTask->pJob->pCtg; + SRequestConnInfo* pConn = &pTask->pJob->conn; + + CTG_ERR_RET(ctgGetDnodeListFromMnode(pCtg, pConn, NULL, pTask)); + return TSDB_CODE_SUCCESS; +} + + int32_t ctgLaunchGetDbCfgTask(SCtgTask *pTask) { SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; @@ -1244,17 +1467,71 @@ int32_t ctgRelaunchGetTbMetaTask(SCtgTask *pTask) { return TSDB_CODE_SUCCESS; } +int32_t ctgGetTbCfgCb(SCtgTask *pTask) { + int32_t code = 0; + + CTG_ERR_JRET(pTask->subRes.code); + + SCtgTbCfgCtx* pCtx = (SCtgTbCfgCtx*)pTask->taskCtx; + if (CTG_TASK_GET_TB_META == pTask->subRes.type) { + pCtx->tbType = ((STableMeta*)pTask->subRes.res)->tableType; + } else if (CTG_TASK_GET_DB_VGROUP == pTask->subRes.type) { + SDBVgInfo* pDb = (SDBVgInfo*)pTask->subRes.res; + + pCtx->pVgInfo = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pTask->pJob->pCtg, pDb, pCtx->pName, pCtx->pVgInfo)); + } + + CTG_RET(ctgLaunchGetTbCfgTask(pTask)); + +_return: + + CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); +} + +int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { + SCtgDbVgCtx* ctx = pTask->taskCtx; + + *equal = (0 == strcmp(ctx->dbFName, param)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgCompTbMetaTasks(SCtgTask* pTask, void* param, bool* equal) { + SCtgTbMetaCtx* ctx = pTask->taskCtx; + + *equal = tNameTbNameEqual(ctx->pName, (SName*)param); + + return TSDB_CODE_SUCCESS; +} + +int32_t ctgCloneTbMeta(SCtgTask* pTask, void** pRes) { + STableMeta* pMeta = (STableMeta*)pTask->res; + + CTG_RET(cloneTableMeta(pMeta, (STableMeta**)pRes)); +} + +int32_t ctgCloneDbVg(SCtgTask* pTask, void** pRes) { + SUseDbOutput* pOut = (SUseDbOutput*)pTask->msgCtx.out; + + CTG_RET(cloneDbVgInfo(pOut->dbVgroup, (SDBVgInfo**)pRes)); +} + + SCtgAsyncFps gCtgAsyncFps[] = { - {ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes}, - {ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes}, - {ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes}, - {ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes}, - {ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes}, - {ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes}, - {ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes}, - {ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes}, - {ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes}, - {ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes}, + {ctgInitGetQnodeTask, ctgLaunchGetQnodeTask, ctgHandleGetQnodeRsp, ctgDumpQnodeRes, NULL, NULL}, + {ctgInitGetDnodeTask, ctgLaunchGetDnodeTask, ctgHandleGetDnodeRsp, ctgDumpDnodeRes, NULL, NULL}, + {ctgInitGetDbVgTask, ctgLaunchGetDbVgTask, ctgHandleGetDbVgRsp, ctgDumpDbVgRes, ctgCompDbVgTasks, ctgCloneDbVg}, + {ctgInitGetDbCfgTask, ctgLaunchGetDbCfgTask, ctgHandleGetDbCfgRsp, ctgDumpDbCfgRes, NULL, NULL}, + {ctgInitGetDbInfoTask, ctgLaunchGetDbInfoTask, ctgHandleGetDbInfoRsp, ctgDumpDbInfoRes, NULL, NULL}, + {ctgInitGetTbMetaTask, ctgLaunchGetTbMetaTask, ctgHandleGetTbMetaRsp, ctgDumpTbMetaRes, ctgCompTbMetaTasks, ctgCloneTbMeta}, + {ctgInitGetTbHashTask, ctgLaunchGetTbHashTask, ctgHandleGetTbHashRsp, ctgDumpTbHashRes, NULL, NULL}, + {ctgInitGetTbIndexTask, ctgLaunchGetTbIndexTask, ctgHandleGetTbIndexRsp, ctgDumpTbIndexRes, NULL, NULL}, + {ctgInitGetTbCfgTask, ctgLaunchGetTbCfgTask, ctgHandleGetTbCfgRsp, ctgDumpTbCfgRes, NULL, NULL}, + {ctgInitGetIndexTask, ctgLaunchGetIndexTask, ctgHandleGetIndexRsp, ctgDumpIndexRes, NULL, NULL}, + {ctgInitGetUdfTask, ctgLaunchGetUdfTask, ctgHandleGetUdfRsp, ctgDumpUdfRes, NULL, NULL}, + {ctgInitGetUserTask, ctgLaunchGetUserTask, ctgHandleGetUserRsp, ctgDumpUserRes, NULL, NULL}, }; int32_t ctgMakeAsyncRes(SCtgJob *pJob) { @@ -1269,6 +1546,86 @@ int32_t ctgMakeAsyncRes(SCtgJob *pJob) { return TSDB_CODE_SUCCESS; } +int32_t ctgSearchExistingTask(SCtgJob *pJob, CTG_TASK_TYPE type, void* param, int32_t* taskId) { + bool equal = false; + SCtgTask* pTask = NULL; + int32_t code = 0; + + CTG_LOCK(CTG_READ, &pJob->taskLock); + + int32_t taskNum = taosArrayGetSize(pJob->pTasks); + for (int32_t i = 0; i < taskNum; ++i) { + pTask = taosArrayGet(pJob->pTasks, i); + if (type != pTask->type) { + continue; + } + + CTG_ERR_JRET((*gCtgAsyncFps[type].compFp)(pTask, param, &equal)); + if (equal) { + break; + } + } + +_return: + + CTG_UNLOCK(CTG_READ, &pJob->taskLock); + if (equal) { + *taskId = pTask->taskId; + } + + CTG_RET(code); +} + +int32_t ctgSetSubTaskCb(SCtgTask *pSub, SCtgTask *pTask) { + int32_t code = 0; + + CTG_LOCK(CTG_WRITE, &pSub->lock); + if (CTG_TASK_DONE == pSub->status) { + pTask->subRes.code = pSub->code; + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].cloneFp)(pSub, &pTask->subRes.res)); + CTG_ERR_JRET(pTask->subRes.fp(pTask)); + } else { + if (NULL == pSub->pParents) { + pSub->pParents = taosArrayInit(4, POINTER_BYTES); + } + + taosArrayPush(pSub->pParents, &pTask); + } + +_return: + + CTG_UNLOCK(CTG_WRITE, &pSub->lock); + + CTG_RET(code); +} + + +int32_t ctgLaunchSubTask(SCtgTask *pTask, CTG_TASK_TYPE type, ctgSubTaskCbFp fp, void* param) { + SCtgJob* pJob = pTask->pJob; + int32_t subTaskId = -1; + bool newTask = false; + + ctgClearSubTaskRes(&pTask->subRes); + pTask->subRes.type = type; + pTask->subRes.fp = fp; + + CTG_ERR_RET(ctgSearchExistingTask(pJob, type, param, &subTaskId)); + if (subTaskId < 0) { + CTG_ERR_RET(ctgInitTask(pJob, type, param, &subTaskId)); + newTask = true; + } + + SCtgTask* pSub = taosArrayGet(pJob->pTasks, subTaskId); + + CTG_ERR_RET(ctgSetSubTaskCb(pSub, pTask)); + + if (newTask) { + CTG_ERR_RET((*gCtgAsyncFps[pSub->type].launchFp)(pSub)); + pSub->status = CTG_TASK_LAUNCHED; + } + + return TSDB_CODE_SUCCESS; +} int32_t ctgLaunchJob(SCtgJob *pJob) { int32_t taskNum = taosArrayGetSize(pJob->pTasks); @@ -1278,6 +1635,7 @@ int32_t ctgLaunchJob(SCtgJob *pJob) { qDebug("QID:0x%" PRIx64 " ctg start to launch task %d", pJob->queryId, pTask->taskId); CTG_ERR_RET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); + pTask->status = CTG_TASK_LAUNCHED; } return TSDB_CODE_SUCCESS; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index 1de5ee3d7d9ea6c3ce24b6890e12cfc0cb9c75d8..eeb627624b5b0eabb34df92f4362c0c1ddff18da 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -326,6 +326,7 @@ _return: int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, SCtgDBCache **pDb, SCtgTbCache** pTb) { SCtgDBCache *dbCache = NULL; + SCtgTbCache* pCache = NULL; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { ctgDebug("db %s not in cache", dbFName); @@ -333,7 +334,7 @@ int32_t ctgAcquireTbIndexFromCache(SCatalog* pCtg, char *dbFName, char* tbName, } int32_t sz = 0; - SCtgTbCache* pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); + pCache = taosHashAcquire(dbCache->tbCache, tbName, strlen(tbName)); if (NULL == pCache) { ctgDebug("tb %s not in cache, dbFName:%s", tbName, dbFName); goto _return; @@ -540,10 +541,10 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, } -int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, int32_t *tbType) { +int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tbName, int32_t *tbType) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tableName, &dbCache, &tbCache)); + CTG_ERR_RET(ctgAcquireTbMetaFromCache(pCtg, dbFName, tbName, &dbCache, &tbCache)); if (NULL == tbCache) { ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); return TSDB_CODE_SUCCESS; @@ -552,7 +553,7 @@ int32_t ctgReadTbTypeFromCache(SCatalog* pCtg, char* dbFName, char *tableName, i *tbType = tbCache->pMeta->tableType; ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); - ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tableName, *tbType, dbFName); + ctgDebug("Got tb %s tbType %d from cache, dbFName:%s", tbName, *tbType, dbFName); return TSDB_CODE_SUCCESS; } @@ -1394,7 +1395,7 @@ int32_t ctgWriteTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNam if (orig) { origType = orig->tableType; - if (origType == meta->tableType && orig->uid == meta->uid && orig->sversion >= meta->sversion && orig->tversion >= meta->tversion) { + if (origType == meta->tableType && orig->uid == meta->uid && (origType == TSDB_CHILD_TABLE || (orig->sversion >= meta->sversion && orig->tversion >= meta->tversion))) { taosMemoryFree(meta); ctgDebug("ignore table %s meta update", tbName); return TSDB_CODE_SUCCESS; @@ -2060,4 +2061,92 @@ int32_t ctgStartUpdateThread() { } +int32_t ctgGetTbMetaFromCache(SCatalog* pCtg, SRequestConnInfo *pConn, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta) { + if (CTG_IS_SYS_DBNAME(ctx->pName->dbname)) { + CTG_FLAG_SET_SYS_DB(ctx->flag); + } + + CTG_ERR_RET(ctgReadTbMetaFromCache(pCtg, ctx, pTableMeta)); + + if (*pTableMeta) { + if (CTG_FLAG_MATCH_STB(ctx->flag, (*pTableMeta)->tableType) && + ((!CTG_FLAG_IS_FORCE_UPDATE(ctx->flag)) || (CTG_FLAG_IS_SYS_DB(ctx->flag)))) { + return TSDB_CODE_SUCCESS; + } + + taosMemoryFreeClear(*pTableMeta); + } + + if (CTG_FLAG_IS_UNKNOWN_STB(ctx->flag)) { + CTG_FLAG_SET_STB(ctx->flag, ctx->tbInfo.tbType); + } + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgRemoveTbMetaFromCache(SCatalog* pCtg, SName* pTableName, bool syncReq) { + int32_t code = 0; + STableMeta* tblMeta = NULL; + SCtgTbMetaCtx tbCtx = {0}; + tbCtx.flag = CTG_FLAG_UNKNOWN_STB; + tbCtx.pName = pTableName; + + CTG_ERR_JRET(ctgReadTbMetaFromCache(pCtg, &tbCtx, &tblMeta)); + + if (NULL == tblMeta) { + ctgDebug("table already not in cache, db:%s, tblName:%s", pTableName->dbname, pTableName->tname); + return TSDB_CODE_SUCCESS; + } + + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + + if (TSDB_SUPER_TABLE == tblMeta->tableType) { + CTG_ERR_JRET(ctgDropStbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, tblMeta->suid, syncReq)); + } else { + CTG_ERR_JRET(ctgDropTbMetaEnqueue(pCtg, dbFName, tbCtx.tbInfo.dbId, pTableName->tname, syncReq)); + } + +_return: + + taosMemoryFreeClear(tblMeta); + + CTG_RET(code); +} + +int32_t ctgGetTbHashVgroupFromCache(SCatalog *pCtg, const SName *pTableName, SVgroupInfo **pVgroup) { + if (CTG_IS_SYS_DBNAME(pTableName->dbname)) { + ctgError("no valid vgInfo for db, dbname:%s", pTableName->dbname); + CTG_ERR_RET(TSDB_CODE_CTG_INVALID_INPUT); + } + + SCtgDBCache* dbCache = NULL; + int32_t code = 0; + char dbFName[TSDB_DB_FNAME_LEN] = {0}; + tNameGetFullDbName(pTableName, dbFName); + + CTG_ERR_RET(ctgAcquireVgInfoFromCache(pCtg, dbFName, &dbCache)); + + if (NULL == dbCache) { + *pVgroup = NULL; + return TSDB_CODE_SUCCESS; + } + + *pVgroup = taosMemoryCalloc(1, sizeof(SVgroupInfo)); + CTG_ERR_JRET(ctgGetVgInfoFromHashValue(pCtg, dbCache->vgCache.vgInfo, pTableName, *pVgroup)); + +_return: + + if (dbCache) { + ctgReleaseVgInfoToCache(pCtg, dbCache); + } + + if (code) { + taosMemoryFreeClear(*pVgroup); + } + + CTG_RET(code); +} + diff --git a/source/libs/catalog/src/ctgRemote.c b/source/libs/catalog/src/ctgRemote.c index fa1a26283286d01cc4574c4babbb28ad5fb36996..97edb1b837d3a226fe5baabf52efc0fbf5419fc0 100644 --- a/source/libs/catalog/src/ctgRemote.c +++ b/source/libs/catalog/src/ctgRemote.c @@ -40,6 +40,21 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, qDebug("Got qnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(out)); break; } + case TDMT_MND_DNODE_LIST: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for dnode list, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process dnode list rsp failed, error:%s", tstrerror(rspCode)); + CTG_ERR_RET(code); + } + + qDebug("Got dnode list from mnode, listNum:%d", (int32_t)taosArrayGetSize(*(SArray**)out)); + break; + } case TDMT_MND_USE_DB: { if (TSDB_CODE_SUCCESS != rspCode) { qError("error rsp for use db, error:%s, dbFName:%s", tstrerror(rspCode), target); @@ -172,6 +187,39 @@ int32_t ctgProcessRspMsg(void* out, int32_t reqType, char* msg, int32_t msgSize, qDebug("Got table meta from vnode, tbFName:%s", target); break; } + case TDMT_VND_TABLE_CFG: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for table cfg from vnode, code:%s, tbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process vnode tb cfg rsp failed, code:%s, tbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got table cfg from vnode, tbFName:%s", target); + break; + } + case TDMT_MND_TABLE_CFG: { + if (TSDB_CODE_SUCCESS != rspCode) { + qError("error rsp for stb cfg from mnode, error:%s, tbFName:%s", tstrerror(rspCode), target); + CTG_ERR_RET(rspCode); + } + + code = queryProcessMsgRsp[TMSG_INDEX(reqType)](out, msg, msgSize); + if (code) { + qError("Process mnode stb cfg rsp failed, error:%s, tbFName:%s", tstrerror(code), target); + CTG_ERR_RET(code); + } + + qDebug("Got stb cfg from mnode, tbFName:%s", target); + break; + } + default: + qError("invalid req type %s", TMSG_INFO(reqType)); + return TSDB_CODE_APP_ERROR; } return TSDB_CODE_SUCCESS; @@ -276,9 +324,6 @@ _return: CTG_RET(code); } - - - int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray *out, SCtgTask* pTask) { char *msg = NULL; int32_t msgLen = 0; @@ -316,6 +361,39 @@ int32_t ctgGetQnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray return TSDB_CODE_SUCCESS; } +int32_t ctgGetDnodeListFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SArray **out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_DNODE_LIST; + void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + + ctgDebug("try to get dnode list from mnode, mgmtEpInUse:%d", pConn->mgmtEps.inUse); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](NULL, &msg, 0, &msgLen, mallocFp); + if (code) { + ctgError("Build dnode list msg failed, error:%s", tstrerror(code)); + CTG_ERR_RET(code); + } + + if (pTask) { + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, NULL)); + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, NULL)); + + return TSDB_CODE_SUCCESS; +} + int32_t ctgGetDBVgInfoFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, SBuildUseDBInput *input, SUseDbOutput *out, SCtgTask* pTask) { char *msg = NULL; @@ -550,7 +628,7 @@ int32_t ctgGetUserDbAuthFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const int32_t ctgGetTbMetaFromMnodeImpl(SCatalog* pCtg, SRequestConnInfo *pConn, char *dbFName, char* tbName, STableMetaOutput* out, SCtgTask* pTask) { - SBuildTableMetaInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; + SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = tbName}; char *msg = NULL; SEpSet *pVnodeEpSet = NULL; int32_t msgLen = 0; @@ -606,9 +684,11 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa sprintf(tbFName, "%s.%s", dbFName, pTableName->tname); void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; - ctgDebug("try to get table meta from vnode, vgId:%d, tbFName:%s", vgroupInfo->vgId, tbFName); + SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; + ctgDebug("try to get table meta from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", + vgroupInfo->vgId, vgroupInfo->epSet.numOfEps, pEp->fqdn, pEp->port, tbFName); - SBuildTableMetaInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; + SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char *)tNameGetTableName(pTableName)}; char *msg = NULL; int32_t msgLen = 0; @@ -646,4 +726,89 @@ int32_t ctgGetTbMetaFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SNa return TSDB_CODE_SUCCESS; } +int32_t ctgGetTableCfgFromVnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, SVgroupInfo *vgroupInfo, STableCfg **out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_VND_TABLE_CFG; + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFName); + void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + SBuildTableInput bInput = {.vgId = vgroupInfo->vgId, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; + + SEp* pEp = &vgroupInfo->epSet.eps[vgroupInfo->epSet.inUse]; + ctgDebug("try to get table cfg from vnode, vgId:%d, ep num:%d, ep %s:%d, tbFName:%s", + vgroupInfo->vgId, vgroupInfo->epSet.numOfEps, pEp->fqdn, pEp->port, tbFName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen, mallocFp); + if (code) { + ctgError("Build get tb cfg msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); + CTG_ERR_RET(code); + } + + if (pTask) { + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + + SRequestConnInfo vConn = {.pTrans = pConn->pTrans, + .requestId = pConn->requestId, + .requestObjRefId = pConn->requestObjRefId, + .mgmtEps = vgroupInfo->epSet}; + CTG_RET(ctgAsyncSendMsg(pCtg, &vConn, pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pConn->pTrans, &vgroupInfo->epSet, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); + + return TSDB_CODE_SUCCESS; +} + + +int32_t ctgGetTableCfgFromMnode(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg **out, SCtgTask* pTask) { + char *msg = NULL; + int32_t msgLen = 0; + int32_t reqType = TDMT_MND_TABLE_CFG; + char tbFName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pTableName, tbFName); + void*(*mallocFp)(int32_t) = pTask ? taosMemoryMalloc : rpcMallocCont; + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(pTableName, dbFName); + SBuildTableInput bInput = {.vgId = 0, .dbFName = dbFName, .tbName = (char*)pTableName->tname}; + + ctgDebug("try to get table cfg from mnode, tbFName:%s", tbFName); + + int32_t code = queryBuildMsg[TMSG_INDEX(reqType)](&bInput, &msg, 0, &msgLen, mallocFp); + if (code) { + ctgError("Build get tb cfg msg failed, code:%s, tbFName:%s", tstrerror(code), tbFName); + CTG_ERR_RET(code); + } + + if (pTask) { + CTG_ERR_RET(ctgUpdateMsgCtx(&pTask->msgCtx, reqType, NULL, (char*)tbFName)); + + CTG_RET(ctgAsyncSendMsg(pCtg, pConn, pTask, reqType, msg, msgLen)); + } + + SRpcMsg rpcMsg = { + .msgType = reqType, + .pCont = msg, + .contLen = msgLen, + }; + + SRpcMsg rpcRsp = {0}; + rpcSendRecv(pConn->pTrans, &pConn->mgmtEps, &rpcMsg, &rpcRsp); + + CTG_ERR_RET(ctgProcessRspMsg(out, reqType, rpcRsp.pCont, rpcRsp.contLen, rpcRsp.code, (char*)tbFName)); + + return TSDB_CODE_SUCCESS; +} + diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index 476eb371b057e27a2420dd6b35582b58db8aca72..cc823adad00e47505e8dd49a0a98fbfab2919c86 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -23,6 +23,8 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { switch (type) { case CTG_TASK_GET_QNODE: return "[get qnode list]"; + case CTG_TASK_GET_DNODE: + return "[get dnode list]"; case CTG_TASK_GET_DB_VGROUP: return "[get db vgroup]"; case CTG_TASK_GET_DB_CFG: @@ -33,6 +35,10 @@ char *ctgTaskTypeStr(CTG_TASK_TYPE type) { return "[get table meta]"; case CTG_TASK_GET_TB_HASH: return "[get table hash]"; + case CTG_TASK_GET_TB_INDEX: + return "[get table index]"; + case CTG_TASK_GET_TB_CFG: + return "[get table cfg]"; case CTG_TASK_GET_INDEX: return "[get index]"; case CTG_TASK_GET_UDF: @@ -96,6 +102,9 @@ void ctgFreeSMetaData(SMetaData* pData) { taosArrayDestroy(pData->pQnodeList); pData->pQnodeList = NULL; + + taosArrayDestroy(pData->pTableCfg); + pData->pTableCfg = NULL; } void ctgFreeSCtgUserAuth(SCtgUserAuth *userCache) { @@ -280,6 +289,13 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { } break; } + case TDMT_VND_TABLE_CFG: + case TDMT_MND_TABLE_CFG: { + STableCfgRsp* pOut = (STableCfgRsp*)pCtx->out; + tFreeSTableCfgRsp(pOut); + taosMemoryFreeClear(pCtx->out); + break; + } case TDMT_MND_RETRIEVE_FUNC: { SFuncInfo* pOut = (SFuncInfo*)pCtx->out; taosMemoryFree(pOut->pCode); @@ -328,14 +344,161 @@ void ctgResetTbMetaTask(SCtgTask* pTask) { taosMemoryFreeClear(pTask->res); } -void ctgFreeTask(SCtgTask* pTask) { - ctgFreeMsgCtx(&pTask->msgCtx); - +void ctgFreeTaskRes(CTG_TASK_TYPE type, void **pRes) { + switch (type) { + case CTG_TASK_GET_QNODE: { + taosArrayDestroy((SArray*)*pRes); + *pRes = NULL; + break; + } + case CTG_TASK_GET_DNODE: { + taosArrayDestroy((SArray*)*pRes); + *pRes = NULL; + break; + } + case CTG_TASK_GET_TB_META: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_DB_VGROUP: { + taosArrayDestroy((SArray*)*pRes); + *pRes = NULL; + break; + } + case CTG_TASK_GET_DB_CFG: { + if (*pRes) { + SDbCfgInfo* pInfo = (SDbCfgInfo*)*pRes; + taosArrayDestroy(pInfo->pRetensions); + taosMemoryFreeClear(*pRes); + } + break; + } + case CTG_TASK_GET_DB_INFO: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_TB_HASH: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_TB_INDEX: { + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + break; + } + case CTG_TASK_GET_TB_CFG: { + if (*pRes) { + STableCfg* pInfo = (STableCfg*)*pRes; + tFreeSTableCfgRsp(pInfo); + taosMemoryFreeClear(*pRes); + } + break; + } + case CTG_TASK_GET_INDEX: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_UDF: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_USER: { + taosMemoryFreeClear(*pRes); + break; + } + default: + qError("invalid task type %d", type); + break; + } +} + + +void ctgFreeSubTaskRes(CTG_TASK_TYPE type, void **pRes) { + switch (type) { + case CTG_TASK_GET_QNODE: { + taosArrayDestroy((SArray*)*pRes); + *pRes = NULL; + break; + } + case CTG_TASK_GET_DNODE: { + taosArrayDestroy((SArray*)*pRes); + *pRes = NULL; + break; + } + case CTG_TASK_GET_TB_META: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_DB_VGROUP: { + if (*pRes) { + SDBVgInfo* pInfo = (SDBVgInfo*)*pRes; + taosHashCleanup(pInfo->vgHash); + taosMemoryFreeClear(*pRes); + } + break; + } + case CTG_TASK_GET_DB_CFG: { + if (*pRes) { + SDbCfgInfo* pInfo = (SDbCfgInfo*)*pRes; + taosArrayDestroy(pInfo->pRetensions); + taosMemoryFreeClear(*pRes); + } + break; + } + case CTG_TASK_GET_DB_INFO: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_TB_HASH: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_TB_INDEX: { + taosArrayDestroyEx(*pRes, tFreeSTableIndexInfo); + *pRes = NULL; + break; + } + case CTG_TASK_GET_TB_CFG: { + if (*pRes) { + STableCfg* pInfo = (STableCfg*)*pRes; + tFreeSTableCfgRsp(pInfo); + taosMemoryFreeClear(*pRes); + } + break; + } + case CTG_TASK_GET_INDEX: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_UDF: { + taosMemoryFreeClear(*pRes); + break; + } + case CTG_TASK_GET_USER: { + taosMemoryFreeClear(*pRes); + break; + } + default: + qError("invalid task type %d", type); + break; + } +} + + +void ctgClearSubTaskRes(SCtgSubRes *pRes) { + pRes->code = 0; + + if (NULL == pRes->res) { + return; + } + + ctgFreeSubTaskRes(pRes->type, &pRes->res); +} + +void ctgFreeTaskCtx(SCtgTask* pTask) { switch (pTask->type) { case CTG_TASK_GET_QNODE: { - taosArrayDestroy((SArray*)pTask->res); taosMemoryFreeClear(pTask->taskCtx); - pTask->res = NULL; break; } case CTG_TASK_GET_TB_META: { @@ -346,56 +509,49 @@ void ctgFreeTask(SCtgTask* pTask) { pTask->msgCtx.lastOut = NULL; } taosMemoryFreeClear(pTask->taskCtx); - taosMemoryFreeClear(pTask->res); break; } case CTG_TASK_GET_DB_VGROUP: { - taosArrayDestroy((SArray*)pTask->res); taosMemoryFreeClear(pTask->taskCtx); - pTask->res = NULL; break; } case CTG_TASK_GET_DB_CFG: { taosMemoryFreeClear(pTask->taskCtx); - if (pTask->res) { - SDbCfgInfo* pInfo = (SDbCfgInfo*)pTask->res; - taosArrayDestroy(pInfo->pRetensions); - taosMemoryFreeClear(pTask->res); - } break; } case CTG_TASK_GET_DB_INFO: { taosMemoryFreeClear(pTask->taskCtx); - taosMemoryFreeClear(pTask->res); break; } case CTG_TASK_GET_TB_HASH: { SCtgTbHashCtx* taskCtx = (SCtgTbHashCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); taosMemoryFreeClear(pTask->taskCtx); - taosMemoryFreeClear(pTask->res); break; } case CTG_TASK_GET_TB_INDEX: { SCtgTbIndexCtx* taskCtx = (SCtgTbIndexCtx*)pTask->taskCtx; taosMemoryFreeClear(taskCtx->pName); taosMemoryFreeClear(pTask->taskCtx); - taosArrayDestroyEx(pTask->res, tFreeSTableIndexInfo); + break; + } + case CTG_TASK_GET_TB_CFG: { + SCtgTbCfgCtx* taskCtx = (SCtgTbCfgCtx*)pTask->taskCtx; + taosMemoryFreeClear(taskCtx->pName); + taosMemoryFreeClear(taskCtx->pVgInfo); + taosMemoryFreeClear(pTask->taskCtx); break; } case CTG_TASK_GET_INDEX: { taosMemoryFreeClear(pTask->taskCtx); - taosMemoryFreeClear(pTask->res); break; } case CTG_TASK_GET_UDF: { taosMemoryFreeClear(pTask->taskCtx); - taosMemoryFreeClear(pTask->res); break; } case CTG_TASK_GET_USER: { taosMemoryFreeClear(pTask->taskCtx); - taosMemoryFreeClear(pTask->res); break; } default: @@ -404,6 +560,16 @@ void ctgFreeTask(SCtgTask* pTask) { } } + +void ctgFreeTask(SCtgTask* pTask) { + ctgFreeMsgCtx(&pTask->msgCtx); + ctgFreeTaskRes(pTask->type, &pTask->res); + ctgFreeTaskCtx(pTask); + + taosArrayDestroy(pTask->pParents); + ctgClearSubTaskRes(&pTask->subRes); +} + void ctgFreeTasks(SArray* pArray) { if (NULL == pArray) { return; diff --git a/source/libs/command/src/command.c b/source/libs/command/src/command.c index c6a0e52a42f6238c7ee3b2cd3af714408debafa2..8d01ed731593d90bd9c32439f155be57db1f33bd 100644 --- a/source/libs/command/src/command.c +++ b/source/libs/command/src/command.c @@ -16,7 +16,9 @@ #include "command.h" #include "catalog.h" #include "tdatablock.h" +#include "tglobal.h" +extern SConfig *tsCfg; static int32_t getSchemaBytes(const SSchema* pSchema) { switch (pSchema->type) { case TSDB_DATA_TYPE_BINARY: @@ -30,31 +32,19 @@ static int32_t getSchemaBytes(const SSchema* pSchema) { } static SSDataBlock* buildDescResultDataBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->info.numOfCols = DESCRIBE_RESULT_COLS; - pBlock->info.hasVarCol = true; + SSDataBlock* pBlock = createDataBlock(); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_FIELD_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_FIELD_LEN; + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_TYPE_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_TYPE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_INT; - infoData.info.bytes = tDataTypes[TSDB_DATA_TYPE_INT].bytes; - - taosArrayPush(pBlock->pDataBlock, &infoData); - - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = DESCRIBE_RESULT_NOTE_LEN; - taosArrayPush(pBlock->pDataBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_INT, tDataTypes[TSDB_DATA_TYPE_INT].bytes, 3); + blockDataAppendColInfo(pBlock, &infoData); + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, DESCRIBE_RESULT_NOTE_LEN, 4); + blockDataAppendColInfo(pBlock, &infoData); return pBlock; } @@ -123,14 +113,494 @@ static int32_t execDescribe(SNode* pStmt, SRetrieveTableRsp** pRsp) { static int32_t execResetQueryCache() { return catalogClearCache(); } -static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt) { return TSDB_CODE_FAILED; } -static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt) { return TSDB_CODE_FAILED; } +static SSDataBlock* buildCreateDBResultDataBlock() { + SSDataBlock* pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_COLS, 1); + blockDataAppendColInfo(pBlock, &infoData); + + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_DB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); + return pBlock; +} + +int64_t getValOfDiffPrecision(int8_t unit, int64_t val) { + int64_t v = 0; + switch (unit) { + case 's': + v = val / 1000; + break; + case 'm': + v = val / tsTickPerMin[TSDB_TIME_PRECISION_MILLI]; + break; + case 'h': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 60); + break; + case 'd': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60); + break; + case 'w': + v = val / (tsTickPerMin[TSDB_TIME_PRECISION_MILLI] * 24 * 60 * 7); + break; + default: + break; + } + + return v; +} + +char *buildRetension(SArray *pRetension) { + size_t size = taosArrayGetSize(pRetension); + if (size == 0) { + return NULL; + } + + char *p1 = taosMemoryCalloc(1, 100); + SRetention *p = taosArrayGet(pRetension, 0); + + int32_t len = 0; + + int64_t v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + int64_t v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + + if (size > 1) { + len += sprintf(p1 + len, ","); + p = taosArrayGet(pRetension, 1); + + v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + } + + if (size > 2) { + len += sprintf(p1 + len, ","); + p = taosArrayGet(pRetension, 2); + + v1 = getValOfDiffPrecision(p->freqUnit, p->freq); + v2 = getValOfDiffPrecision(p->keepUnit, p->keep); + len += sprintf(p1 + len, "%" PRId64 "%c:%" PRId64 "%c", v1, p->freqUnit, v2, p->keepUnit); + } + + return p1; +} + + +static void setCreateDBResultIntoDataBlock(SSDataBlock* pBlock, char *dbFName, SDbCfgInfo* pCfg) { + blockDataEnsureCapacity(pBlock, 1); + pBlock->info.rows = 1; + + SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); + char buf1[SHOW_CREATE_DB_RESULT_FIELD1_LEN] = {0}; + STR_TO_VARSTR(buf1, dbFName); + colDataAppend(pCol1, 0, buf1, false); + + SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); + char buf2[SHOW_CREATE_DB_RESULT_FIELD2_LEN] = {0}; + int32_t len = 0; + char *prec = NULL; + switch (pCfg->precision) { + case TSDB_TIME_PRECISION_MILLI: + prec = TSDB_TIME_PRECISION_MILLI_STR; + break; + case TSDB_TIME_PRECISION_MICRO: + prec = TSDB_TIME_PRECISION_MICRO_STR; + break; + case TSDB_TIME_PRECISION_NANO: + prec = TSDB_TIME_PRECISION_NANO_STR; + break; + default: + prec = "none"; + break; + } + + char *retentions = buildRetension(pCfg->pRetensions); + + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE DATABASE `%s` BUFFER %d CACHELAST %d COMP %d DURATION %dm " + "FSYNC %d MAXROWS %d MINROWS %d KEEP %dm,%dm,%dm PAGES %d PAGESIZE %d PRECISION '%s' REPLICA %d " + "STRICT %d WAL %d VGROUPS %d SINGLE_STABLE %d", + dbFName, pCfg->buffer, pCfg->cacheLastRow, pCfg->compression, pCfg->daysPerFile, + pCfg->fsyncPeriod, pCfg->maxRows, pCfg->minRows, pCfg->daysToKeep0, pCfg->daysToKeep1, pCfg->daysToKeep2, + pCfg->pages, pCfg->pageSize, prec, pCfg->replications, pCfg->strict, pCfg->walLevel, pCfg->numOfVgroups, + 1 == pCfg->numOfStables); + + if (retentions) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, " RETENTIONS %s", retentions); + taosMemoryFree(retentions); + } + + (varDataLen(buf2)) = len; + + colDataAppend(pCol2, 0, buf2, false); +} + + +static int32_t execShowCreateDatabase(SShowCreateDatabaseStmt* pStmt, SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = buildCreateDBResultDataBlock(); + setCreateDBResultIntoDataBlock(pBlock, pStmt->dbName, pStmt->pCfg); + + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + *pRsp = taosMemoryCalloc(1, rspSize); + if (NULL == *pRsp) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pRsp)->useconds = 0; + (*pRsp)->completed = 1; + (*pRsp)->precision = 0; + (*pRsp)->compressed = 0; + (*pRsp)->compLen = 0; + (*pRsp)->numOfRows = htonl(1); + (*pRsp)->numOfCols = htonl(SHOW_CREATE_DB_RESULT_COLS); + + int32_t len = 0; + blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_DB_RESULT_COLS, false); + ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); + + blockDataDestroy(pBlock); + return TSDB_CODE_SUCCESS; +} + +static SSDataBlock* buildCreateTbResultDataBlock() { + SSDataBlock* pBlock = createDataBlock(); + + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD1_LEN, 1); + blockDataAppendColInfo(pBlock, &infoData); + + infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, SHOW_CREATE_TB_RESULT_FIELD2_LEN, 2); + blockDataAppendColInfo(pBlock, &infoData); + + return pBlock; +} + +void appendColumnFields(char* buf, int32_t* len, STableCfg* pCfg) { + for (int32_t i = 0; i < pCfg->numOfColumns; ++i) { + SSchema* pSchema = pCfg->pSchemas + i; + char type[32]; + sprintf(type, "%s", tDataTypes[pSchema->type].name); + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { + sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); + } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { + sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE)); + } + + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + } +} + +void appendTagFields(char* buf, int32_t* len, STableCfg* pCfg) { + for (int32_t i = 0; i < pCfg->numOfTags; ++i) { + SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; + char type[32]; + sprintf(type, "%s", tDataTypes[pSchema->type].name); + if (TSDB_DATA_TYPE_VARCHAR == pSchema->type) { + sprintf(type + strlen(type), "(%d)", (int32_t)(pSchema->bytes - VARSTR_HEADER_SIZE)); + } else if (TSDB_DATA_TYPE_NCHAR == pSchema->type) { + sprintf(type + strlen(type), "(%d)", (int32_t)((pSchema->bytes - VARSTR_HEADER_SIZE)/TSDB_NCHAR_SIZE)); + } + + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s` %s", ((i > 0) ? ", " : ""), pSchema->name, type); + } +} + + +void appendTagNameFields(char* buf, int32_t* len, STableCfg* pCfg) { + for (int32_t i = 0; i < pCfg->numOfTags; ++i) { + SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s`%s`", ((i > 0) ? ", " : ""), pSchema->name); + } +} + + +int32_t appendTagValues(char* buf, int32_t* len, STableCfg* pCfg) { + SArray *pTagVals = NULL; + STag *pTag = (STag*)pCfg->pTags; + + if (pCfg->pTags && pTag->flags & TD_TAG_JSON) { + char *pJson = parseTagDatatoJson(pTag); + if (pJson) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s", pJson); + taosMemoryFree(pJson); + } + + return TSDB_CODE_SUCCESS; + } + + int32_t code = tTagToValArray((const STag *)pCfg->pTags, &pTagVals); + if (code) { + return code; + } -static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt) { return TSDB_CODE_FAILED; } + int16_t valueNum = taosArrayGetSize(pTagVals); + int32_t num = 0; + int32_t j = 0; + for (int32_t i = 0; i < pCfg->numOfTags; ++i) { + SSchema* pSchema = pCfg->pSchemas + pCfg->numOfColumns + i; + if (i > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); + } + + if (j >= valueNum) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); + continue; + } + + STagVal *pTagVal = (STagVal *)taosArrayGet(pTagVals, j); + if (pSchema->colId > pTagVal->cid) { + qError("tag value and column mismatch, schemaId:%d, valId:%d", pSchema->colId, pTagVal->cid); + taosArrayDestroy(pTagVals); + return TSDB_CODE_APP_ERROR; + } else if (pSchema->colId == pTagVal->cid) { + char type = pTagVal->type; + int32_t tlen = 0; + + if (IS_VAR_DATA_TYPE(type)) { + dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, pTagVal->pData, pTagVal->nData, &tlen); + } else { + dataConverToStr(buf + VARSTR_HEADER_SIZE + *len, type, &pTagVal->i64, tDataTypes[type].bytes, &tlen); + } + *len += tlen; + j++; + } else { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "NULL"); + } + + + /* + if (type == TSDB_DATA_TYPE_BINARY) { + if (pTagVal->nData > 0) { + if (num) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); + } + + memcpy(buf + VARSTR_HEADER_SIZE + *len, pTagVal->pData, pTagVal->nData); + *len += pTagVal->nData; + } + } else if (type == TSDB_DATA_TYPE_NCHAR) { + if (pTagVal->nData > 0) { + if (num) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", "); + } + int32_t tlen = taosUcs4ToMbs((TdUcs4 *)pTagVal->pData, pTagVal->nData, buf + VARSTR_HEADER_SIZE + *len); + } + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double val = *(double *)(&pTagVal->i64); + int len = 0; + term = indexTermCreate(suid, ADD_VALUE, type, key, nKey, (const char *)&val, len); + } else if (type == TSDB_DATA_TYPE_BOOL) { + int val = *(int *)(&pTagVal->i64); + int len = 0; + term = indexTermCreate(suid, ADD_VALUE, TSDB_DATA_TYPE_INT, key, nKey, (const char *)&val, len); + } + */ + } + + taosArrayDestroy(pTagVals); + + return TSDB_CODE_SUCCESS; +} + +void appendTableOptions(char* buf, int32_t* len, STableCfg* pCfg) { + if (pCfg->commentLen > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT '%s'", pCfg->pComment); + } else if (0 == pCfg->commentLen) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " COMMENT ''"); + } + + if (pCfg->watermark1 > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " WATERMARK %" PRId64 "a", pCfg->watermark1); + if (pCfg->watermark2 > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->watermark2); + } + } + + if (pCfg->delay1 > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " MAX_DELAY %" PRId64 "a", pCfg->delay1); + if (pCfg->delay2 > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ", %" PRId64 "a", pCfg->delay2); + } + } + + int32_t funcNum = taosArrayGetSize(pCfg->pFuncs); + if (funcNum > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " ROLLUP("); + for (int32_t i = 0; i < funcNum; ++i) { + char* pFunc = taosArrayGet(pCfg->pFuncs, i); + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, "%s%s", ((i > 0) ? ", " : ""), pFunc); + } + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, ")"); + } + + if (pCfg->ttl > 0) { + *len += sprintf(buf + VARSTR_HEADER_SIZE + *len, " TTL %d", pCfg->ttl); + } +} + +static int32_t setCreateTBResultIntoDataBlock(SSDataBlock* pBlock, char *tbName, STableCfg* pCfg) { + int32_t code = 0; + blockDataEnsureCapacity(pBlock, 1); + pBlock->info.rows = 1; + + SColumnInfoData* pCol1 = taosArrayGet(pBlock->pDataBlock, 0); + char buf1[SHOW_CREATE_TB_RESULT_FIELD1_LEN] = {0}; + STR_TO_VARSTR(buf1, tbName); + colDataAppend(pCol1, 0, buf1, false); + + SColumnInfoData* pCol2 = taosArrayGet(pBlock->pDataBlock, 1); + char buf2[SHOW_CREATE_TB_RESULT_FIELD2_LEN] = {0}; + int32_t len = 0; + + if (TSDB_SUPER_TABLE == pCfg->tableType) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE STABLE `%s` (", tbName); + appendColumnFields(buf2, &len, pCfg); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); + appendTagFields(buf2, &len, pCfg); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + appendTableOptions(buf2, &len, pCfg); + } else if (TSDB_CHILD_TABLE == pCfg->tableType) { + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` USING `%s` (", tbName, pCfg->stbName); + appendTagNameFields(buf2, &len, pCfg); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ") TAGS ("); + code = appendTagValues(buf2, &len, pCfg); + if (code) { + return code; + } + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + appendTableOptions(buf2, &len, pCfg); + } else { + len += sprintf(buf2 + VARSTR_HEADER_SIZE, "CREATE TABLE `%s` (", tbName); + appendColumnFields(buf2, &len, pCfg); + len += sprintf(buf2 + VARSTR_HEADER_SIZE + len, ")"); + } + + varDataLen(buf2) = len; + + colDataAppend(pCol2, 0, buf2, false); + + return TSDB_CODE_SUCCESS; +} + + +static int32_t execShowCreateTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = buildCreateTbResultDataBlock(); + int32_t code = setCreateTBResultIntoDataBlock(pBlock, pStmt->tableName, pStmt->pCfg); + if (code) { + return code; + } + + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + *pRsp = taosMemoryCalloc(1, rspSize); + if (NULL == *pRsp) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pRsp)->useconds = 0; + (*pRsp)->completed = 1; + (*pRsp)->precision = 0; + (*pRsp)->compressed = 0; + (*pRsp)->compLen = 0; + (*pRsp)->numOfRows = htonl(1); + (*pRsp)->numOfCols = htonl(SHOW_CREATE_TB_RESULT_COLS); + + int32_t len = 0; + blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_CREATE_TB_RESULT_COLS, false); + ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); + + blockDataDestroy(pBlock); + return TSDB_CODE_SUCCESS; +} + +static int32_t execShowCreateSTable(SShowCreateTableStmt* pStmt, SRetrieveTableRsp** pRsp) { + STableCfg* pCfg = (STableCfg*)pStmt->pCfg; + if (TSDB_SUPER_TABLE != pCfg->tableType) { + terrno = TSDB_CODE_TSC_NOT_STABLE_ERROR; + return terrno; + } + + return execShowCreateTable(pStmt, pRsp); +} static int32_t execAlterLocal(SAlterLocalStmt* pStmt) { return TSDB_CODE_FAILED; } +static SSDataBlock* buildLocalVariablesResultDataBlock() { + SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); + pBlock->info.hasVarCol = true; + + pBlock->pDataBlock = taosArrayInit(SHOW_LOCAL_VARIABLES_RESULT_COLS, sizeof(SColumnInfoData)); + + SColumnInfoData infoData = {0}; + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD1_LEN; + + taosArrayPush(pBlock->pDataBlock, &infoData); + + infoData.info.type = TSDB_DATA_TYPE_VARCHAR; + infoData.info.bytes = SHOW_LOCAL_VARIABLES_RESULT_FIELD2_LEN; + taosArrayPush(pBlock->pDataBlock, &infoData); + + return pBlock; +} + + +int32_t setLocalVariablesResultIntoDataBlock(SSDataBlock* pBlock) { + int32_t numOfCfg = taosArrayGetSize(tsCfg->array); + int32_t numOfRows = 0; + blockDataEnsureCapacity(pBlock, numOfCfg); + + for (int32_t i = 0, c = 0; i < numOfCfg; ++i, c = 0) { + SConfigItem *pItem = taosArrayGet(tsCfg->array, i); + + char name[TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(name, pItem->name, TSDB_CONFIG_OPTION_LEN + VARSTR_HEADER_SIZE); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, name, false); + + char value[TSDB_CONFIG_VALUE_LEN + VARSTR_HEADER_SIZE] = {0}; + int32_t valueLen = 0; + cfgDumpItemValue(pItem, &value[VARSTR_HEADER_SIZE], TSDB_CONFIG_VALUE_LEN, &valueLen); + varDataSetLen(value, valueLen); + pColInfo = taosArrayGet(pBlock->pDataBlock, c++); + colDataAppend(pColInfo, i, value, false); + + numOfRows++; + } + + + pBlock->info.rows = numOfRows; + + return TSDB_CODE_SUCCESS; +} + + +static int32_t execShowLocalVariables(SRetrieveTableRsp** pRsp) { + SSDataBlock* pBlock = buildLocalVariablesResultDataBlock(); + int32_t code = setLocalVariablesResultIntoDataBlock(pBlock); + if (code) { + return code; + } + + size_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + *pRsp = taosMemoryCalloc(1, rspSize); + if (NULL == *pRsp) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pRsp)->useconds = 0; + (*pRsp)->completed = 1; + (*pRsp)->precision = 0; + (*pRsp)->compressed = 0; + (*pRsp)->compLen = 0; + (*pRsp)->numOfRows = htonl(pBlock->info.rows); + (*pRsp)->numOfCols = htonl(SHOW_LOCAL_VARIABLES_RESULT_COLS); + + int32_t len = 0; + blockCompressEncode(pBlock, (*pRsp)->data, &len, SHOW_LOCAL_VARIABLES_RESULT_COLS, false); + ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); + + blockDataDestroy(pBlock); + return TSDB_CODE_SUCCESS; +} + int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) { switch (nodeType(pStmt)) { case QUERY_NODE_DESCRIBE_STMT: @@ -138,13 +608,15 @@ int32_t qExecCommand(SNode* pStmt, SRetrieveTableRsp** pRsp) { case QUERY_NODE_RESET_QUERY_CACHE_STMT: return execResetQueryCache(); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: - return execShowCreateDatabase((SShowCreateDatabaseStmt*)pStmt); + return execShowCreateDatabase((SShowCreateDatabaseStmt*)pStmt, pRsp); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: - return execShowCreateTable((SShowCreateTableStmt*)pStmt); + return execShowCreateTable((SShowCreateTableStmt*)pStmt, pRsp); case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - return execShowCreateSTable((SShowCreateTableStmt*)pStmt); + return execShowCreateSTable((SShowCreateTableStmt*)pStmt, pRsp); case QUERY_NODE_ALTER_LOCAL_STMT: return execAlterLocal((SAlterLocalStmt*)pStmt); + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return execShowLocalVariables(pRsp); default: break; } diff --git a/source/libs/command/src/explain.c b/source/libs/command/src/explain.c index ae0f669d656a458a310339e461ad2e55918ee211..ef4ac0b639b72d8808fb2cdb9f6701762240b651 100644 --- a/source/libs/command/src/explain.c +++ b/source/libs/command/src/explain.c @@ -184,8 +184,8 @@ int32_t qExplainGenerateResChildren(SPhysiNode *pNode, SExplainGroup *group, SNo pPhysiChildren = indefPhysiNode->node.pChildren; break; } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: { - SMergeIntervalPhysiNode *intPhysiNode = (SMergeIntervalPhysiNode *)pNode; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: { + SMergeAlignedIntervalPhysiNode *intPhysiNode = (SMergeAlignedIntervalPhysiNode *)pNode; pPhysiChildren = intPhysiNode->window.node.pChildren; break; } @@ -841,8 +841,8 @@ int32_t qExplainResNodeToRowsImpl(SExplainResNode *pResNode, SExplainCtx *ctx, i } break; } - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: { - SMergeIntervalPhysiNode *pIntNode = (SMergeIntervalPhysiNode *)pNode; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: { + SMergeAlignedIntervalPhysiNode *pIntNode = (SMergeAlignedIntervalPhysiNode *)pNode; EXPLAIN_ROW_NEW(level, EXPLAIN_INTERVAL_FORMAT, nodesGetNameFromColumnNode(pIntNode->window.pTspk)); EXPLAIN_ROW_APPEND(EXPLAIN_LEFT_PARENTHESIS_FORMAT); if (pResNode->pExecInfo) { @@ -1190,17 +1190,12 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { QRY_ERR_RET(TSDB_CODE_QRY_APP_ERROR); } - SSDataBlock *pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_VARCHAR; - infoData.info.bytes = TSDB_EXPLAIN_RESULT_ROW_SIZE; - - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - taosArrayPush(pBlock->pDataBlock, &infoData); + SSDataBlock *pBlock = createDataBlock(); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_VARCHAR, TSDB_EXPLAIN_RESULT_ROW_SIZE, 1); + blockDataAppendColInfo(pBlock, &infoData); + blockDataEnsureCapacity(pBlock, rowNum); SColumnInfoData* pInfoData = taosArrayGet(pBlock->pDataBlock, 0); - pInfoData->hasNull = false; - colInfoDataEnsureCapacity(pInfoData, 0, rowNum); char buf[1024] = {0}; for (int32_t i = 0; i < rowNum; ++i) { @@ -1210,9 +1205,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { colDataAppend(pInfoData, i, buf, false); } - pBlock->info.numOfCols = 1; pBlock->info.rows = rowNum; - pBlock->info.hasVarCol = true; int32_t rspSize = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); @@ -1226,7 +1219,7 @@ int32_t qExplainGetRspFromCtx(void *ctx, SRetrieveTableRsp **pRsp) { rsp->numOfRows = htonl(rowNum); int32_t len = 0; - blockCompressEncode(pBlock, rsp->data, &len, pBlock->info.numOfCols, 0); + blockCompressEncode(pBlock, rsp->data, &len, taosArrayGetSize(pBlock->pDataBlock), 0); ASSERT(len == rspSize - sizeof(SRetrieveTableRsp)); rsp->compLen = htonl(len); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 1117be5db0fd0002b8a951f1ededd185b1c3f30a..f3e1eb47e8fa7b8494fc4dbeed71bce0d6ba4a1f 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -106,7 +106,8 @@ int32_t getNumOfTotalRes(SGroupResInfo* pGroupResInfo); SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode); -int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo, SNode* pTagCond); +EDealRes doTranslateTagExpr(SNode** pNode, void* pContext); +int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo); SArray* createSortInfo(SNodeList* pNodeList); SArray* extractPartitionColInfo(SNodeList* pNodeList); SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNodeList, int32_t* numOfOutputCols, int32_t type); @@ -123,4 +124,6 @@ SColumn extractColumnFromColumnNode(SColumnNode* pColNode); int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysiNode* pTableScanNode); void cleanupQueryTableDataCond(SQueryTableDataCond* pCond); +int32_t convertFillType(int32_t mode); + #endif // TDENGINE_QUERYUTIL_H diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 9e6613d3c297d67e5f19f6af37608d6ba9148cc0..91931c2fd81e4895eac2c9f134eb44efb648edb6 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -234,6 +234,7 @@ typedef struct SColMatchInfo { int32_t colId; int32_t targetSlotId; bool output; + bool reserved; int32_t matchType; // determinate the source according to col id or slot id } SColMatchInfo; @@ -253,7 +254,6 @@ typedef struct STableScanInfo { SFileBlockLoadRecorder readRecorder; int64_t numOfRows; -// int32_t prevGroupId; // previous table group id SScanInfo scanInfo; int32_t scanTimes; SNode* pFilterNode; // filter info, which is push down by optimizer @@ -282,7 +282,6 @@ typedef struct STagScanInfo { int32_t curPos; SReadHandle readHandle; STableListInfo *pTableList; - SNode* pFilterNode; // filter info, } STagScanInfo; typedef enum EStreamScanMode { @@ -341,7 +340,7 @@ typedef struct SStreamBlockScanInfo { SReadHandle readHandle; uint64_t tableUid; // queried super table uid EStreamScanMode scanMode; - SOperatorInfo* pOperatorDumy; + SOperatorInfo* pSnapshotReadOp; SInterval interval; // if the upstream is an interval operator, the interval info is also kept here. SArray* childIds; SessionWindowSupporter sessionSup; @@ -566,13 +565,14 @@ typedef struct SStreamSessionAggOperatorInfo { } SStreamSessionAggOperatorInfo; typedef struct STimeSliceOperatorInfo { - SOptrBasicInfo binfo; + SSDataBlock* pRes; STimeWindow win; SInterval interval; int64_t current; SArray* pPrevRow; // SArray - SArray* pCols; // SArray int32_t fillType; // fill type + SColumn tsCol; // primary timestamp column + SExprSupp scalarSup; // scalar calculation struct SFillColInfo* pFillColInfo; // fill column info } STimeSliceOperatorInfo; @@ -670,7 +670,7 @@ int32_t appendDownstream(SOperatorInfo* p, SOperatorInfo** pDownstream, int32_t void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock); void cleanupBasicInfo(SOptrBasicInfo* pInfo); -void initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr); void cleanupExprSup(SExprSupp* pSup); int32_t initAggInfo(SExprSupp *pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey); @@ -732,6 +732,10 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResBlock, SInterval* pInterval, int32_t primaryTsSlotId, + SExecTaskInfo* pTaskInfo); + SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild); SOperatorInfo* createStreamIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, @@ -757,8 +761,8 @@ SOperatorInfo* createStatewindowOperatorInfo(SOperatorInfo* downstream, SExprInf SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartitionPhysiNode* pPartNode, SExecTaskInfo* pTaskInfo); -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, const SNodeListNode* pValNode, SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pNode, /*SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock, const SNodeListNode* pValNode, */SExecTaskInfo* pTaskInfo); SOperatorInfo* createMergeJoinOperatorInfo(SOperatorInfo** pDownstream, int32_t numOfDownstream, SJoinPhysiNode* pJoinNode, SExecTaskInfo* pTaskInfo); @@ -833,14 +837,17 @@ int32_t finalizeResultRowIntoResultDataBlock(SDiskbasedBuf* pBuf, SResultRowPosi SqlFunctionCtx* pCtx, SExprInfo* pExprInfo, int32_t numOfExprs, const int32_t* rowCellOffset, SSDataBlock* pBlock, SExecTaskInfo* pTaskInfo); -int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId, - uint64_t taskId, SNode* pTagCond); -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo); +int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); +SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, + SExecTaskInfo* pTaskInfo); +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo *pTableListInfo, + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, uint64_t taskId); void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex); +int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, SArray* groupKey); + #ifdef __cplusplus } #endif diff --git a/source/libs/executor/inc/tsort.h b/source/libs/executor/inc/tsort.h index 363f379ee471e9d9d604784f1fcd24880b4ae3e7..35bdd4ee554e4a1af7b040293909000d77f2fcdd 100644 --- a/source/libs/executor/inc/tsort.h +++ b/source/libs/executor/inc/tsort.h @@ -50,6 +50,7 @@ typedef struct SMsortComparParam { void **pSources; int32_t numOfSources; SArray *orderInfo; // SArray + bool cmpGroupId; } SMsortComparParam; typedef struct SSortHandle SSortHandle; @@ -99,6 +100,11 @@ int32_t tsortSetFetchRawDataFp(SSortHandle* pHandle, _sort_fetch_block_fn_t fetc */ int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp); +/** + * + */ +int32_t tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId); + /** * * @param pHandle diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index 33b7811e6c0c26032d1bca8cc8643dce2b0ea984..8c220134f026c4be7021ecaba1b7ea6acfd91691 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -75,7 +75,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp SDataCacheEntry* pEntry = (SDataCacheEntry*)pBuf->pData; pEntry->compressed = 0; pEntry->numOfRows = pInput->pData->info.rows; - pEntry->numOfCols = pInput->pData->info.numOfCols; + pEntry->numOfCols = taosArrayGetSize(pInput->pData->pDataBlock); pEntry->dataLen = sizeof(SDeleterRes); ASSERT(1 == pEntry->numOfRows); @@ -86,7 +86,7 @@ static void toDataCacheEntry(SDataDeleterHandle* pHandle, const SInputData* pInp SColumnInfoData* pColRes = (SColumnInfoData*)taosArrayGet(pInput->pData->pDataBlock, 0); SDeleterRes* pRes = (SDeleterRes*)pEntry->data; - pRes->uid = pHandle->pDeleter->tableId; + pRes->suid = pHandle->pParam->suid; pRes->uidList = pHandle->pParam->pUidList; pRes->skey = pHandle->pDeleter->deleteTimeRange.skey; pRes->ekey = pHandle->pDeleter->deleteTimeRange.ekey; diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index aadab4f22ad6272bc868984f803b9b2dd1c99a71..8669c2c28c462f5f77b9e45b0e0242f77f74ba32 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -183,71 +183,159 @@ SArray* createSortInfo(SNodeList* pNodeList) { SSDataBlock* createResDataBlock(SDataBlockDescNode* pNode) { int32_t numOfCols = LIST_LENGTH(pNode->pSlots); - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); pBlock->info.blockId = pNode->dataBlockId; - pBlock->info.rowSize = pNode->totalRowSize; // todo ?? pBlock->info.type = STREAM_INVALID; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {{0}}; SSlotDescNode* pDescNode = (SSlotDescNode*)nodesListGetNode(pNode->pSlots, i); // if (!pDescNode->output) { // todo disable it temporarily // continue; // } - idata.info.type = pDescNode->dataType.type; - idata.info.bytes = pDescNode->dataType.bytes; + SColumnInfoData idata = createColumnInfoData(pDescNode->dataType.type, pDescNode->dataType.bytes, pDescNode->slotId); idata.info.scale = pDescNode->dataType.scale; - idata.info.slotId = pDescNode->slotId; idata.info.precision = pDescNode->dataType.precision; - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; + blockDataAppendColInfo(pBlock, &idata); + } + + return pBlock; +} + +EDealRes doTranslateTagExpr(SNode** pNode, void* pContext) { + SMetaReader* mr = (SMetaReader*)pContext; + if(nodeType(*pNode) == QUERY_NODE_COLUMN){ + SColumnNode* pSColumnNode = *(SColumnNode**)pNode; + + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + return DEAL_RES_ERROR; } - taosArrayPush(pBlock->pDataBlock, &idata); + res->translate = true; + res->node.resType = pSColumnNode->node.resType; + + STagVal tagVal = {0}; + tagVal.cid = pSColumnNode->colId; + const char* p = metaGetTableTagVal(&mr->me, pSColumnNode->node.resType.type, &tagVal); + if (p == NULL) { + res->node.resType.type = TSDB_DATA_TYPE_NULL; + }else if (pSColumnNode->node.resType.type == TSDB_DATA_TYPE_JSON) { + int32_t len = ((const STag*)p) -> len; + res->datum.p = taosMemoryCalloc(len + 1, 1); + memcpy(res->datum.p, p, len); + } else if (IS_VAR_DATA_TYPE(pSColumnNode->node.resType.type)) { + res->datum.p = taosMemoryCalloc(tagVal.nData + VARSTR_HEADER_SIZE + 1, 1); + memcpy(varDataVal(res->datum.p), tagVal.pData, tagVal.nData); + varDataSetLen(res->datum.p, tagVal.nData); + } else { + nodesSetValueNodeValue(res, &(tagVal.i64)); + } + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + }else if (nodeType(*pNode) == QUERY_NODE_FUNCTION){ + SFunctionNode * pFuncNode = *(SFunctionNode**)pNode; + if(pFuncNode->funcType == FUNCTION_TYPE_TBNAME){ + SValueNode *res = (SValueNode *)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == res) { + return DEAL_RES_ERROR; + } + + res->translate = true; + res->node.resType = pFuncNode->node.resType; + + int32_t len = strlen(mr->me.name); + res->datum.p = taosMemoryCalloc(len + VARSTR_HEADER_SIZE + 1, 1); + memcpy(varDataVal(res->datum.p), mr->me.name, len); + varDataSetLen(res->datum.p, len); + nodesDestroyNode(*pNode); + *pNode = (SNode*)res; + } } - pBlock->info.numOfCols = taosArrayGetSize(pBlock->pDataBlock); - return pBlock; + return DEAL_RES_CONTINUE; +} + +static bool isTableOk(STableKeyInfo* info, SNode *pTagCond, SMeta *metaHandle){ + SMetaReader mr = {0}; + metaReaderInit(&mr, metaHandle, 0); + metaGetTableEntryByUid(&mr, info->uid); + + SNode *pTagCondTmp = nodesCloneNode(pTagCond); + + nodesRewriteExprPostOrder(&pTagCondTmp, doTranslateTagExpr, &mr); + metaReaderClear(&mr); + + SNode* pNew = NULL; + int32_t code = scalarCalculateConstants(pTagCondTmp, &pNew); + if (TSDB_CODE_SUCCESS != code) { + nodesDestroyNode(pTagCondTmp); + return false; + } + + ASSERT(nodeType(pNew) == QUERY_NODE_VALUE); + SValueNode *pValue = (SValueNode *)pNew; + + ASSERT(pValue->node.resType.type == TSDB_DATA_TYPE_BOOL); + bool result = pValue->datum.b; + nodesDestroyNode(pNew); + return result; } -int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo, SNode* pTagCond) { +int32_t getTableList(void* metaHandle, SScanPhysiNode* pScanNode, STableListInfo* pListInfo) { int32_t code = TSDB_CODE_SUCCESS; pListInfo->pTableList = taosArrayInit(8, sizeof(STableKeyInfo)); uint64_t tableUid = pScanNode->uid; + pListInfo->suid = pScanNode->suid; + + SNode* pTagCond = (SNode*)pListInfo->pTagCond; + SNode* pTagIndexCond = (SNode*)pListInfo->pTagIndexCond; if (pScanNode->tableType == TSDB_SUPER_TABLE) { - if (pTagCond) { + if (pTagIndexCond) { SIndexMetaArg metaArg = { .metaEx = metaHandle, .idx = tsdbGetIdx(metaHandle), .ivtIdx = tsdbGetIvtIdx(metaHandle), .suid = tableUid}; SArray* res = taosArrayInit(8, sizeof(uint64_t)); - code = doFilterTag(pTagCond, &metaArg, res); - if (code == TSDB_CODE_INDEX_REBUILDING) { // todo - // doFilter(); + //code = doFilterTag(pTagIndexCond, &metaArg, res); + code = TSDB_CODE_INDEX_REBUILDING; + if (code == TSDB_CODE_INDEX_REBUILDING) { + code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList); } else if (code != TSDB_CODE_SUCCESS) { qError("failed to get tableIds, reason: %s, suid: %" PRIu64 "", tstrerror(code), tableUid); taosArrayDestroy(res); terrno = code; return code; } else { - qDebug("success to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); + qDebug("sucess to get tableIds, size: %d, suid: %" PRIu64 "", (int)taosArrayGetSize(res), tableUid); } for (int i = 0; i < taosArrayGetSize(res); i++) { - STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i)}; + STableKeyInfo info = {.lastKey = TSKEY_INITIAL_VAL, .uid = *(uint64_t*)taosArrayGet(res, i), .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } taosArrayDestroy(res); } else { code = tsdbGetAllTableList(metaHandle, tableUid, pListInfo->pTableList); } - } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = tableUid}; + + if(pTagCond){ + int32_t i = 0; + while(i < taosArrayGetSize(pListInfo->pTableList)) { + STableKeyInfo* info = taosArrayGet(pListInfo->pTableList, i); + bool isOk = isTableOk(info, pTagCond, metaHandle); + if(!isOk){ + taosArrayRemove(pListInfo->pTableList, i); + continue; + } + i++; + } + } + }else { // Create one table group. + STableKeyInfo info = {.lastKey = 0, .uid = tableUid, .groupId = 0}; taosArrayPush(pListInfo->pTableList, &info); } @@ -300,7 +388,7 @@ SArray* extractColMatchInfo(SNodeList* pNodeList, SDataBlockDescNode* pOutputNod SColMatchInfo c = {0}; c.output = true; - c.colId = pColNode->colId; + c.colId = pColNode->colId; c.srcSlotId = pColNode->slotId; c.matchType = type; c.targetSlotId = pNode->slotId; @@ -600,7 +688,7 @@ void relocateColumnData(SSDataBlock* pBlock, const SArray* pColMatchInfo, SArray if (p->info.colId == pmInfo->colId) { SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, p, pBlock->info.rows); + colDataAssign(pDst, p, pBlock->info.rows, &pBlock->info); i++; j++; } else if (p->info.colId < pmInfo->colId) { @@ -694,4 +782,32 @@ int32_t initQueryTableDataCond(SQueryTableDataCond* pCond, const STableScanPhysi void cleanupQueryTableDataCond(SQueryTableDataCond* pCond) { taosMemoryFree(pCond->twindows); taosMemoryFree(pCond->colList); -} \ No newline at end of file +} + +int32_t convertFillType(int32_t mode) { + int32_t type = TSDB_FILL_NONE; + switch (mode) { + case FILL_MODE_PREV: + type = TSDB_FILL_PREV; + break; + case FILL_MODE_NONE: + type = TSDB_FILL_NONE; + break; + case FILL_MODE_NULL: + type = TSDB_FILL_NULL; + break; + case FILL_MODE_NEXT: + type = TSDB_FILL_NEXT; + break; + case FILL_MODE_VALUE: + type = TSDB_FILL_SET_VALUE; + break; + case FILL_MODE_LINEAR: + type = TSDB_FILL_LINEAR; + break; + default: + type = TSDB_FILL_NONE; + } + + return type; +} diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index cf72dfd7966deec976b90d615d8bb90329f69cc3..1bd1ce14b1d3f6102f3a909ba53462a1f6e5e538 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -40,7 +40,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu SStreamBlockScanInfo* pInfo = pOperator->info; pInfo->assignBlockUid = assignUid; - // the block type can not be changed in the streamscan operators + // no need to check #if 0 if (pInfo->blockType == 0) { pInfo->blockType = type; @@ -49,10 +49,7 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu return TSDB_CODE_QRY_APP_ERROR; } #endif - // rollup sma, the same qTaskInfo is used to insert data by SubmitReq and fetch result by SSDataBlock - if (pInfo->blockType != type) { - pInfo->blockType = type; - } + pInfo->blockType = type; if (type == STREAM_DATA_TYPE_SUBMIT_BLOCK) { if (tqReadHandleSetMsg(pInfo->streamBlockReader, input, 0) < 0) { @@ -70,6 +67,9 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu taosArrayAddAll(p->pDataBlock, pDataBlock->pDataBlock); taosArrayPush(pInfo->pBlockLists, &p); } + } else if (type == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + // do nothing + ASSERT(pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT); } else { ASSERT(0); } @@ -78,6 +78,14 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu } } +int32_t qStreamScanSnapshot(qTaskInfo_t tinfo) { + if (tinfo == NULL) { + return TSDB_CODE_QRY_APP_ERROR; + } + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + return doSetStreamBlock(pTaskInfo->pRoot, NULL, 0, STREAM_DATA_TYPE_FROM_SNAPSHOT, 0, NULL); +} + int32_t qSetStreamInput(qTaskInfo_t tinfo, const void* input, int32_t type, bool assignUid) { return qSetMultiStreamInput(tinfo, input, 1, type, assignUid); } @@ -109,14 +117,6 @@ qTaskInfo_t qCreateStreamExecTaskInfo(void* msg, void* streamReadHandle) { return NULL; } - // print those info into log -#if 0 - pMsg->sId = pMsg->sId; - pMsg->queryId = pMsg->queryId; - pMsg->taskId = pMsg->taskId; - pMsg->contentLen = pMsg->contentLen; -#endif - /*qDebugL("stream task string %s", (const char*)msg);*/ struct SSubplan* plan = NULL; diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 81ca0f4f8c459283c9d1562f522daec0e756e82d..962b7b9c518991d92ed335acdf7a836a65708f18 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -356,7 +356,7 @@ void initExecTimeWindowInfo(SColumnInfoData* pColData, STimeWindow* pQueryWindow pColData->info.type = TSDB_DATA_TYPE_TIMESTAMP; pColData->info.bytes = sizeof(int64_t); - colInfoDataEnsureCapacity(pColData, 0, 5); + colInfoDataEnsureCapacity(pColData, 5); colDataAppendInt64(pColData, 0, &pQueryWindow->skey); colDataAppendInt64(pColData, 1, &pQueryWindow->ekey); @@ -458,7 +458,7 @@ static int32_t doCreateConstantValColumnInfo(SInputColumnInfoData* pInput, SFunc pColInfo = pInput->pData[paramIndex]; } - colInfoDataEnsureCapacity(pColInfo, 0, numOfRows); + colInfoDataEnsureCapacity(pColInfo, numOfRows); int8_t type = pFuncParam->param.nType; if (type == TSDB_DATA_TYPE_BIGINT || type == TSDB_DATA_TYPE_UBIGINT) { @@ -579,7 +579,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc colDataMergeCol(pColInfoData, pResult->info.rows, &pResult->info.capacity, pfCtx->input.pData[0], pfCtx->input.numOfRows); } else { - colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows); + colDataAssign(pColInfoData, pfCtx->input.pData[0], pfCtx->input.numOfRows, &pResult->info); } numOfRows = pfCtx->input.numOfRows; @@ -609,7 +609,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); + ASSERT(pResult->info.capacity > 0); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; @@ -649,7 +649,7 @@ int32_t projectApplyFunctions(SExprInfo* pExpr, SSDataBlock* pResult, SSDataBloc } int32_t startOffset = createNewColModel ? 0 : pResult->info.rows; - colInfoDataEnsureCapacity(pResColData, startOffset, pResult->info.capacity); + ASSERT(pResult->info.capacity > 0); colDataMergeCol(pResColData, startOffset, &pResult->info.capacity, &idata, dest.numOfRows); numOfRows = dest.numOfRows; @@ -1186,7 +1186,7 @@ int32_t loadDataBlockOnDemand(SExecTaskInfo* pTaskInfo, STableScanInfo* pTableSc // } // if (pQueryAttr->pFilters != NULL) { -// filterSetColFieldData(pQueryAttr->pFilters, pBlock->info.numOfCols, pBlock->pDataBlock); +// filterSetColFieldData(pQueryAttr->pFilters, taosArrayGetSize(pBlock->pDataBlock), pBlock->pDataBlock); // } // if (pQueryAttr->pFilters != NULL || pRuntimeEnv->pTsBuf != NULL) { @@ -1321,7 +1321,8 @@ void doFilter(const SNode* pFilterNode, SSDataBlock* pBlock) { // todo move to the initialization function int32_t code = filterInitFromNode((SNode*)pFilterNode, &filter, 0); - SFilterColumnParam param1 = {.numOfCols = pBlock->info.numOfCols, .pDataBlock = pBlock->pDataBlock}; + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + SFilterColumnParam param1 = {.numOfCols = numOfCols, .pDataBlock = pBlock->pDataBlock}; code = filterSetDataFromSlotId(filter, ¶m1); int8_t* rowRes = NULL; @@ -1340,10 +1341,11 @@ void extractQualifiedTupleByFilterResult(SSDataBlock* pBlock, const int8_t* rowR } if (rowRes != NULL) { + int32_t totalRows = pBlock->info.rows; SSDataBlock* px = createOneDataBlock(pBlock, true); - int32_t totalRows = pBlock->info.rows; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pSrc = taosArrayGet(px->pDataBlock, i); SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, i); // it is a reserved column for scalar function, and no data in this column yet. @@ -2045,8 +2047,9 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo int32_t compLen, int32_t numOfOutput, int64_t startTs, uint64_t* total, SArray* pColList) { if (pColList == NULL) { // data from other sources + blockDataCleanup(pRes); +// blockDataEnsureCapacity(pRes, numOfRows); blockCompressDecode(pRes, numOfOutput, numOfRows, pData); - pRes->info.rows = numOfRows; } else { // extract data according to pColList ASSERT(numOfOutput == taosArrayGetSize(pColList)); char* pStart = pData; @@ -2064,32 +2067,19 @@ int32_t extractDataBlockFromFetchRsp(SSDataBlock* pRes, SLoadRemoteDataInfo* pLo pStart += sizeof(SSysTableSchema); } - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - pBlock->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = numOfCols; - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData idata = {0}; - - idata.info.type = pSchema[i].type; - idata.info.bytes = pSchema[i].bytes; - idata.info.colId = pSchema[i].colId; - idata.hasNull = true; - - taosArrayPush(pBlock->pDataBlock, &idata); - if (IS_VAR_DATA_TYPE(idata.info.type)) { - pBlock->info.hasVarCol = true; - } + SColumnInfoData idata = createColumnInfoData(pSchema[i].type, pSchema[i].bytes, pSchema[i].colId); + blockDataAppendColInfo(pBlock, &idata); } blockCompressDecode(pBlock, numOfCols, numOfRows, pStart); + blockDataEnsureCapacity(pRes, numOfRows); // data from mnode pRes->info.rows = numOfRows; relocateColumnData(pRes, pColList, pBlock->pDataBlock, false); - taosArrayDestroy(pBlock->pDataBlock); - taosMemoryFree(pBlock); - // blockDataDestroy(pBlock); + blockDataDestroy(pBlock); } // todo move this to time window aggregator, since the primary timestamp may not be known by exchange operator. @@ -2412,15 +2402,15 @@ SOperatorInfo* createExchangeOperatorInfo(void* pTransporter, SExchangePhysiNode tsem_init(&pInfo->ready, 0, 0); - pInfo->seqLoadData = false; + pInfo->seqLoadData = false; pInfo->pTransporter = pTransporter; - pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); - pOperator->name = "ExchangeOperator"; + pInfo->pResult = createResDataBlock(pExNode->node.pOutputDataBlockDesc); + pOperator->name = "ExchangeOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_EXCHANGE; pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pResult->info.numOfCols; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pResult->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(prepareLoadRemoteData, doLoadRemoteData, NULL, NULL, @@ -2542,9 +2532,6 @@ static void doMergeImpl(SOperatorInfo* pOperator, int32_t numOfExpr, SSDataBlock SSortedMergeOperatorInfo* pInfo = pOperator->info; SqlFunctionCtx* pCtx = pOperator->exprSupp.pCtx; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - // pCtx[i].size = 1; - } for (int32_t i = 0; i < pBlock->info.rows; ++i) { if (!pInfo->hasGroupVal) { @@ -2653,7 +2640,7 @@ SSDataBlock* getSortedMergeBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlo SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; @@ -2754,15 +2741,19 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t goto _error; } - pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, num, &pOperator->exprSupp.rowEntryInfoOffset); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, num); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + initResultRowInfo(&pInfo->binfo.resultRowInfo); if (pOperator->exprSupp.pCtx == NULL || pInfo->binfo.pRes == NULL) { goto _error; } - size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; - int32_t code = doInitAggInfoSup(&pInfo->aggSup, pOperator->exprSupp.pCtx, num, keyBufSize, pTaskInfo->id.str); + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + code = doInitAggInfoSup(&pInfo->aggSup, pOperator->exprSupp.pCtx, num, keyBufSize, pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -2786,9 +2777,6 @@ SOperatorInfo* createSortedMergeOperatorInfo(SOperatorInfo** downstream, int32_t pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = num; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doSortedMerge, NULL, NULL, destroySortedMergeOperatorInfo, @@ -2843,8 +2831,8 @@ static int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SAggOperatorInfo* pAggInfo = pOperator->info; - SExprSupp* pSup = &pOperator->exprSupp; - SOperatorInfo* downstream = pOperator->pDownstream[0]; + SExprSupp* pSup = &pOperator->exprSupp; + SOperatorInfo* downstream = pOperator->pDownstream[0]; int64_t st = taosGetTimestampUs(); @@ -3408,7 +3396,11 @@ void cleanupAggSup(SAggSupporter* pAggSup) { int32_t initAggInfo(SExprSupp* pSup, SAggSupporter* pAggSup, SExprInfo* pExprInfo, int32_t numOfCols, size_t keyBufSize, const char* pkey) { - initExprSupp(pSup, pExprInfo, numOfCols); + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + doInitAggInfoSup(pAggSup, pSup->pCtx, numOfCols, keyBufSize, pkey); for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].pBuf = pAggSup->pResultBuf; @@ -3431,12 +3423,17 @@ void initBasicInfo(SOptrBasicInfo* pInfo, SSDataBlock* pBlock) { initResultRowInfo(&pInfo->resultRowInfo); } -void initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { +int32_t initExprSupp(SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfExpr) { pSup->pExprInfo = pExprInfo; pSup->numOfExprs = numOfExpr; if (pSup->pExprInfo != NULL) { pSup->pCtx = createSqlFunctionCtx(pExprInfo, numOfExpr, &pSup->rowEntryInfoOffset); + if (pSup->pCtx == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } } + + return TSDB_CODE_SUCCESS; } SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, @@ -3458,7 +3455,10 @@ SOperatorInfo* createAggregateOperatorInfo(SOperatorInfo* downstream, SExprInfo* } initBasicInfo(&pInfo->binfo, pResultBlock); - initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + code = initExprSupp(&pInfo->scalarExprSup, pScalarExprInfo, numOfScalarExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } pInfo->groupId = INT32_MIN; pOperator->name = "TableAggregate"; @@ -3723,13 +3723,15 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy SExprInfo* pExprInfo = createExprInfo(pPhyNode->pFuncs, NULL, &numOfExpr); if (pPhyNode->pExprs != NULL) { - SExprSupp* pSup1 = &pInfo->scalarSup; - pSup1->pExprInfo = createExprInfo(pPhyNode->pExprs, NULL, &pSup1->numOfExprs); - pSup1->pCtx = createSqlFunctionCtx(pSup1->pExprInfo, pSup1->numOfExprs, &pSup1->rowEntryInfoOffset); + int32_t num = 0; + SExprInfo* pSExpr = createExprInfo(pPhyNode->pExprs, NULL, &num); + int32_t code = initExprSupp(&pInfo->scalarSup, pSExpr, num); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } } SSDataBlock* pResBlock = createResDataBlock(pPhyNode->node.pOutputDataBlockDesc); - ; int32_t numOfRows = 4096; size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; @@ -3754,7 +3756,6 @@ SOperatorInfo* createIndefinitOutputOperatorInfo(SOperatorInfo* downstream, SPhy pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.pExprInfo = pExprInfo; pOperator->exprSupp.numOfExprs = numOfExpr; pOperator->pTaskInfo = pTaskInfo; @@ -3795,34 +3796,6 @@ static int32_t initFillInfo(SFillOperatorInfo* pInfo, SExprInfo* pExpr, int32_t } } -static int32_t convertFillType(int32_t mode) { - int32_t type = TSDB_FILL_NONE; - switch (mode) { - case FILL_MODE_PREV: - type = TSDB_FILL_PREV; - break; - case FILL_MODE_NONE: - type = TSDB_FILL_NONE; - break; - case FILL_MODE_NULL: - type = TSDB_FILL_NULL; - break; - case FILL_MODE_NEXT: - type = TSDB_FILL_NEXT; - break; - case FILL_MODE_VALUE: - type = TSDB_FILL_SET_VALUE; - break; - case FILL_MODE_LINEAR: - type = TSDB_FILL_LINEAR; - break; - default: - type = TSDB_FILL_NONE; - } - - return type; -} - SOperatorInfo* createFillOperatorInfo(SOperatorInfo* downstream, SFillPhysiNode* pPhyFillNode, bool multigroupResult, SExecTaskInfo* pTaskInfo) { SFillOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SFillOperatorInfo)); @@ -3886,8 +3859,7 @@ static SExecTaskInfo* createExecTaskInfo(uint64_t queryId, uint64_t taskId, EOPT } static tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, - SNode* pTagCond); + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId); static SArray* extractColumnInfo(SNodeList* pNodeList); @@ -3982,14 +3954,16 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } } } - int32_t len = (int32_t)(pStart - (char*)keyBuf); - uint64_t* groupId = taosHashGet(pTableListInfo->map, keyBuf, len); - if (groupId) { - taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), groupId, sizeof(uint64_t)); - } else { + + uint64_t* pGroupId = taosHashGet(pTableListInfo->map, keyBuf, len); + + if (!pGroupId) { uint64_t tmpId = calcGroupId(keyBuf, len); + info->groupId = tmpId; taosHashPut(pTableListInfo->map, &(info->uid), sizeof(uint64_t), &tmpId, sizeof(uint64_t)); + } else { + info->groupId = *pGroupId; } metaReaderClear(&mr); @@ -3999,7 +3973,7 @@ int32_t generateGroupIdMap(STableListInfo* pTableListInfo, SReadHandle* pHandle, } SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, SReadHandle* pHandle, - uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo, SNode* pTagCond) { + uint64_t queryId, uint64_t taskId, STableListInfo* pTableListInfo) { int32_t type = nodeType(pPhyNode); if (pPhyNode->pChildren == NULL || LIST_LENGTH(pPhyNode->pChildren) == 0) { @@ -4007,7 +3981,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo STableScanPhysiNode* pTableScanNode = (STableScanPhysiNode*)pPhyNode; tsdbReaderT pDataReader = - doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); + doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); if (pDataReader == NULL && terrno != 0) { pTaskInfo->code = terrno; return NULL; @@ -4035,14 +4009,9 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return pOperator; } else if (QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN == type) { STableMergeScanPhysiNode* pTableScanNode = (STableMergeScanPhysiNode*)pPhyNode; - - SArray* dataReaders = taosArrayInit(8, POINTER_BYTES); - createMultipleDataReaders(pTableScanNode, pHandle, pTableListInfo, dataReaders, queryId, taskId, pTagCond); + createScanTableListInfo(pTableScanNode, pHandle, pTableListInfo, queryId, taskId); extractTableSchemaVersion(pHandle, pTableScanNode->scan.uid, pTaskInfo); - SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); - generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json - taosArrayDestroy(groupKeys); - SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, dataReaders, pHandle, pTaskInfo); + SOperatorInfo* pOperator = createTableMergeScanOperatorInfo(pTableScanNode, pTableListInfo, pHandle, pTaskInfo, queryId, taskId); STableScanInfo* pScanInfo = pOperator->info; pTaskInfo->cost.pRecoder = &pScanInfo->readRecorder; return pOperator; @@ -4059,16 +4028,17 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo tsdbReaderT pDataReader = NULL; if (pHandle) { - if (pHandle->vnode) { - // for stram + if (pHandle->initTsdbReader) { + // for stream + ASSERT(pHandle->vnode); pDataReader = - doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId, pTagCond); + doCreateDataReader(pTableScanNode, pHandle, pTableListInfo, (uint64_t)queryId, taskId); } else { // for tq - getTableList(pHandle->meta, pScanPhyNode, pTableListInfo, pTagCond); + ASSERT(pHandle->meta); + getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); } } - if (pDataReader == NULL && terrno != 0) { qDebug("%s pDataReader is NULL", GET_TASKID(pTaskInfo)); // return NULL; @@ -4093,7 +4063,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo } else if (QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN == type) { STagScanPhysiNode* pScanPhyNode = (STagScanPhysiNode*)pPhyNode; - int32_t code = getTableList(pHandle->meta, pScanPhyNode, pTableListInfo, pScanPhyNode->node.pConditions); + int32_t code = getTableList(pHandle->meta, pScanPhyNode, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = terrno; return NULL; @@ -4101,7 +4071,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return createTagScanOperatorInfo(pHandle, pScanPhyNode, pTableListInfo, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN == type) { - SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*) pPhyNode; + SBlockDistScanPhysiNode* pBlockNode = (SBlockDistScanPhysiNode*)pPhyNode; pTableListInfo->pTableList = taosArrayInit(4, sizeof(STableKeyInfo)); if (pBlockNode->tableType == TSDB_SUPER_TABLE) { @@ -4111,7 +4081,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo return NULL; } } else { // Create one table group. - STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid}; + STableKeyInfo info = {.lastKey = 0, .uid = pBlockNode->uid, .groupId = 0}; taosArrayPush(pTableListInfo->pTableList, &info); } @@ -4151,7 +4121,7 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo SOperatorInfo** ops = taosMemoryCalloc(size, POINTER_BYTES); for (int32_t i = 0; i < size; ++i) { SPhysiNode* pChildNode = (SPhysiNode*)nodesListGetNode(pPhyNode->pChildren, i); - ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo, pTagCond); + ops[i] = createOperatorTree(pChildNode, pTaskInfo, pHandle, queryId, taskId, pTableListInfo); if (ops[i] == NULL) { return NULL; } @@ -4204,6 +4174,21 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, &as, pTaskInfo, isStream); + } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL == type) { + SMergeAlignedIntervalPhysiNode* pIntervalPhyNode = (SMergeAlignedIntervalPhysiNode*)pPhyNode; + + SExprInfo* pExprInfo = createExprInfo(pIntervalPhyNode->window.pFuncs, NULL, &num); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + + SInterval interval = {.interval = pIntervalPhyNode->interval, + .sliding = pIntervalPhyNode->sliding, + .intervalUnit = pIntervalPhyNode->intervalUnit, + .slidingUnit = pIntervalPhyNode->slidingUnit, + .offset = pIntervalPhyNode->offset, + .precision = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->node.resType.precision}; + + int32_t tsSlotId = ((SColumnNode*)pIntervalPhyNode->window.pTspk)->slotId; + pOptr = createMergeAlignedIntervalOperatorInfo(ops[0], pExprInfo, num, pResBlock, &interval, tsSlotId, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL == type) { SMergeIntervalPhysiNode* pIntervalPhyNode = (SMergeIntervalPhysiNode*)pPhyNode; @@ -4227,6 +4212,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createStreamFinalIntervalOperatorInfo(ops[0], pPhyNode, pTaskInfo, children); } else if (QUERY_NODE_PHYSICAL_PLAN_SORT == type) { pOptr = createSortOperatorInfo(ops[0], (SSortPhysiNode*)pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT == type) { + pOptr = createGroupSortOperatorInfo(ops[0], (SGroupSortPhysiNode*)pPhyNode, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_MERGE == type) { SMergePhysiNode* pMergePhyNode = (SMergePhysiNode*)pPhyNode; @@ -4282,6 +4269,8 @@ SOperatorInfo* createOperatorTree(SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo pOptr = createFillOperatorInfo(ops[0], (SFillPhysiNode*)pPhyNode, false, pTaskInfo); } else if (QUERY_NODE_PHYSICAL_PLAN_INDEF_ROWS_FUNC == type) { pOptr = createIndefinitOutputOperatorInfo(ops[0], pPhyNode, pTaskInfo); + } else if (QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC == type) { + pOptr = createTimeSliceOperatorInfo(ops[0], pPhyNode, pTaskInfo); } else { ASSERT(0); } @@ -4336,8 +4325,8 @@ SArray* extractColumnInfo(SNodeList* pNodeList) { } tsdbReaderT doCreateDataReader(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId, SNode* pTagCond) { - int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo, pTagCond); + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { + int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -4463,6 +4452,7 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, qTaskInfo_t* pT return TSDB_CODE_OUT_OF_MEMORY; } int32_t tbNum = taosArrayGetSize(pTask->tableqinfoList.pTableList); + pDeleterParam->suid = pTask->tableqinfoList.suid; pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t)); if (NULL == pDeleterParam->pUidList) { taosMemoryFree(pDeleterParam); @@ -4495,8 +4485,10 @@ int32_t createExecTaskInfoImpl(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SRead } (*pTaskInfo)->sql = sql; + (*pTaskInfo)->tableqinfoList.pTagCond = pPlan->pTagCond; + (*pTaskInfo)->tableqinfoList.pTagIndexCond = pPlan->pTagIndexCond; (*pTaskInfo)->pRoot = createOperatorTree(pPlan->pNode, *pTaskInfo, pHandle, queryId, taskId, - &(*pTaskInfo)->tableqinfoList, pPlan->pTagCond); + &(*pTaskInfo)->tableqinfoList); if (NULL == (*pTaskInfo)->pRoot) { code = (*pTaskInfo)->code; goto _complete; diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 482326a30eb24b1af1180bc4ba233d6d7e49cbe4..4e4aaba7f466595417b9878ebb367cd23ccf785e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -387,11 +387,12 @@ SOperatorInfo* createGroupOperatorInfo(SOperatorInfo* downstream, SExprInfo* pEx pInfo->pGroupCols = pGroupColList; pInfo->pCondition = pCondition; - pInfo->scalarSup.pExprInfo = pScalarExprInfo; - pInfo->scalarSup.numOfExprs = numOfScalarExpr; - pInfo->scalarSup.pCtx = createSqlFunctionCtx(pScalarExprInfo, numOfScalarExpr, &pInfo->scalarSup.rowEntryInfoOffset); + int32_t code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, numOfScalarExpr); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - int32_t code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); + code = initGroupOptrInfo(&pInfo->pGroupColVals, &pInfo->groupKeyLen, &pInfo->keyBuf, pGroupColList); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -558,8 +559,8 @@ uint64_t calcGroupId(char* pData, int32_t len) { } int32_t* setupColumnOffset(const SSDataBlock* pBlock, int32_t rowCapacity) { - size_t numOfCols = pBlock->info.numOfCols; - int32_t* offset = taosMemoryCalloc(pBlock->info.numOfCols, sizeof(int32_t)); + size_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + int32_t* offset = taosMemoryCalloc(numOfCols, sizeof(int32_t)); offset[0] = sizeof(int32_t) + sizeof(uint64_t); // the number of rows in current page, ref to SSDataBlock paged serialization format @@ -718,10 +719,12 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition pInfo->pGroupCols = extractPartitionColInfo(pPartNode->pPartitionKeys); if (pPartNode->pExprs != NULL) { - pInfo->scalarSup.numOfExprs = 0; - pInfo->scalarSup.pExprInfo = createExprInfo(pPartNode->pExprs, NULL, &pInfo->scalarSup.numOfExprs); - pInfo->scalarSup.pCtx = createSqlFunctionCtx( - pInfo->scalarSup.pExprInfo, pInfo->scalarSup.numOfExprs, &pInfo->scalarSup.rowEntryInfoOffset); + int32_t num = 0; + SExprInfo* pExprInfo1 = createExprInfo(pPartNode->pExprs, NULL, &num); + int32_t code = initExprSupp(&pInfo->scalarSup, pExprInfo1, num); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } } _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index e4f940d991789d7341f558de70e0ed8785fb82a2..b521983d8e83a88962d871ab05372c4ee8143d38 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -112,12 +112,12 @@ static void getNextTimeWindow(SInterval* pInterval, STimeWindow* tw, int32_t ord int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); mon = (int)(mon + interval); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, pInterval->precision); + tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, pInterval->precision); tw->ekey -= 1; } @@ -203,7 +203,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -217,14 +217,14 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanInfo* pTableSca tsdbRetrieveDataBlockStatisInfo(pTableScanInfo->dataReader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { pBlock->pBlockAgg = taosMemoryCalloc(numOfCols, POINTER_BYTES); } - for (int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pTableScanInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { continue; @@ -316,7 +316,7 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, dstSlotId); - colInfoDataEnsureCapacity(pColInfoData, 0, pBlock->info.rows); + colInfoDataEnsureCapacity(pColInfoData, pBlock->info.rows); colInfoDataCleanup(pColInfoData, pBlock->info.rows); int32_t functionId = pExpr->pExpr->_function.functionId; @@ -337,7 +337,8 @@ void addTagPseudoColumnData(SReadHandle* pHandle, SExprInfo* pPseudoExpr, int32_ } for (int32_t i = 0; i < pBlock->info.rows; ++i) { - colDataAppend(pColInfoData, i, data, (data == NULL)); + colDataAppend(pColInfoData, i, data, + (data == NULL) || (pColInfoData->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); } if (data && (pColInfoData->info.type != TSDB_DATA_TYPE_JSON) && p != NULL && @@ -354,10 +355,8 @@ void setTbNameColData(void* pMeta, const SSDataBlock* pBlock, SColumnInfoData* p struct SScalarFuncExecFuncs fpSet = {0}; fmGetScalarFuncExecFuncs(functionId, &fpSet); - SColumnInfoData infoData = {0}; - infoData.info.type = TSDB_DATA_TYPE_BIGINT; - infoData.info.bytes = sizeof(uint64_t); - colInfoDataEnsureCapacity(&infoData, 0, 1); + SColumnInfoData infoData = createColumnInfoData(TSDB_DATA_TYPE_BIGINT, sizeof(uint64_t), 1); + colInfoDataEnsureCapacity(&infoData, 1); colDataAppendInt64(&infoData, 0, (int64_t*)&pBlock->info.uid); SScalarParam srcParam = {.numOfRows = pBlock->info.rows, .param = pMeta, .columnData = &infoData}; @@ -384,7 +383,14 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { continue; } - tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &pBlock->info); + blockDataCleanup(pBlock); + + SDataBlockInfo binfo = pBlock->info; + tsdbRetrieveDataBlockInfo(pTableScanInfo->dataReader, &binfo); + + binfo.capacity = binfo.rows; + blockDataEnsureCapacity(pBlock, binfo.rows); + pBlock->info = binfo; uint32_t status = 0; int32_t code = loadDataBlock(pOperator, pTableScanInfo, pBlock, &status); @@ -530,8 +536,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, // taosSsleep(20); SDataBlockDescNode* pDescNode = pTableScanNode->scan.node.pOutputDataBlockDesc; - - int32_t numOfCols = 0; + int32_t numOfCols = 0; SArray* pColList = extractColMatchInfo(pTableScanNode->scan.pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID); int32_t code = initQueryTableDataCond(&pInfo->cond, pTableScanNode); @@ -591,12 +596,12 @@ SOperatorInfo* createTableSeqScanOperatorInfo(void* pReadHandle, SExecTaskInfo* pInfo->dataReader = pReadHandle; // pInfo->prevGroupId = -1; - pOperator->name = "TableSeqScanOperator"; + pOperator->name = "TableSeqScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_SEQ_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTableScanImpl, NULL, NULL, NULL, NULL, NULL, NULL); return pOperator; @@ -610,7 +615,7 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) { metaGetTableEntryByUid(&mr, uid); if (mr.me.type == TSDB_SUPER_TABLE) { int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes; } } else if (mr.me.type == TSDB_CHILD_TABLE) { @@ -618,12 +623,12 @@ static int32_t doGetTableRowSize(void* pMeta, uint64_t uid) { metaGetTableEntryByUid(&mr, suid); int32_t numOfCols = mr.me.stbEntry.schemaRow.nCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { rowLen += mr.me.stbEntry.schemaRow.pSchema[i].bytes; } } else if (mr.me.type == TSDB_NORMAL_TABLE) { int32_t numOfCols = mr.me.ntbEntry.schemaRow.nCols; - for(int32_t i = 0; i < numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { rowLen += mr.me.ntbEntry.schemaRow.pSchema[i].bytes; } } @@ -647,7 +652,7 @@ static SSDataBlock* doBlockInfoScan(SOperatorInfo* pOperator) { SSDataBlock* pBlock = pBlockScanInfo->pResBlock; - int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; + int32_t slotId = pOperator->exprSupp.pExprInfo->base.resSchema.slotId; SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, slotId); int32_t len = tSerializeBlockDistInfo(NULL, 0, &blockDistInfo); @@ -679,20 +684,23 @@ SOperatorInfo* createDataBlockInfoScanOperator(void* dataReader, SReadHandle* re goto _error; } - pInfo->pHandle = dataReader; + pInfo->pHandle = dataReader; pInfo->readHandle = *readHandle; - pInfo->uid = uid; - pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc); + pInfo->uid = uid; + pInfo->pResBlock = createResDataBlock(pBlockScanNode->node.pOutputDataBlockDesc); - int32_t numOfCols = 0; + int32_t numOfCols = 0; SExprInfo* pExprInfo = createExprInfo(pBlockScanNode->pScanPseudoCols, NULL, &numOfCols); - initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - pOperator->name = "DataBlockDistScanOperator"; + pOperator->name = "DataBlockDistScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doBlockInfoScan, NULL, NULL, @@ -767,7 +775,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { if (!needRead) { return false; } - STableScanInfo* pTableScanInfo = pInfo->pOperatorDumy->info; + STableScanInfo* pTableScanInfo = pInfo->pSnapshotReadOp->info; pTableScanInfo->cond.twindows[0] = win; pTableScanInfo->curTWinIdx = 0; tsdbResetReadHandle(pTableScanInfo->dataReader, &pTableScanInfo->cond, 0); @@ -776,7 +784,7 @@ static bool prepareDataScan(SStreamBlockScanInfo* pInfo) { } static void copyOneRow(SSDataBlock* dest, SSDataBlock* source, int32_t sourceRowId) { - for (int32_t j = 0; j < source->info.numOfCols; j++) { + for (int32_t j = 0; j < taosArrayGetSize(source->pDataBlock); j++) { SColumnInfoData* pDestCol = (SColumnInfoData*)taosArrayGet(dest->pDataBlock, j); SColumnInfoData* pSourceCol = (SColumnInfoData*)taosArrayGet(source->pDataBlock, j); if (colDataIsNull_s(pSourceCol, sourceRowId)) { @@ -812,11 +820,11 @@ static uint64_t getGroupId(SOperatorInfo* pOperator, SSDataBlock* pBlock, int32_ static SSDataBlock* doDataScan(SStreamBlockScanInfo* pInfo) { while (1) { SSDataBlock* pResult = NULL; - pResult = doTableScan(pInfo->pOperatorDumy); + pResult = doTableScan(pInfo->pSnapshotReadOp); if (pResult == NULL) { if (prepareDataScan(pInfo)) { // scan next window data - pResult = doTableScan(pInfo->pOperatorDumy); + pResult = doTableScan(pInfo->pSnapshotReadOp); } } if (!pResult) { @@ -848,14 +856,13 @@ static void setUpdateData(SStreamBlockScanInfo* pInfo, SSDataBlock* pBlock, SSDa SColumnInfoData* pCol = (SColumnInfoData*)taosArrayGet(pUpdateBlock->pDataBlock, pInfo->primaryTsIndex); ASSERT(pCol->info.type == TSDB_DATA_TYPE_TIMESTAMP); blockDataEnsureCapacity(pUpdateBlock, size); - ASSERT(pBlock->info.numOfCols == pUpdateBlock->info.numOfCols); int32_t rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, pInfo->tsArrayIndex); - pInfo->groupId = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + pInfo->groupId = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId); int32_t i = 0; for (; i < size; i++) { rowId = *(int32_t*)taosArrayGet(pInfo->tsArray, i + pInfo->tsArrayIndex); - uint64_t id = getGroupId(pInfo->pOperatorDumy, pBlock, rowId); + uint64_t id = getGroupId(pInfo->pSnapshotReadOp, pBlock, rowId); if (pInfo->groupId != id) { break; } @@ -900,7 +907,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // NOTE: this operator does never check if current status is done or not SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamBlockScanInfo* pInfo = pOperator->info; - int32_t rows = 0; pTaskInfo->code = pOperator->fpSet._openFn(pOperator); if (pTaskInfo->code != TSDB_CODE_SUCCESS || pOperator->status == OP_EXEC_DONE) { @@ -908,6 +914,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } size_t total = taosArrayGetSize(pInfo->pBlockLists); + // TODO: refactor if (pInfo->blockType == STREAM_DATA_TYPE_SSDATA_BLOCK) { if (pInfo->validBlockIndex >= total) { /*doClearBufferedBlocks(pInfo);*/ @@ -919,7 +926,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { SSDataBlock* pBlock = taosArrayGetP(pInfo->pBlockLists, current); blockDataUpdateTsWindow(pBlock, 0); return pBlock; - } else { + } else if (pInfo->blockType == STREAM_DATA_TYPE_SUBMIT_BLOCK) { if (pInfo->scanMode == STREAM_SCAN_FROM_RES) { blockDataDestroy(pInfo->pUpdateRes); pInfo->scanMode = STREAM_SCAN_FROM_READERHANDLE; @@ -959,13 +966,13 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { blockDataCleanup(pInfo->pRes); while (tqNextDataBlock(pInfo->streamBlockReader)) { - SArray* pCols = NULL; + SSDataBlock block = {0}; uint64_t groupId = 0; uint64_t uid = 0; int32_t numOfRows = 0; - int16_t outputCol = 0; - int32_t code = tqRetrieveDataBlock(&pCols, pInfo->streamBlockReader, &groupId, &uid, &numOfRows, &outputCol); + // todo refactor + int32_t code = tqRetrieveDataBlock(&block, pInfo->streamBlockReader, &groupId, &uid, &numOfRows); if (code != TSDB_CODE_SUCCESS || numOfRows == 0) { pTaskInfo->code = code; @@ -976,6 +983,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.rows = numOfRows; pInfo->pRes->info.uid = uid; pInfo->pRes->info.type = STREAM_NORMAL; + pInfo->pRes->info.capacity = numOfRows; // for generating rollup SMA result, each time is an independent time serie. // TODO temporarily used, when the statement of "partition by tbname" is ready, remove this @@ -990,6 +998,7 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pInfo->pRes->info.groupId = *groupIdPre; } + // todo extract method for (int32_t i = 0; i < taosArrayGetSize(pInfo->pColMatchInfo); ++i) { SColMatchInfo* pColMatchInfo = taosArrayGet(pInfo->pColMatchInfo, i); if (!pColMatchInfo->output) { @@ -997,8 +1006,8 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } bool colExists = false; - for (int32_t j = 0; j < taosArrayGetSize(pCols); ++j) { - SColumnInfoData* pResCol = taosArrayGet(pCols, j); + for (int32_t j = 0; j < blockDataGetNumOfCols(&block); ++j) { + SColumnInfoData* pResCol = bdGetColumnInfoData(&block, j); if (pResCol->info.colId == pColMatchInfo->colId) { taosArraySet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId, pResCol); colExists = true; @@ -1009,7 +1018,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { // the required column does not exists in submit block, let's set it to be all null value if (!colExists) { SColumnInfoData* pDst = taosArrayGet(pInfo->pRes->pDataBlock, pColMatchInfo->targetSlotId); - colInfoDataEnsureCapacity(pDst, 0, pBlockInfo->rows); colDataAppendNNULL(pDst, 0, pBlockInfo->rows); } } @@ -1020,9 +1028,6 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { pTaskInfo->code = terrno; return NULL; } - - rows = pBlockInfo->rows; - // currently only the tbname pseudo column if (pInfo->numOfPseudoExpr > 0) { addTagPseudoColumnData(&pInfo->readHandle, pInfo->pPseudoExpr, pInfo->numOfPseudoExpr, pInfo->pRes); @@ -1030,14 +1035,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { doFilter(pInfo->pCondition, pInfo->pRes); blockDataUpdateTsWindow(pInfo->pRes, pInfo->primaryTsIndex); - break; + if (pBlockInfo->rows > 0) { + break; + } } // record the scan action. pInfo->numOfExec++; pOperator->resultInfo.totalRows += pBlockInfo->rows; - if (rows == 0) { + if (pBlockInfo->rows == 0) { pOperator->status = OP_EXEC_DONE; } else if (pInfo->pUpdateInfo) { pInfo->tsArrayIndex = 0; @@ -1053,7 +1060,16 @@ static SSDataBlock* doStreamBlockScan(SOperatorInfo* pOperator) { } } - return (rows == 0) ? NULL : pInfo->pRes; + return (pBlockInfo->rows == 0) ? NULL : pInfo->pRes; + } else if (pInfo->blockType == STREAM_DATA_TYPE_FROM_SNAPSHOT) { + SSDataBlock* pResult = doTableScan(pInfo->pSnapshotReadOp); + if (pResult) { + return pResult->info.rows > 0 ? pResult : NULL; + } + return NULL; + } else { + ASSERT(0); + return NULL; } } @@ -1118,7 +1134,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan } else { pInfo->pUpdateInfo = NULL; } - pInfo->pOperatorDumy = pTableScanDummy; + pInfo->pSnapshotReadOp = pTableScanDummy; pInfo->interval = pSTInfo->interval; pInfo->readHandle = *pHandle; @@ -1155,7 +1171,7 @@ SOperatorInfo* createStreamScanOperatorInfo(void* pDataReader, SReadHandle* pHan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pInfo->pRes->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -1283,7 +1299,7 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { SColumnInfoData* pSrc = taosArrayGet(pInfo->pRes->pDataBlock, i); if (keep) { - colDataAssign(pDest, pSrc, pInfo->pRes->info.rows); + colDataAssign(pDest, pSrc, pInfo->pRes->info.rows, &px->info); numOfRow = pInfo->pRes->info.rows; } else if (NULL != rowRes) { numOfRow = 0; @@ -1313,8 +1329,6 @@ static SSDataBlock* doFilterResult(SSysTableScanInfo* pInfo) { } static SSDataBlock* buildSysTableMetaBlock() { - SSDataBlock* pBlock = taosMemoryCalloc(1, sizeof(SSDataBlock)); - size_t size = 0; const SSysTableMeta* pMeta = NULL; getInfosDbMeta(&pMeta, &size); @@ -1327,19 +1341,12 @@ static SSDataBlock* buildSysTableMetaBlock() { } } - pBlock->pDataBlock = taosArrayInit(pBlock->info.numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pBlock = createDataBlock(); for (int32_t i = 0; i < pMeta[index].colNum; ++i) { - SColumnInfoData colInfoData = {0}; - colInfoData.info.colId = i + 1; - colInfoData.info.type = pMeta[index].schema[i].type; - colInfoData.info.bytes = pMeta[index].schema[i].bytes; - taosArrayPush(pBlock->pDataBlock, &colInfoData); + SColumnInfoData colInfoData = createColumnInfoData(pMeta[index].schema[i].type, pMeta[index].schema[i].bytes, i + 1); + blockDataAppendColInfo(pBlock, &colInfoData); } - pBlock->info.numOfCols = pMeta[index].colNum; - pBlock->info.hasVarCol = true; - return pBlock; } @@ -1370,7 +1377,6 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { } blockDataCleanup(pInfo->pRes); - int32_t numOfRows = 0; const char* db = NULL; @@ -1540,11 +1546,15 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { return NULL; } + int32_t msgType = (strcasecmp(name, TSDB_INS_TABLE_DNODE_VARIABLES) == 0) ? TDMT_DND_SYSTABLE_RETRIEVE + : TDMT_MND_SYSTABLE_RETRIEVE; + pMsgSendInfo->param = pOperator; pMsgSendInfo->msgInfo.pData = buf1; pMsgSendInfo->msgInfo.len = contLen; - pMsgSendInfo->msgType = TDMT_MND_SYSTABLE_RETRIEVE; + pMsgSendInfo->msgType = msgType; pMsgSendInfo->fp = loadSysTableCallback; + pMsgSendInfo->requestId = pTaskInfo->id.queryId; int64_t transporterId = 0; int32_t code = @@ -1562,7 +1572,7 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { if (pRsp->numOfRows == 0 || pRsp->completed) { pOperator->status = OP_EXEC_DONE; - qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64 " ", GET_TASKID(pTaskInfo), + qDebug("%s load meta data from mnode completed, rowsOfSource:%d, totalRows:%" PRIu64, GET_TASKID(pTaskInfo), pRsp->numOfRows, pInfo->loadInfo.totalRows); if (pRsp->numOfRows == 0) { @@ -1579,6 +1589,8 @@ static SSDataBlock* doSysTableScan(SOperatorInfo* pOperator) { taosMemoryFree(pRsp); if (pInfo->pRes->info.rows > 0) { return pInfo->pRes; + } else if (pOperator->status == OP_EXEC_DONE) { + return NULL; } } } @@ -1687,7 +1699,7 @@ SOperatorInfo* createSysTableScanOperatorInfo(void* readHandle, SSystemTableScan pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; pOperator->info = pInfo; - pOperator->exprSupp.numOfExprs = pResBlock->info.numOfCols; + pOperator->exprSupp.numOfExprs = taosArrayGetSize(pResBlock->pDataBlock); pOperator->pTaskInfo = pTaskInfo; pOperator->fpSet = @@ -1821,7 +1833,8 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } else { data = (char*)p; } - colDataAppend(pDst, count, data, (data == NULL)); + colDataAppend(pDst, count, data, + (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && data != NULL) { @@ -1844,9 +1857,7 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { } pRes->info.rows = count; - doFilter(pInfo->pFilterNode, pRes); - - pOperator->resultInfo.totalRows += pRes->info.rows; + pOperator->resultInfo.totalRows += count; return (pRes->info.rows == 0) ? NULL : pInfo->pRes; } @@ -1871,23 +1882,23 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi SExprInfo* pExprInfo = createExprInfo(pPhyNode->pScanPseudoCols, NULL, &numOfExprs); SArray* colList = extractColMatchInfo(pPhyNode->pScanPseudoCols, pDescNode, &num, COL_MATCH_FROM_COL_ID); + int32_t code = initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } - initExprSupp(&pOperator->exprSupp, pExprInfo, numOfExprs); - - pInfo->pTableList = pTableListInfo; - pInfo->pColMatchInfo = colList; - pInfo->pRes = createResDataBlock(pDescNode); - pInfo->readHandle = *pReadHandle; - pInfo->curPos = 0; - pInfo->pFilterNode = pPhyNode->node.pConditions; - + pInfo->pTableList = pTableListInfo; + pInfo->pColMatchInfo = colList; + pInfo->pRes = createResDataBlock(pDescNode); + pInfo->readHandle = *pReadHandle; + pInfo->curPos = 0; pOperator->name = "TagScanOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; initResultSizeInfo(pOperator, 4096); blockDataEnsureCapacity(pInfo->pRes, pOperator->resultInfo.capacity); @@ -1905,6 +1916,12 @@ _error: } typedef struct STableMergeScanInfo { + STableListInfo* tableListInfo; + int32_t tableStartIndex; + int32_t tableEndIndex; + bool hasGroupId; + uint64_t groupId; + SArray* dataReaders; // array of tsdbReaderT* SReadHandle readHandle; @@ -1917,11 +1934,9 @@ typedef struct STableMergeScanInfo { SSDataBlock* pSortInputBlock; int64_t startTs; // sort start time - bool hasGroupId; - uint64_t groupId; - STupleHandle* prefetchedTuple; - - SArray* sortSourceParams; + SArray* sortSourceParams; + uint64_t queryId; + uint64_t taskId; SFileBlockLoadRecorder readRecorder; int64_t numOfRows; @@ -1949,29 +1964,42 @@ typedef struct STableMergeScanInfo { // window to check if current data block needs to be loaded. SSampleExecInfo sample; // sample execution info - int32_t curTWinIdx; - } STableMergeScanInfo; -int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, - STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId, - uint64_t taskId, SNode* pTagCond) { - int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo, pTagCond); +int32_t compareTableKeyInfoByGid(const void* p1, const void* p2) { + const STableKeyInfo* info1 = p1; + const STableKeyInfo* info2 = p2; + return info1->groupId - info2->groupId; +} + +int32_t createScanTableListInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + STableListInfo* pTableListInfo, uint64_t queryId, uint64_t taskId) { + int32_t code = getTableList(pHandle->meta, &pTableScanNode->scan, pTableListInfo); if (code != TSDB_CODE_SUCCESS) { - goto _error; + return code; } if (taosArrayGetSize(pTableListInfo->pTableList) == 0) { qDebug("no table qualified for query, TID:0x%" PRIx64 ", QID:0x%" PRIx64, taskId, queryId); - goto _error; + return TSDB_CODE_SUCCESS; + } + SArray* groupKeys = extractPartitionColInfo(pTableScanNode->pPartitionTags); + generateGroupIdMap(pTableListInfo, pHandle, groupKeys); // todo for json + if (groupKeys) { + taosArraySort(pTableListInfo->pTableList, compareTableKeyInfoByGid); } + taosArrayDestroy(groupKeys); + return TSDB_CODE_SUCCESS; +} +int32_t doCreateMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHandle* pHandle, + STableListInfo* pTableListInfo, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { SQueryTableDataCond cond = {0}; - code = initQueryTableDataCond(&cond, pTableScanNode); + int32_t code = initQueryTableDataCond(&cond, pTableScanNode); if (code != TSDB_CODE_SUCCESS) { goto _error; } - // TODO: free the sublist info and the table list in it for (int32_t i = 0; i < taosArrayGetSize(pTableListInfo->pTableList); ++i) { STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo)); subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); @@ -1985,12 +2013,31 @@ int32_t createMultipleDataReaders(STableScanPhysiNode* pTableScanNode, SReadHand } cleanupQueryTableDataCond(&cond); - return 0; + return TSDB_CODE_SUCCESS; _error: return code; } +int32_t createMultipleDataReaders(SQueryTableDataCond* pQueryCond, SReadHandle* pHandle, STableListInfo* pTableListInfo, + int32_t tableStartIdx, int32_t tableEndIdx, SArray* arrayReader, uint64_t queryId, + uint64_t taskId) { + for (int32_t i = tableStartIdx; i <= tableEndIdx; ++i) { + STableListInfo* subListInfo = taosMemoryCalloc(1, sizeof(subListInfo)); + subListInfo->pTableList = taosArrayInit(1, sizeof(STableKeyInfo)); + taosArrayPush(subListInfo->pTableList, taosArrayGet(pTableListInfo->pTableList, i)); + + tsdbReaderT* pReader = tsdbReaderOpen(pHandle->vnode, pQueryCond, subListInfo, queryId, taskId); + taosArrayPush(arrayReader, &pReader); + + taosArrayDestroy(subListInfo->pTableList); + taosMemoryFree(subListInfo); + } + + return TSDB_CODE_SUCCESS; +} + +// todo refactor static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeScanInfo* pTableScanInfo, int32_t readerIdx, SSDataBlock* pBlock, uint32_t* status) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; @@ -2021,7 +2068,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc pCost->skipBlocks += 1; // clear all data in pBlock that are set when handing the previous block - for (int32_t i = 0; i < pBlockInfo->numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pcol = taosArrayGet(pBlock->pDataBlock, i); pcol->pData = NULL; } @@ -2036,7 +2083,7 @@ static int32_t loadDataBlockFromOneTable(SOperatorInfo* pOperator, STableMergeSc tsdbRetrieveDataBlockStatisInfo(reader, &pColAgg, &allColumnsHaveAgg); if (allColumnsHaveAgg == true) { - int32_t numOfCols = pBlock->info.numOfCols; + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); // todo create this buffer during creating operator if (pBlock->pBlockAgg == NULL) { @@ -2131,7 +2178,13 @@ static SSDataBlock* getTableDataBlock(void* param) { continue; } - tsdbRetrieveDataBlockInfo(reader, &pBlock->info); + blockDataCleanup(pBlock); + SDataBlockInfo binfo = pBlock->info; + tsdbRetrieveDataBlockInfo(reader, &binfo); + + binfo.capacity = binfo.rows; + blockDataEnsureCapacity(pBlock, binfo.capacity); + pBlock->info = binfo; uint32_t status = 0; int32_t code = loadDataBlockFromOneTable(pOperator, pTableScanInfo, readerIdx, pBlock, &status); @@ -2170,22 +2223,47 @@ SArray* generateSortByTsInfo(int32_t order) { return pList; } -int32_t doOpenTableMergeScanOperator(SOperatorInfo* pOperator) { +int32_t startGroupTableMergeScan(SOperatorInfo* pOperator) { STableMergeScanInfo* pInfo = pOperator->info; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (OPTR_IS_OPENED(pOperator)) { - return TSDB_CODE_SUCCESS; + { + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + int32_t i = pInfo->tableStartIndex + 1; + for (; i < tableListSize; ++i) { + STableKeyInfo* tableKeyInfo = taosArrayGet(pInfo->tableListInfo->pTableList, i); + if (tableKeyInfo->groupId != pInfo->groupId) { + break; + } + } + pInfo->tableEndIndex = i - 1; } - int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; + int32_t tableStartIdx = pInfo->tableStartIndex; + int32_t tableEndIdx = pInfo->tableEndIndex; + + STableListInfo* tableListInfo = pInfo->tableListInfo; + createMultipleDataReaders(&pInfo->cond, &pInfo->readHandle, tableListInfo, tableStartIdx, tableEndIdx, + pInfo->dataReaders, pInfo->queryId, pInfo->taskId); + // todo the total available buffer should be determined by total capacity of buffer of this task. + // the additional one is reserved for merge result + pInfo->sortBufSize = pInfo->bufPageSize * (tableEndIdx - tableStartIdx + 1 + 1); + int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, pInfo->pSortInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, getTableDataBlock, NULL, NULL); size_t numReaders = taosArrayGetSize(pInfo->dataReaders); + for (int32_t i = 0; i < numReaders; ++i) { + STableMergeScanSortSourceParam param = {0}; + param.readerIdx = i; + param.pOperator = pOperator; + param.inputBlock = createOneDataBlock(pInfo->pResBlock, false); + taosArrayPush(pInfo->sortSourceParams, ¶m); + } + for (int32_t i = 0; i < numReaders; ++i) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); STableMergeScanSortSourceParam* param = taosArrayGet(pInfo->sortSourceParams, i); @@ -2199,9 +2277,22 @@ int32_t doOpenTableMergeScanOperator(SOperatorInfo* pOperator) { longjmp(pTaskInfo->env, terrno); } - pOperator->status = OP_RES_TO_RETURN; + return TSDB_CODE_SUCCESS; +} + +int32_t stopGroupTableMergeScan(SOperatorInfo* pOperator) { + STableMergeScanInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + tsortDestroySortHandle(pInfo->pSortHandle); + taosArrayClear(pInfo->sortSourceParams); + + for (int32_t i = 0; i < taosArrayGetSize(pInfo->dataReaders); ++i) { + tsdbReaderT* reader = taosArrayGetP(pInfo->dataReaders, i); + tsdbCleanupReadHandle(reader); + } + taosArrayDestroy(pInfo->dataReaders); - OPTR_SET_OPENED(pOperator); return TSDB_CODE_SUCCESS; } @@ -2244,14 +2335,38 @@ SSDataBlock* doTableMergeScan(SOperatorInfo* pOperator) { if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, code); } + size_t tableListSize = taosArrayGetSize(pInfo->tableListInfo->pTableList); + if (!pInfo->hasGroupId) { + pInfo->hasGroupId = true; - SSDataBlock* pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); - - if (pBlock != NULL) { - pOperator->resultInfo.totalRows += pBlock->info.rows; - } else { - doSetOperatorCompleted(pOperator); + if (tableListSize == 0) { + doSetOperatorCompleted(pOperator); + return NULL; + } + pInfo->tableStartIndex = 0; + pInfo->groupId = ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; + startGroupTableMergeScan(pOperator); + } + SSDataBlock* pBlock = NULL; + while (pInfo->tableStartIndex < tableListSize) { + pBlock = getSortedTableMergeScanBlockData(pInfo->pSortHandle, pOperator->resultInfo.capacity, pOperator); + if (pBlock != NULL) { + pBlock->info.groupId = pInfo->groupId; + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } else { + stopGroupTableMergeScan(pOperator); + if (pInfo->tableEndIndex >= tableListSize - 1) { + doSetOperatorCompleted(pOperator); + break; + } + pInfo->tableStartIndex = pInfo->tableEndIndex + 1; + pInfo->groupId = + ((STableKeyInfo*)taosArrayGet(pInfo->tableListInfo->pTableList, pInfo->tableStartIndex))->groupId; + startGroupTableMergeScan(pOperator); + } } + return pBlock; } @@ -2259,17 +2374,10 @@ void destroyTableMergeScanOperatorInfo(void* param, int32_t numOfOutput) { STableMergeScanInfo* pTableScanInfo = (STableMergeScanInfo*)param; cleanupQueryTableDataCond(&pTableScanInfo->cond); - for (int32_t i = 0; i < taosArrayGetSize(pTableScanInfo->dataReaders); ++i) { - tsdbReaderT* reader = taosArrayGetP(pTableScanInfo->dataReaders, i); - tsdbCleanupReadHandle(reader); - } - taosArrayDestroy(pTableScanInfo->dataReaders); - if (pTableScanInfo->pColMatchInfo != NULL) { taosArrayDestroy(pTableScanInfo->pColMatchInfo); } - taosArrayDestroy(pTableScanInfo->sortSourceParams); pTableScanInfo->pResBlock = blockDataDestroy(pTableScanInfo->pResBlock); pTableScanInfo->pSortInputBlock = blockDataDestroy(pTableScanInfo->pSortInputBlock); @@ -2295,8 +2403,9 @@ int32_t getTableMergeScanExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExpla return TSDB_CODE_SUCCESS; } -SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SArray* dataReaders, - SReadHandle* readHandle, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanNode, STableListInfo* pTableListInfo, + SReadHandle* readHandle, SExecTaskInfo* pTaskInfo, uint64_t queryId, + uint64_t taskId) { STableMergeScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableMergeScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { @@ -2326,22 +2435,16 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN pInfo->sample.seed = taosGetTimestampSec(); pInfo->dataBlockLoadFlag = pTableScanNode->dataRequired; pInfo->pFilterNode = pTableScanNode->scan.node.pConditions; - pInfo->dataReaders = dataReaders; + pInfo->tableListInfo = pTableListInfo; pInfo->scanFlag = MAIN_SCAN; pInfo->pColMatchInfo = pColList; - pInfo->curTWinIdx = 0; pInfo->pResBlock = createResDataBlock(pDescNode); + pInfo->dataReaders = taosArrayInit(64, POINTER_BYTES); + pInfo->queryId = queryId; + pInfo->taskId = taskId; - pInfo->sortSourceParams = taosArrayInit(taosArrayGetSize(dataReaders), sizeof(STableMergeScanSortSourceParam)); - for (int32_t i = 0; i < taosArrayGetSize(dataReaders); ++i) { - STableMergeScanSortSourceParam* param = taosMemoryCalloc(1, sizeof(STableMergeScanSortSourceParam)); - param->readerIdx = i; - param->pOperator = pOperator; - param->inputBlock = createOneDataBlock(pInfo->pResBlock, false); - taosArrayPush(pInfo->sortSourceParams, param); - taosMemoryFree(param); - } + pInfo->sortSourceParams = taosArrayInit(64, sizeof(STableMergeScanSortSourceParam)); pInfo->pSortInfo = generateSortByTsInfo(pInfo->cond.order); pInfo->pSortInputBlock = createOneDataBlock(pInfo->pResBlock, false); @@ -2349,14 +2452,7 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN int32_t rowSize = pInfo->pResBlock->info.rowSize; pInfo->bufPageSize = getProperSortPageSize(rowSize); - // todo the total available buffer should be determined by total capacity of buffer of this task. - // the additional one is reserved for merge result - pInfo->sortBufSize = pInfo->bufPageSize * (taosArrayGetSize(dataReaders) + 1); - pInfo->hasGroupId = false; - pInfo->prefetchedTuple = NULL; - pOperator->name = "TableMergeScanOperator"; - // TODO : change it pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; pOperator->blocking = false; pOperator->status = OP_NOT_OPENED; @@ -2366,8 +2462,8 @@ SOperatorInfo* createTableMergeScanOperatorInfo(STableScanPhysiNode* pTableScanN initResultSizeInfo(pOperator, 1024); pOperator->fpSet = - createOperatorFpSet(doOpenTableMergeScanOperator, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo, - NULL, NULL, getTableMergeScanExplainExecInfo); + createOperatorFpSet(operatorDummyOpenFn, doTableMergeScan, NULL, NULL, destroyTableMergeScanOperatorInfo, NULL, + NULL, getTableMergeScanExplainExecInfo); pOperator->cost.openCost = 0; return pOperator; diff --git a/source/libs/executor/src/sortoperator.c b/source/libs/executor/src/sortoperator.c index 9d13276e6d68d2aaf380d0ac3111c2214c6f0e75..fe95dc613e8c1be656619fecb3333dae04e3d017 100644 --- a/source/libs/executor/src/sortoperator.c +++ b/source/libs/executor/src/sortoperator.c @@ -22,10 +22,11 @@ static int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain static void destroyOrderOperatorInfo(void* param, int32_t numOfOutput); -SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* pSortPhyNode, + SExecTaskInfo* pTaskInfo) { SSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SSortOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); - if (pInfo == NULL || pOperator == NULL/* || rowSize > 100 * 1024 * 1024*/) { + if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { goto _error; } @@ -44,16 +45,17 @@ SOperatorInfo* createSortOperatorInfo(SOperatorInfo* downstream, SSortPhysiNode* initResultSizeInfo(pOperator, 1024); - pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys);; - pInfo->pColMatchInfo = pColMatchColInfo; - pOperator->name = "SortOperator"; + pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); + ; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "SortOperator"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_SORT; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->pTaskInfo = pTaskInfo; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->pTaskInfo = pTaskInfo; // lazy evaluation for the following parameter since the input datablock is not known till now. // pInfo->bufPageSize = rowSize < 1024 ? 1024 * 2 : rowSize * 2; @@ -78,7 +80,7 @@ _error: } void appendOneRowToDataBlock(SSDataBlock* pBlock, STupleHandle* pTupleHandle) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); bool isNull = tsortIsNullVal(pTupleHandle, i); if (isNull) { @@ -118,6 +120,9 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i } if (p->info.rows > 0) { + blockDataEnsureCapacity(pDataBlock, capacity); + + // todo extract function to handle this int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -125,11 +130,10 @@ SSDataBlock* getSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, i SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; } blockDataDestroy(p); @@ -146,8 +150,8 @@ void applyScalarFunction(SSDataBlock* pBlock, void* param) { SOperatorInfo* pOperator = param; SSortOperatorInfo* pSort = pOperator->info; if (pOperator->exprSupp.pExprInfo != NULL) { - int32_t code = - projectApplyFunctions(pOperator->exprSupp.pExprInfo, pBlock, pBlock, pOperator->exprSupp.pCtx, pOperator->exprSupp.numOfExprs, NULL); + int32_t code = projectApplyFunctions(pOperator->exprSupp.pExprInfo, pBlock, pBlock, pOperator->exprSupp.pCtx, + pOperator->exprSupp.numOfExprs, NULL); if (code != TSDB_CODE_SUCCESS) { longjmp(pOperator->pTaskInfo->env, code); } @@ -165,8 +169,7 @@ int32_t doOpenSortOperator(SOperatorInfo* pOperator) { pInfo->startTs = taosGetTimestampUs(); // pInfo->binfo.pRes is not equalled to the input datablock. - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, - NULL, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, applyScalarFunction, pOperator); @@ -232,6 +235,260 @@ int32_t getExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* return TSDB_CODE_SUCCESS; } +//===================================================================================== +// Group Sort Operator +typedef enum EChildOperatorStatus { CHILD_OP_NEW_GROUP, CHILD_OP_SAME_GROUP, CHILD_OP_FINISHED } EChildOperatorStatus; + +typedef struct SGroupSortOperatorInfo { + SOptrBasicInfo binfo; + SArray* pSortInfo; + SArray* pColMatchInfo; + + int64_t startTs; + uint64_t sortElapsed; + bool hasGroupId; + uint64_t currGroupId; + + SSDataBlock* prefetchedSortInput; + SSortHandle* pCurrSortHandle; + EChildOperatorStatus childOpStatus; + + SSortExecInfo sortExecInfo; +} SGroupSortOperatorInfo; + +SSDataBlock* getGroupSortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, + SArray* pColMatchInfo, SGroupSortOperatorInfo* pInfo) { + blockDataCleanup(pDataBlock); + blockDataEnsureCapacity(pDataBlock, capacity); + + SSDataBlock* p = tsortGetSortedDataBlock(pHandle); + if (p == NULL) { + return NULL; + } + + blockDataEnsureCapacity(p, capacity); + + while (1) { + STupleHandle* pTupleHandle = tsortNextTuple(pHandle); + if (pTupleHandle == NULL) { + break; + } + + appendOneRowToDataBlock(p, pTupleHandle); + if (p->info.rows >= capacity) { + break; + } + } + + if (p->info.rows > 0) { + int32_t numOfCols = taosArrayGetSize(pColMatchInfo); + for (int32_t i = 0; i < numOfCols; ++i) { + SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); + ASSERT(pmInfo->matchType == COL_MATCH_FROM_SLOT_ID); + + SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); + SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); + } + + pDataBlock->info.rows = p->info.rows; + pDataBlock->info.capacity = p->info.rows; + } + + blockDataDestroy(p); + return (pDataBlock->info.rows > 0) ? pDataBlock : NULL; +} + +typedef struct SGroupSortSourceParam { + SOperatorInfo* childOpInfo; + SGroupSortOperatorInfo* grpSortOpInfo; +} SGroupSortSourceParam; + +SSDataBlock* fetchNextGroupSortDataBlock(void* param) { + SGroupSortSourceParam* source = param; + SGroupSortOperatorInfo* grpSortOpInfo = source->grpSortOpInfo; + if (grpSortOpInfo->prefetchedSortInput) { + SSDataBlock* block = grpSortOpInfo->prefetchedSortInput; + grpSortOpInfo->prefetchedSortInput = NULL; + return block; + } else { + SOperatorInfo* childOp = source->childOpInfo; + SSDataBlock* block = childOp->fpSet.getNextFn(childOp); + if (block != NULL) { + if (block->info.groupId == grpSortOpInfo->currGroupId) { + grpSortOpInfo->childOpStatus = CHILD_OP_SAME_GROUP; + return block; + } else { + grpSortOpInfo->childOpStatus = CHILD_OP_NEW_GROUP; + grpSortOpInfo->prefetchedSortInput = block; + return NULL; + } + } else { + grpSortOpInfo->childOpStatus = CHILD_OP_FINISHED; + return NULL; + } + } +} + +int32_t beginSortGroup(SOperatorInfo* pOperator) { + SGroupSortOperatorInfo* pInfo = pOperator->info; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + // pInfo->binfo.pRes is not equalled to the input datablock. + pInfo->pCurrSortHandle = + tsortCreateSortHandle(pInfo->pSortInfo, SORT_SINGLESOURCE_SORT, -1, -1, NULL, pTaskInfo->id.str); + + tsortSetFetchRawDataFp(pInfo->pCurrSortHandle, fetchNextGroupSortDataBlock, applyScalarFunction, pOperator); + + SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); + SGroupSortSourceParam* param = taosMemoryCalloc(1, sizeof(SGroupSortSourceParam)); + param->childOpInfo = pOperator->pDownstream[0]; + param->grpSortOpInfo = pInfo; + ps->param = param; + tsortAddSource(pInfo->pCurrSortHandle, ps); + + int32_t code = tsortOpen(pInfo->pCurrSortHandle); + taosMemoryFreeClear(ps); + + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, terrno); + } + + return TSDB_CODE_SUCCESS; +} + +int32_t finishSortGroup(SOperatorInfo* pOperator) { + SGroupSortOperatorInfo* pInfo = pOperator->info; + + SSortExecInfo sortExecInfo = tsortGetSortExecInfo(pInfo->pCurrSortHandle); + pInfo->sortExecInfo.sortMethod = sortExecInfo.sortMethod; + pInfo->sortExecInfo.sortBuffer = sortExecInfo.sortBuffer; + pInfo->sortExecInfo.loops += sortExecInfo.loops; + pInfo->sortExecInfo.readBytes += sortExecInfo.readBytes; + pInfo->sortExecInfo.writeBytes += sortExecInfo.writeBytes; + if (pInfo->pCurrSortHandle != NULL) { + tsortDestroySortHandle(pInfo->pCurrSortHandle); + } + pInfo->pCurrSortHandle = NULL; + return TSDB_CODE_SUCCESS; +} + +SSDataBlock* doGroupSort(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SGroupSortOperatorInfo* pInfo = pOperator->info; + + int32_t code = pOperator->fpSet._openFn(pOperator); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + if (!pInfo->hasGroupId) { + pInfo->hasGroupId = true; + + pInfo->prefetchedSortInput = pOperator->pDownstream[0]->fpSet.getNextFn(pOperator->pDownstream[0]); + pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; + pInfo->childOpStatus = CHILD_OP_NEW_GROUP; + beginSortGroup(pOperator); + } + + SSDataBlock* pBlock = NULL; + while (pInfo->pCurrSortHandle != NULL) { + // beginSortGroup would fetch all child blocks of pInfo->currGroupId; + ASSERT(pInfo->childOpStatus != CHILD_OP_SAME_GROUP); + pBlock = getGroupSortedBlockData(pInfo->pCurrSortHandle, pInfo->binfo.pRes, pOperator->resultInfo.capacity, + pInfo->pColMatchInfo, pInfo); + if (pBlock != NULL) { + pBlock->info.groupId = pInfo->currGroupId; + pOperator->resultInfo.totalRows += pBlock->info.rows; + return pBlock; + } else { + if (pInfo->childOpStatus == CHILD_OP_NEW_GROUP) { + finishSortGroup(pOperator); + pInfo->currGroupId = pInfo->prefetchedSortInput->info.groupId; + beginSortGroup(pOperator); + } else if (pInfo->childOpStatus == CHILD_OP_FINISHED) { + finishSortGroup(pOperator); + doSetOperatorCompleted(pOperator); + return NULL; + } + } + } + return NULL; +} + +int32_t getGroupSortExplainExecInfo(SOperatorInfo* pOptr, void** pOptrExplain, uint32_t* len) { + SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)pOptr->info; + *pOptrExplain = &pInfo->sortExecInfo; + *len = sizeof(SSortExecInfo); + return TSDB_CODE_SUCCESS; +} + +void destroyGroupSortOperatorInfo(void* param, int32_t numOfOutput) { + SGroupSortOperatorInfo* pInfo = (SGroupSortOperatorInfo*)param; + pInfo->binfo.pRes = blockDataDestroy(pInfo->binfo.pRes); + + taosArrayDestroy(pInfo->pSortInfo); + taosArrayDestroy(pInfo->pColMatchInfo); +} + +SOperatorInfo* createGroupSortOperatorInfo(SOperatorInfo* downstream, SGroupSortPhysiNode* pSortPhyNode, + SExecTaskInfo* pTaskInfo) { + SGroupSortOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SGroupSortOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (pInfo == NULL || pOperator == NULL /* || rowSize > 100 * 1024 * 1024*/) { + goto _error; + } + + SDataBlockDescNode* pDescNode = pSortPhyNode->node.pOutputDataBlockDesc; + + int32_t numOfCols = 0; + SSDataBlock* pResBlock = createResDataBlock(pDescNode); + SExprInfo* pExprInfo = createExprInfo(pSortPhyNode->pExprs, NULL, &numOfCols); + + int32_t numOfOutputCols = 0; + SArray* pColMatchColInfo = + extractColMatchInfo(pSortPhyNode->pTargets, pDescNode, &numOfOutputCols, COL_MATCH_FROM_SLOT_ID); + + pOperator->exprSupp.pCtx = createSqlFunctionCtx(pExprInfo, numOfCols, &pOperator->exprSupp.rowEntryInfoOffset); + pInfo->binfo.pRes = pResBlock; + + initResultSizeInfo(pOperator, 1024); + + pInfo->pSortInfo = createSortInfo(pSortPhyNode->pSortKeys); + ; + pInfo->pColMatchInfo = pColMatchColInfo; + pOperator->name = "GroupSortOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_GROUP_SORT; + pOperator->blocking = true; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->pTaskInfo = pTaskInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doGroupSort, NULL, NULL, destroyGroupSortOperatorInfo, NULL, + NULL, getGroupSortExplainExecInfo); + + int32_t code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + pTaskInfo->code = TSDB_CODE_OUT_OF_MEMORY; + taosMemoryFree(pInfo); + taosMemoryFree(pOperator); + return NULL; +} + +//===================================================================================== +// Multiway Sort Merge operator typedef struct SMultiwaySortMergeOperatorInfo { SOptrBasicInfo binfo; @@ -259,11 +516,12 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { int32_t numOfBufPage = pInfo->sortBufSize / pInfo->bufPageSize; - pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, - pInfo->bufPageSize, numOfBufPage, pInfo->pInputBlock, pTaskInfo->id.str); + pInfo->pSortHandle = tsortCreateSortHandle(pInfo->pSortInfo, SORT_MULTISOURCE_MERGE, pInfo->bufPageSize, numOfBufPage, + pInfo->pInputBlock, pTaskInfo->id.str); tsortSetFetchRawDataFp(pInfo->pSortHandle, loadNextDataBlock, NULL, NULL); - + tsortSetCompareGroupId(pInfo->pSortHandle, true); + for (int32_t i = 0; i < pOperator->numOfDownstream; ++i) { SSortSource* ps = taosMemoryCalloc(1, sizeof(SSortSource)); ps->param = pOperator->pDownstream[i]; @@ -286,7 +544,7 @@ int32_t doOpenMultiwaySortMergeOperator(SOperatorInfo* pOperator) { SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pDataBlock, int32_t capacity, SArray* pColMatchInfo, SOperatorInfo* pOperator) { SMultiwaySortMergeOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; blockDataCleanup(pDataBlock); @@ -309,7 +567,8 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData } } - if (p->info.rows > 0) { + if (p->info.rows > 0) {// todo extract method + blockDataEnsureCapacity(pDataBlock, p->info.rows); int32_t numOfCols = taosArrayGetSize(pColMatchInfo); for (int32_t i = 0; i < numOfCols; ++i) { SColMatchInfo* pmInfo = taosArrayGet(pColMatchInfo, i); @@ -317,11 +576,10 @@ SSDataBlock* getMultiwaySortedBlockData(SSortHandle* pHandle, SSDataBlock* pData SColumnInfoData* pSrc = taosArrayGet(p->pDataBlock, pmInfo->srcSlotId); SColumnInfoData* pDst = taosArrayGet(pDataBlock->pDataBlock, pmInfo->targetSlotId); - colDataAssign(pDst, pSrc, p->info.rows); + colDataAssign(pDst, pSrc, p->info.rows, &pDataBlock->info); } pDataBlock->info.rows = p->info.rows; - pDataBlock->info.capacity = p->info.rows; pDataBlock->info.groupId = pInfo->groupId; } @@ -387,24 +645,23 @@ SOperatorInfo* createMultiwaySortMergeOperatorInfo(SOperatorInfo** downStreams, goto _error; } - initResultSizeInfo(pOperator, 1024); - pInfo->binfo.pRes = pResBlock; - pInfo->pSortInfo = pSortInfo; + pInfo->binfo.pRes = pResBlock; + pInfo->pSortInfo = pSortInfo; pInfo->pColMatchInfo = pColMatchColInfo; - pInfo->pInputBlock = pInputBlock; - pOperator->name = "MultiwaySortMerge"; + pInfo->pInputBlock = pInputBlock; + pOperator->name = "MultiwaySortMerge"; pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE; - pOperator->blocking = false; - pOperator->status = OP_NOT_OPENED; - pOperator->info = pInfo; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; pOperator->pTaskInfo = pTaskInfo; - pInfo->bufPageSize = getProperSortPageSize(rowSize); + pInfo->bufPageSize = getProperSortPageSize(rowSize); // one additional is reserved for merged result. - pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); + pInfo->sortBufSize = pInfo->bufPageSize * (numStreams + 1); pOperator->fpSet = createOperatorFpSet(doOpenMultiwaySortMergeOperator, doMultiwaySortMerge, NULL, NULL, @@ -421,4 +678,4 @@ _error: taosMemoryFree(pInfo); taosMemoryFree(pOperator); return NULL; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index 110c2d6a8f896ee040d03402780564e44df3216b..1d82d6a6447b917436d4f68a4b474ca14d7eba4f 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -50,7 +50,7 @@ static void setTagsValue(SFillInfo* pFillInfo, void** data, int32_t genRows) { static void setNullRow(SSDataBlock* pBlock, int32_t numOfCol, int32_t rowIndex) { // the first are always the timestamp column, so start from the second column. - for (int32_t i = 1; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* p = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(p, rowIndex); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index c556e94c74b8f3795cca197fd26544579aceeaad..f365be906c322108f964aca1ec6ebcc794c82303 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -16,8 +16,8 @@ #include "function.h" #include "functionMgt.h" #include "tdatablock.h" -#include "ttime.h" #include "tfill.h" +#include "ttime.h" typedef enum SResultTsInterpType { RESULT_ROW_START_INTERP = 1, @@ -42,7 +42,7 @@ static void doCloseWindow(SResultRowInfo* pResultRowInfo, const SIntervalAggOper // * merged during merge stage. In this case, we need the pTableQueryInfo->lastResRows to decide if there // * is a previous result generated or not. // */ -//static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { +// static void setIntervalQueryRange(STableQueryInfo* pTableQueryInfo, TSKEY key, STimeWindow* pQRange) { // // do nothing //} @@ -314,18 +314,18 @@ static void getNextTimeWindow(SInterval* pInterval, int32_t precision, int32_t o int mon = (int)(tm.tm_year * 12 + tm.tm_mon + interval * factor); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + tw->skey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); mon = (int)(mon + interval); tm.tm_year = mon / 12; tm.tm_mon = mon % 12; - tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000L, TSDB_TIME_PRECISION_MILLI, precision); + tw->ekey = convertTimePrecision((int64_t)taosMktime(&tm) * 1000LL, TSDB_TIME_PRECISION_MILLI, precision); tw->ekey -= 1; } -void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, - TSKEY curTs, int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { +void doTimeWindowInterpolation(SArray* pPrevValues, SArray* pDataBlock, TSKEY prevTs, int32_t prevRowIndex, TSKEY curTs, + int32_t curRowIndex, TSKEY windowKey, int32_t type, SExprSupp* pSup) { SqlFunctionCtx* pCtx = pSup->pCtx; int32_t index = 1; @@ -405,8 +405,8 @@ static void setNotInterpoWindowKey(SqlFunctionCtx* pCtx, int32_t numOfOutput, in } } -static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock, const TSKEY* tsCols, - STimeWindow* win, SExprSupp* pSup) { +static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, int32_t pos, SSDataBlock* pBlock, + const TSKEY* tsCols, STimeWindow* win, SExprSupp* pSup) { bool ascQuery = (pInfo->order == TSDB_ORDER_ASC); TSKEY curTs = tsCols[pos]; @@ -434,9 +434,9 @@ static bool setTimeWindowInterpolationStartTs(SIntervalAggOperatorInfo* pInfo, i return true; } -static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, - int32_t endRowIndex, SArray* pDataBlock, const TSKEY* tsCols, - TSKEY blockEkey, STimeWindow* win) { +static bool setTimeWindowInterpolationEndTs(SIntervalAggOperatorInfo* pInfo, SExprSupp* pSup, int32_t endRowIndex, + SArray* pDataBlock, const TSKEY* tsCols, TSKEY blockEkey, + STimeWindow* win) { int32_t order = pInfo->order; TSKEY actualEndKey = tsCols[endRowIndex]; @@ -548,8 +548,8 @@ static void setResultRowInterpo(SResultRow* pResult, SResultTsInterpType type) { } } -static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult, STimeWindow* win, int32_t startPos, - int32_t forwardRows, SExprSupp* pSup) { +static void doWindowBorderInterpolation(SIntervalAggOperatorInfo* pInfo, SSDataBlock* pBlock, SResultRow* pResult, + STimeWindow* win, int32_t startPos, int32_t forwardRows, SExprSupp* pSup) { if (!pInfo->timeWindowInterpo) { return; } @@ -628,7 +628,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; - SExprSupp* pSup = &pOperatorInfo->exprSupp; + SExprSupp* pSup = &pOperatorInfo->exprSupp; int32_t startPos = 0; int32_t numOfOutput = pSup->numOfExprs; @@ -655,9 +655,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } STimeWindow w = pr->win; - int32_t ret = - setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &w, (scanFlag == MAIN_SCAN), &pResult, groupId, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -672,8 +671,8 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num setResultRowInterpo(pResult, RESULT_ROW_END_INTERP); setNotInterpoWindowKey(pSup->pCtx, numOfExprs, RESULT_ROW_START_INTERP); - doApplyFunctions(pTaskInfo, pSup->pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols, - pBlock->info.rows, numOfExprs, pInfo->order); + doApplyFunctions(pTaskInfo, pSup->pCtx, &w, &pInfo->twAggSup.timeWindowData, startPos, 0, tsCols, pBlock->info.rows, + numOfExprs, pInfo->order); if (isResultRowInterpolated(pResult, RESULT_ROW_END_INTERP)) { closeResultRow(pr); @@ -685,6 +684,7 @@ static void doInterpUnclosedTimeWindow(SOperatorInfo* pOperatorInfo, int32_t num } void printDataBlock(SSDataBlock* pBlock, const char* flag) { + if (pBlock == NULL) return; SArray* blocks = taosArrayInit(1, sizeof(SSDataBlock)); taosArrayPush(blocks, pBlock); blockDebugShowData(blocks, flag); @@ -753,8 +753,7 @@ int64_t getReskey(void* data, int32_t index) { return *(int64_t*)pos->key; } -static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, - SArray* pUpdated) { +static int32_t saveResult(int64_t ts, int32_t pageId, int32_t offset, uint64_t groupId, SArray* pUpdated) { int32_t size = taosArrayGetSize(pUpdated); int32_t index = binarySearch(pUpdated, size, ts, TSDB_ORDER_DESC, getReskey); if (index == -1) { @@ -806,7 +805,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul SIntervalAggOperatorInfo* pInfo = (SIntervalAggOperatorInfo*)pOperatorInfo->info; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; + SExprSupp* pSup = &pOperatorInfo->exprSupp; int32_t startPos = 0; int32_t numOfOutput = pSup->numOfExprs; @@ -819,9 +818,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul STimeWindow win = getActiveTimeWindow(pInfo->aggSup.pResultBuf, pResultRowInfo, ts, &pInfo->interval, pInfo->interval.precision, &pInfo->win); - int32_t ret = - setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -843,9 +841,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); // restore current time window - ret = - setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -870,8 +867,7 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul // null data, failed to allocate more memory buffer int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, - pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, - &pInfo->aggSup, pTaskInfo); + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -890,8 +886,8 @@ static void hashIntervalAgg(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResul doWindowBorderInterpolation(pInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pSup); updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - tsCols, pBlock->info.rows, numOfOutput, pInfo->order); + doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, pInfo->order); doCloseWindow(pResultRowInfo, pInfo, pResult); } @@ -1002,7 +998,7 @@ static bool compareVal(const char* v, const SStateKeys* pKey) { static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; SColumnInfoData* pStateColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->stateCol.slotId); int64_t gid = pBlock->info.groupId; @@ -1050,9 +1046,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI STimeWindow window = pRowSup->win; pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1076,9 +1071,8 @@ static void doStateWindowAggImpl(SOperatorInfo* pOperator, SStateWindowOperatorI SResultRow* pResult = NULL; pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1095,8 +1089,8 @@ static SSDataBlock* doStateWindowAgg(SOperatorInfo* pOperator) { SStateWindowOperatorInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SExprSupp* pSup = &pOperator->exprSupp; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + SExprSupp* pSup = &pOperator->exprSupp; SOptrBasicInfo* pBInfo = &pInfo->binfo; @@ -1207,7 +1201,7 @@ static void setInverFunction(SqlFunctionCtx* pCtx, int32_t num, EStreamType type } } -void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp *pSup, int32_t numOfOutput) { +void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprSupp* pSup, int32_t numOfOutput) { SResultRow* pResult = getResultRowByPos(pResultBuf, p1); SqlFunctionCtx* pCtx = pSup->pCtx; for (int32_t i = 0; i < numOfOutput; ++i) { @@ -1223,7 +1217,7 @@ void doClearWindowImpl(SResultRowPosition* p1, SDiskbasedBuf* pResultBuf, SExprS } } -void doClearWindow(SAggSupporter* pAggSup, SExprSupp *pSup, char* pData, int16_t bytes, uint64_t groupId, +void doClearWindow(SAggSupporter* pAggSup, SExprSupp* pSup, char* pData, int16_t bytes, uint64_t groupId, int32_t numOfOutput) { SET_RES_WINDOW_KEY(pAggSup->keyBuf, pData, bytes, groupId); SResultRowPosition* p1 = @@ -1259,9 +1253,9 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { void* key = taosHashGetKey(pIte, &keyLen); uint64_t groupId = *(uint64_t*)key; ASSERT(keyLen == GET_RES_WINDOW_KEY_LEN(sizeof(TSKEY))); - TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); + TSKEY ts = *(int64_t*)((char*)key + sizeof(uint64_t)); SResultRowPosition* pPos = (SResultRowPosition*)pIte; - int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins); + int32_t code = saveResult(ts, pPos->pageId, pPos->offset, groupId, resWins); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1269,8 +1263,9 @@ static int32_t getAllIntervalWindow(SHashObj* pHashMap, SArray* resWins) { return TSDB_CODE_SUCCESS; } -bool isCloseWindow(STimeWindow *pWin, STimeWindowAggSupp* pSup) { - return pWin->ekey < pSup->maxTs - pSup->waterMark; +bool isCloseWindow(STimeWindow* pWin, STimeWindowAggSupp* pSup) { + ASSERT(pSup->maxTs == INT64_MIN || pSup->maxTs > 0); + return pSup->maxTs != INT64_MIN && pWin->ekey < pSup->maxTs - pSup->waterMark; } static int32_t closeIntervalWindow(SHashObj* pHashMap, STimeWindowAggSupp* pSup, SInterval* pInterval, @@ -1330,7 +1325,8 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { } if (pBlock->info.type == STREAM_REPROCESS) { - doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, NULL); + doClearWindows(&pInfo->aggSup, &pOperator->exprSupp, &pInfo->interval, 0, pOperator->exprSupp.numOfExprs, pBlock, + NULL); qDebug("%s clear existed time window results for updates checked", GET_TASKID(pTaskInfo)); continue; } else if (pBlock->info.type == STREAM_GET_ALL) { @@ -1580,7 +1576,7 @@ _error: // todo handle multiple tables cases. static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperatorInfo* pInfo, SSDataBlock* pBlock) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, pInfo->tsSlotId); @@ -1617,9 +1613,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator STimeWindow window = pRowSup->win; pRowSup->win.ekey = pRowSup->win.skey; - int32_t ret = - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &window, masterScan, &pResult, gid, pSup->pCtx, + numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1637,9 +1632,8 @@ static void doSessionWindowAggImpl(SOperatorInfo* pOperator, SSessionAggOperator SResultRow* pResult = NULL; pRowSup->win.ekey = tsList[pBlock->info.rows - 1]; - int32_t ret = - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + int32_t ret = setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, &pRowSup->win, masterScan, &pResult, gid, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS) { // null data, too many state code longjmp(pTaskInfo->env, TSDB_CODE_QRY_APP_ERROR); } @@ -1656,16 +1650,15 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { SSessionAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; if (pOperator->status == OP_RES_TO_RETURN) { doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->aggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); - return NULL; } - return pBInfo->pRes; + return pBInfo->pRes->info.rows > 0 ? pBInfo->pRes : NULL; } int64_t st = taosGetTimestampUs(); @@ -1705,118 +1698,63 @@ static SSDataBlock* doSessionWindowAgg(SOperatorInfo* pOperator) { return (rows == 0) ? NULL : pBInfo->pRes; } -static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock) { +static void doKeepPrevRows(STimeSliceOperatorInfo* pSliceInfo, const SSDataBlock* pBlock, int32_t rowIndex) { int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); - for(int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, i); // null data should not be kept since it can not be used to perform interpolation if (!colDataIsNull_s(pColInfoData, i)) { SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, i); pkey->isNull = false; - char* val = colDataGetData(pColInfoData, i); + char* val = colDataGetData(pColInfoData, rowIndex); memcpy(pkey->pData, val, pkey->bytes); } } } -static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { - if (pOperator->status == OP_EXEC_DONE) { - return NULL; - } - - STimeSliceOperatorInfo* pSliceInfo = pOperator->info; - SSDataBlock* pResBlock = pSliceInfo->binfo.pRes; - SExprSupp* pSup = &pOperator->exprSupp; - -// if (pOperator->status == OP_RES_TO_RETURN) { -// // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); -// if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) { -// doSetOperatorCompleted(pOperator); -// } -// -// return pResBlock; -// } +static void genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pBlock, + int32_t rowIndex, SSDataBlock* pResBlock) { + int32_t rows = pResBlock->info.rows; - int32_t order = TSDB_ORDER_ASC; - SInterval* pInterval = &pSliceInfo->interval; - SOperatorInfo* downstream = pOperator->pDownstream[0]; - - int32_t numOfRows = 0; - while (1) { - SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); - if (pBlock == NULL) { - break; - } + // todo set the correct primary timestamp column - // the pDataBlock are always the same one, no need to call this again - setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true); + // output the result + for (int32_t j = 0; j < pExprSup->numOfExprs; ++j) { + SExprInfo* pExprInfo = &pExprSup->pExprInfo[j]; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, 0); - for(int32_t i = 0; i < pBlock->info.rows; ++i) { - int64_t ts = *(int64_t*) colDataGetData(pTsCol, i); - - if (ts == pSliceInfo->current) { - for(int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot); + switch (pSliceInfo->fillType) { + case TSDB_FILL_NULL: + colDataAppendNULL(pDst, rows); + break; - char* v = colDataGetData(pSrc, i); - colDataAppend(pDst, numOfRows, v, false); + case TSDB_FILL_SET_VALUE: { + SVariant* pVar = &pSliceInfo->pFillColInfo[j].fillVal; + + if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { + float v = 0; + GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); + colDataAppend(pDst, rows, (char*)&v, false); + } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { + double v = 0; + GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); + colDataAppend(pDst, rows, (char*)&v, false); + } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { + int64_t v = 0; + GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); + colDataAppend(pDst, rows, (char*)&v, false); } + } break; - numOfRows += 1; - - pSliceInfo->current += taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); - break; - } - } else if (ts < pSliceInfo->current) { - if (i != pBlock->info.window.ekey) { - int64_t nextTs = *(int64_t*) colDataGetData(pTsCol, i + 1); - if (nextTs > pSliceInfo->current) { - // output the result - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; - int32_t dstSlot = pExprInfo->base.resSchema.slotId; - int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; - - SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); - SColumnInfoData* pDst = taosArrayGet(pBlock->pDataBlock, dstSlot); - - switch (pSliceInfo->fillType) { - case TSDB_FILL_NULL: - colDataAppendNULL(pDst, numOfRows); - break; - - case TSDB_FILL_SET_VALUE: { - SVariant* pVar = &pSliceInfo->pFillColInfo[i].fillVal; - - if (pDst->info.type == TSDB_DATA_TYPE_FLOAT) { - float v = 0; - GET_TYPED_DATA(v, float, pVar->nType, &pVar->i); - colDataAppend(pDst, numOfRows, (char*)&v, false); - } else if (pDst->info.type == TSDB_DATA_TYPE_DOUBLE) { - double v = 0; - GET_TYPED_DATA(v, double, pVar->nType, &pVar->i); - colDataAppend(pDst, numOfRows, (char*)&v, false); - } else if (IS_SIGNED_NUMERIC_TYPE(pDst->info.type)) { - int64_t v = 0; - GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); - colDataAppend(pDst, numOfRows, (char*)&v, false); - } - } - break; - - case TSDB_FILL_LINEAR: + case TSDB_FILL_LINEAR: #if 0 - if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs + if (pCtx->start.key == INT64_MIN || pCtx->start.key > pCtx->startTs || pCtx->end.key == INT64_MIN || pCtx->end.key < pCtx->startTs) { // goto interp_exit; } @@ -1845,33 +1783,159 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { } } #endif - break; + break; - case TSDB_FILL_PREV: { - SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); - colDataAppend(pDst, numOfRows, pkey->pData, false); - } break; + case TSDB_FILL_PREV: { + SGroupKeys* pkey = taosArrayGet(pSliceInfo->pPrevRow, srcSlot); + colDataAppend(pDst, rows, pkey->pData, false); + } break; - case TSDB_FILL_NEXT: { - } break; + case TSDB_FILL_NEXT: { + char* p = colDataGetData(pSrc, rowIndex); + colDataAppend(pDst, rows, p, colDataIsNull_s(pSrc, rowIndex)); + } break; - case TSDB_FILL_NONE: - default: - break; - } + case TSDB_FILL_NONE: + default: + break; + } + } + + pResBlock->info.rows += 1; +} + +static int32_t initPrevRowsKeeper(STimeSliceOperatorInfo* pInfo, SSDataBlock* pBlock) { + if (pInfo->pPrevRow != NULL) { + return TSDB_CODE_SUCCESS; + } + + pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys)); + if (pInfo->pPrevRow == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + for (int32_t i = 0; i < numOfCols; ++i) { + SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); + + SGroupKeys key = {0}; + key.bytes = pColInfo->info.bytes; + key.type = pColInfo->info.type; + key.isNull = false; + key.pData = taosMemoryCalloc(1, pColInfo->info.bytes); + taosArrayPush(pInfo->pPrevRow, &key); + } + + return TSDB_CODE_SUCCESS; +} - pSliceInfo->current += +static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + STimeSliceOperatorInfo* pSliceInfo = pOperator->info; + SSDataBlock* pResBlock = pSliceInfo->pRes; + SExprSupp* pSup = &pOperator->exprSupp; + + blockDataEnsureCapacity(pResBlock, pOperator->resultInfo.capacity); + + // if (pOperator->status == OP_RES_TO_RETURN) { + // // doBuildResultDatablock(&pRuntimeEnv->groupResInfo, pRuntimeEnv, pIntervalInfo->pRes); + // if (pResBlock->info.rows == 0 || !hasDataInGroupInfo(&pSliceInfo->groupResInfo)) { + // doSetOperatorCompleted(pOperator); + // } + // + // return pResBlock; + // } + + int32_t order = TSDB_ORDER_ASC; + SInterval* pInterval = &pSliceInfo->interval; + SOperatorInfo* downstream = pOperator->pDownstream[0]; + + int32_t numOfRows = 0; + while (1) { + SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); + if (pBlock == NULL) { + break; + } + + int32_t code = initPrevRowsKeeper(pSliceInfo, pBlock); + if (code != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, code); + } + + // the pDataBlock are always the same one, no need to call this again + setInputDataBlock(pOperator, pSup->pCtx, pBlock, order, MAIN_SCAN, true); + + SColumnInfoData* pTsCol = taosArrayGet(pBlock->pDataBlock, pSliceInfo->tsCol.slotId); + for (int32_t i = 0; i < pBlock->info.rows; ++i) { + int64_t ts = *(int64_t*)colDataGetData(pTsCol, i); + + if (ts == pSliceInfo->current) { + for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { + SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[j]; + int32_t dstSlot = pExprInfo->base.resSchema.slotId; + int32_t srcSlot = pExprInfo->base.pParam[0].pCol->slotId; + + SColumnInfoData* pSrc = taosArrayGet(pBlock->pDataBlock, srcSlot); + SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); + + char* v = colDataGetData(pSrc, i); + colDataAppend(pDst, numOfRows, v, false); + } + + pResBlock->info.rows += 1; + doKeepPrevRows(pSliceInfo, pBlock, i); + + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } + + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } else if (ts < pSliceInfo->current) { + if (i < pBlock->info.rows - 1) { + int64_t nextTs = *(int64_t*)colDataGetData(pTsCol, i + 1); + if (nextTs > pSliceInfo->current) { + while (pSliceInfo->current < nextTs && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + pSliceInfo->current = taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); - if (pSliceInfo->current > pSliceInfo->win.ekey) { - doSetOperatorCompleted(pOperator); + if (pResBlock->info.rows >= pResBlock->info.capacity) { break; } } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; + } } else { // ignore current row, and do nothing } } else { // it is the last row of current block - doKeepPrevRows(pSliceInfo, pBlock); + doKeepPrevRows(pSliceInfo, pBlock, i); + } + } else { // ts > pSliceInfo->current + while (pSliceInfo->current < ts && pSliceInfo->current <= pSliceInfo->win.ekey) { + genInterpolationResult(pSliceInfo, &pOperator->exprSupp, pBlock, i, pResBlock); + pSliceInfo->current = + taosTimeAdd(pSliceInfo->current, pInterval->interval, pInterval->intervalUnit, pInterval->precision); + if (pResBlock->info.rows >= pResBlock->info.capacity) { + break; + } + } + + if (pSliceInfo->current > pSliceInfo->win.ekey) { + doSetOperatorCompleted(pOperator); + break; } } } @@ -1886,64 +1950,51 @@ static SSDataBlock* doTimeslice(SOperatorInfo* pOperator) { return pResBlock->info.rows == 0 ? NULL : pResBlock; } -static int32_t initTimesliceInfo(STimeSliceOperatorInfo* pInfo, SqlFunctionCtx* pCtx, int32_t numOfCols) { - pInfo->pPrevRow = taosArrayInit(4, sizeof(SGroupKeys)); - pInfo->pCols = taosArrayInit(4, sizeof(SColumn)); - - if (pInfo->pPrevRow == NULL || pInfo->pCols == NULL) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - for (int32_t i = 0; i < numOfCols; ++i) { - SExprInfo* pExpr = pCtx[i].pExpr; - - SFunctParam* pParam = &pExpr->base.pParam[0]; - - SColumn c = *pParam->pCol; - taosArrayPush(pInfo->pCols, &c); - - SGroupKeys key = {0}; - key.bytes = c.bytes; - key.type = c.type; - key.isNull = false; - key.pData = taosMemoryCalloc(1, c.bytes); - taosArrayPush(pInfo->pPrevRow, &key); - } - - return TSDB_CODE_SUCCESS; -} - -SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, int32_t numOfCols, - SSDataBlock* pResultBlock, const SNodeListNode* pValNode, SExecTaskInfo* pTaskInfo) { +SOperatorInfo* createTimeSliceOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { STimeSliceOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(STimeSliceOperatorInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pOperator == NULL || pInfo == NULL) { goto _error; } - SExprSupp* pSup = &pOperator->exprSupp; + SInterpFuncPhysiNode* pInterpPhyNode = (SInterpFuncPhysiNode*)pPhyNode; + SExprSupp* pSup = &pOperator->exprSupp; - int32_t code = initTimesliceInfo(pInfo, pSup->pCtx, numOfCols); + int32_t numOfExprs = 0; + SExprInfo* pExprInfo = createExprInfo(pInterpPhyNode->pFuncs, NULL, &numOfExprs); + int32_t code = initExprSupp(pSup, pExprInfo, numOfExprs); if (code != TSDB_CODE_SUCCESS) { goto _error; } - initResultRowInfo(&pInfo->binfo.resultRowInfo); - pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfCols, pValNode); + if (pInterpPhyNode->pExprs != NULL) { + int32_t num = 0; + SExprInfo* pScalarExprInfo = createExprInfo(pInterpPhyNode->pExprs, NULL, &num); + code = initExprSupp(&pInfo->scalarSup, pScalarExprInfo, num); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + } - pInfo->binfo.pRes = pResultBlock; + pInfo->tsCol = extractColumnFromColumnNode((SColumnNode*)pInterpPhyNode->pTimeSeries); + pInfo->fillType = convertFillType(pInterpPhyNode->fillMode); + initResultSizeInfo(pOperator, 4096); - pOperator->name = "TimeSliceOperator"; - // pOperator->operatorType = OP_AllTimeWindow; - pOperator->blocking = true; - pOperator->status = OP_NOT_OPENED; - pOperator->exprSupp.pExprInfo = pExprInfo; - pOperator->exprSupp.numOfExprs = numOfCols; - pOperator->info = pInfo; - pOperator->pTaskInfo = pTaskInfo; + pInfo->pFillColInfo = createFillColInfo(pExprInfo, numOfExprs, (SNodeListNode*)pInterpPhyNode->pFillValues); + pInfo->pRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + pInfo->win = pInterpPhyNode->timeRange; + pInfo->interval.interval = pInterpPhyNode->interval; + pInfo->current = pInfo->win.skey; + + pOperator->name = "TimeSliceOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_INTERP_FUNC; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->info = pInfo; + pOperator->pTaskInfo = pTaskInfo; - pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo, - NULL, NULL, NULL); + pOperator->fpSet = + createOperatorFpSet(operatorDummyOpenFn, doTimeslice, NULL, NULL, destroyBasicOperatorInfo, NULL, NULL, NULL); code = appendDownstream(pOperator, &downstream, 1); return pOperator; @@ -2081,22 +2132,24 @@ void compactFunctions(SqlFunctionCtx* pDestCtx, SqlFunctionCtx* pSourceCtx, int3 } } -static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, int32_t groupId, - int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { +static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExprSupp* pSup, SArray* pWinArray, + int32_t groupId, int32_t numOfOutput, SExecTaskInfo* pTaskInfo) { int32_t size = taosArrayGetSize(pWinArray); - ASSERT(pInfo->pChildren); + if (!pInfo->pChildren) { + return; + } for (int32_t i = 0; i < size; i++) { STimeWindow* pParentWin = taosArrayGet(pWinArray, i); SResultRow* pCurResult = NULL; - setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); + setTimeWindowOutputBuf(&pInfo->binfo.resultRowInfo, pParentWin, true, &pCurResult, 0, pSup->pCtx, numOfOutput, + pSup->rowEntryInfoOffset, &pInfo->aggSup, pTaskInfo); int32_t numOfChildren = taosArrayGetSize(pInfo->pChildren); for (int32_t j = 0; j < numOfChildren; j++) { SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, j); SIntervalAggOperatorInfo* pChInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; + SExprSupp* pChildSup = &pChildOp->exprSupp; - SResultRow* pChResult = NULL; + SResultRow* pChResult = NULL; setTimeWindowOutputBuf(&pChInfo->binfo.resultRowInfo, pParentWin, true, &pChResult, 0, pChildSup->pCtx, pChildSup->numOfExprs, pChildSup->rowEntryInfoOffset, &pChInfo->aggSup, pTaskInfo); compactFunctions(pSup->pCtx, pChildSup->pCtx, numOfOutput, pTaskInfo); @@ -2106,8 +2159,8 @@ static void rebuildIntervalWindow(SStreamFinalIntervalOperatorInfo* pInfo, SExpr bool isDeletedWindow(STimeWindow* pWin, uint64_t groupId, SAggSupporter* pSup) { SET_RES_WINDOW_KEY(pSup->keyBuf, &pWin->skey, sizeof(int64_t), groupId); - SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, - pSup->keyBuf, GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(pSup->pResultRowHashTable, pSup->keyBuf, + GET_RES_WINDOW_KEY_LEN(sizeof(int64_t))); return p1 == NULL; } @@ -2140,7 +2193,8 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc isDeletedWindow(&nextWin, tableGroupId, &pInfo->aggSup)) { SArray* pUpWins = taosArrayInit(8, sizeof(STimeWindow)); taosArrayPush(pUpWins, &nextWin); - rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs, pOperatorInfo->pTaskInfo); + rebuildIntervalWindow(pInfo, pSup, pUpWins, pInfo->binfo.pRes->info.groupId, pSup->numOfExprs, + pOperatorInfo->pTaskInfo); taosArrayDestroy(pUpWins); } int32_t code = setTimeWindowOutputBuf(pResultRowInfo, &nextWin, true, &pResult, tableGroupId, pSup->pCtx, @@ -2158,8 +2212,8 @@ static void doHashInterval(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBloc saveResultRow(pResult, tableGroupId, pUpdated); } updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, - tsCols, pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); + doApplyFunctions(pTaskInfo, pSup->pCtx, &nextWin, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pSDataBlock->info.rows, numOfOutput, TSDB_ORDER_ASC); int32_t prevEndPos = (forwardRows - 1) * step + startPos; ASSERT(pSDataBlock->info.window.skey > 0 && pSDataBlock->info.window.ekey > 0); startPos = getNextQualifiedWindow(&pInfo->interval, &nextWin, &pSDataBlock->info, tsCols, prevEndPos, pInfo->order); @@ -2177,32 +2231,30 @@ static void clearStreamIntervalOperator(SStreamFinalIntervalOperatorInfo* pInfo) } static void clearUpdateDataBlock(SSDataBlock* pBlock) { - if (pBlock->info.rows <= 0) { - return; - } blockDataCleanup(pBlock); } void copyUpdateDataBlock(SSDataBlock* pDest, SSDataBlock* pSource, int32_t tsColIndex) { - ASSERT(pDest->info.capacity >= pSource->info.rows); + // ASSERT(pDest->info.capacity >= pSource->info.rows); + blockDataEnsureCapacity(pDest, pSource->info.rows); clearUpdateDataBlock(pDest); SColumnInfoData* pDestCol = taosArrayGet(pDest->pDataBlock, 0); SColumnInfoData* pSourceCol = taosArrayGet(pSource->pDataBlock, tsColIndex); + // copy timestamp column - colDataAssign(pDestCol, pSourceCol, pSource->info.rows); - for (int32_t i = 1; i < pDest->info.numOfCols; i++) { + colDataAssign(pDestCol, pSourceCol, pSource->info.rows, &pDest->info); + for (int32_t i = 1; i < taosArrayGetSize(pDest->pDataBlock); i++) { SColumnInfoData* pCol = taosArrayGet(pDest->pDataBlock, i); colDataAppendNNULL(pCol, 0, pSource->info.rows); } + pDest->info.rows = pSource->info.rows; pDest->info.groupId = pSource->info.groupId; pDest->info.type = pSource->info.type; blockDataUpdateTsWindow(pDest, 0); } -static int32_t getChildIndex(SSDataBlock* pBlock) { - return pBlock->info.childId; -} +static int32_t getChildIndex(SSDataBlock* pBlock) { return pBlock->info.childId; } static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pInfo = pOperator->info; @@ -2247,7 +2299,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); SIntervalAggOperatorInfo* pChildInfo = pChildOp->info; - SExprSupp* pChildSup = &pChildOp->exprSupp; + SExprSupp* pChildSup = &pChildOp->exprSupp; doClearWindows(&pChildInfo->aggSup, pChildSup, &pChildInfo->interval, pChildInfo->primaryTsIndex, pChildSup->numOfExprs, pBlock, NULL); @@ -2282,6 +2334,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { SStreamFinalIntervalOperatorInfo* pChInfo = pChildOp->info; setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, pChInfo->order, MAIN_SCAN, true); doHashInterval(pChildOp, pBlock, pBlock->info.groupId, NULL); + pChInfo->twAggSup.maxTs = TMAX(pChInfo->twAggSup.maxTs, pBlock->info.window.ekey); } maxTs = TMAX(maxTs, pBlock->info.window.ekey); } @@ -2349,7 +2402,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, initResultRowInfo(&pInfo->binfo.resultRowInfo); pInfo->pChildren = NULL; if (numOfChild > 0) { - pInfo->pChildren = taosArrayInit(numOfChild, sizeof(SOperatorInfo)); + pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { SOperatorInfo* pChildOp = createStreamFinalIntervalOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); if (pChildOp) { @@ -2360,9 +2413,6 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, } } // semi interval operator does not catch result - if (!IS_FINAL_OP(pInfo)) { - pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; - } pInfo->pUpdateRes = createResDataBlock(pPhyNode->pOutputDataBlockDesc); pInfo->pUpdateRes->info.type = STREAM_REPROCESS; blockDataEnsureCapacity(pInfo->pUpdateRes, 128); @@ -2376,6 +2426,10 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pOperator->name = "StreamSemiIntervalOperator"; } + if (!IS_FINAL_OP(pInfo)) { + pInfo->twAggSup.calTrigger = STREAM_TRIGGER_AT_ONCE; + } + pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; pOperator->status = OP_NOT_OPENED; @@ -2405,9 +2459,9 @@ _error: void destroyStreamAggSupporter(SStreamAggSupporter* pSup) { taosMemoryFreeClear(pSup->pKeyBuf); - void **pIte = NULL; + void** pIte = NULL; while ((pIte = taosHashIterate(pSup->pResultRows, pIte)) != NULL) { - SArray *pWins = (SArray *) (*pIte); + SArray* pWins = (SArray*)(*pIte); taosArrayDestroy(pWins); } taosHashCleanup(pSup->pResultRows); @@ -2431,14 +2485,19 @@ void destroyStreamSessionAggOperatorInfo(void* param, int32_t numOfOutput) { } } -int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, SSDataBlock* pResultBlock) { - initExprSupp(pSup, pExprInfo, numOfCols); +int32_t initBasicInfoEx(SOptrBasicInfo* pBasicInfo, SExprSupp* pSup, SExprInfo* pExprInfo, int32_t numOfCols, + SSDataBlock* pResultBlock) { + int32_t code = initExprSupp(pSup, pExprInfo, numOfCols); + if (code != TSDB_CODE_SUCCESS) { + return code; + } + initBasicInfo(pBasicInfo, pResultBlock); - + for (int32_t i = 0; i < numOfCols; ++i) { pSup->pCtx[i].pBuf = NULL; } - + ASSERT(numOfCols > 0); increaseTs(pSup->pCtx); return TSDB_CODE_SUCCESS; @@ -2457,18 +2516,20 @@ void initDownStream(SOperatorInfo* downstream, SStreamAggSupporter* pAggSup, int pScanInfo->pUpdateInfo = updateInfoInit(60000, TSDB_TIME_PRECISION_MILLI, waterMark); } -int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, int32_t numOfOutput) { +int32_t initSessionAggSupporter(SStreamAggSupporter* pSup, const char* pKey, SqlFunctionCtx* pCtx, + int32_t numOfOutput) { return initStreamAggSupporter(pSup, pKey, pCtx, numOfOutput, sizeof(SResultWindowInfo)); } -SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo) { - SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; - int32_t numOfCols = 0; - SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); - SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); - int32_t code = TSDB_CODE_OUT_OF_MEMORY; +SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo) { + SSessionWinodwPhysiNode* pSessionNode = (SSessionWinodwPhysiNode*)pPhyNode; + int32_t numOfCols = 0; + SExprInfo* pExprInfo = createExprInfo(pSessionNode->window.pFuncs, NULL, &numOfCols); + SSDataBlock* pResBlock = createResDataBlock(pPhyNode->pOutputDataBlockDesc); + int32_t code = TSDB_CODE_OUT_OF_MEMORY; SStreamSessionAggOperatorInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamSessionAggOperatorInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { goto _error; } @@ -2480,8 +2541,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh if (code != TSDB_CODE_SUCCESS) { goto _error; } - - + code = initSessionAggSupporter(&pInfo->streamAggSup, "StreamSessionAggOperatorInfo", pSup->pCtx, numOfCols); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -2493,10 +2553,8 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh } initDummyFunction(pInfo->pDummyCtx, pSup->pCtx, numOfCols); - pInfo->twAggSup = (STimeWindowAggSupp) { - .waterMark = pSessionNode->window.watermark, - .calTrigger = pSessionNode->window.triggerType, - .maxTs = INT64_MIN}; + pInfo->twAggSup = (STimeWindowAggSupp){ + .waterMark = pSessionNode->window.watermark, .calTrigger = pSessionNode->window.triggerType, .maxTs = INT64_MIN}; initResultRowInfo(&pInfo->binfo.resultRowInfo); initExecTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &pTaskInfo->window); @@ -2512,6 +2570,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->pStDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; pInfo->pDelRes = createOneDataBlock(pResBlock, false); + pInfo->pDelRes->info.type = STREAM_DELETE; blockDataEnsureCapacity(pInfo->pDelRes, 64); pInfo->pChildren = NULL; pInfo->isFinal = false; @@ -2570,19 +2629,19 @@ static SResultWindowInfo* addNewSessionWindow(SArray* pWinInfos, TSKEY ts) { } SArray* getWinInfos(SStreamAggSupporter* pAggSup, uint64_t groupId) { - void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t)); + void** ite = taosHashGet(pAggSup->pResultRows, &groupId, sizeof(uint64_t)); SArray* pWinInfos = NULL; if (ite == NULL) { pWinInfos = taosArrayInit(1024, pAggSup->valueSize); - taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void *)); + taosHashPut(pAggSup->pResultRows, &groupId, sizeof(uint64_t), &pWinInfos, sizeof(void*)); } else { pWinInfos = *ite; } return pWinInfos; } -SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, - TSKEY endTs, uint64_t groupId, int64_t gap, int32_t* pIndex) { +SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY startTs, TSKEY endTs, uint64_t groupId, + int64_t gap, int32_t* pIndex) { SArray* pWinInfos = getWinInfos(pAggSup, groupId); pAggSup->pCurWins = pWinInfos; @@ -2621,10 +2680,10 @@ SResultWindowInfo* getSessionTimeWindow(SStreamAggSupporter* pAggSup, TSKEY star return insertNewSessionWindow(pWinInfos, startTs, index + 1); } -int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, - TSKEY* pEndTs, int32_t rows, int32_t start, int64_t gap, SHashObj* pStDeleted) { +int32_t updateSessionWindowInfo(SResultWindowInfo* pWinInfo, TSKEY* pStartTs, TSKEY* pEndTs, int32_t rows, + int32_t start, int64_t gap, SHashObj* pStDeleted) { for (int32_t i = start; i < rows; ++i) { - if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap)) ) { + if (!isInWindow(pWinInfo, pStartTs[i], gap) && (!pEndTs || !isInWindow(pWinInfo, pEndTs[i], gap))) { return i - start; } if (pWinInfo->win.skey > pStartTs[i]) { @@ -2680,7 +2739,7 @@ static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStre SColumnInfoData* pTimeWindowData, SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, tsColId); @@ -2698,14 +2757,14 @@ static int32_t doOneWindowAggImpl(int32_t tsColId, SOptrBasicInfo* pBinfo, SStre static int32_t doOneWindowAgg(SStreamSessionAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, int32_t winRows, - int32_t numOutput, SOperatorInfo * pOperator) { + int32_t numOutput, SOperatorInfo* pOperator) { return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator); } static int32_t doOneStateWindowAgg(SStreamStateAggOperatorInfo* pInfo, SSDataBlock* pSDataBlock, SResultWindowInfo* pCurWin, SResultRow** pResult, int32_t startIndex, - int32_t winRows, int32_t numOutput, SOperatorInfo * pOperator) { + int32_t winRows, int32_t numOutput, SOperatorInfo* pOperator) { return doOneWindowAggImpl(pInfo->primaryTsIndex, &pInfo->binfo, &pInfo->streamAggSup, &pInfo->twAggSup.timeWindowData, pSDataBlock, pCurWin, pResult, startIndex, winRows, numOutput, pOperator); } @@ -2726,7 +2785,7 @@ int32_t getNumCompactWindow(SArray* pWinInfos, int32_t startIndex, int64_t gap) void compactTimeWindow(SStreamSessionAggOperatorInfo* pInfo, int32_t startIndex, int32_t num, uint64_t groupId, int32_t numOfOutput, SHashObj* pStUpdated, SHashObj* pStDeleted, SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SResultWindowInfo* pCurWin = taosArrayGet(pInfo->streamAggSup.pCurWins, startIndex); @@ -2757,8 +2816,8 @@ typedef struct SWinRes { uint64_t groupId; } SWinRes; -static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, - SHashObj* pStUpdated, SHashObj* pStDeleted, bool hasEndTs) { +static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBlock, SHashObj* pStUpdated, + SHashObj* pStDeleted, bool hasEndTs) { SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; SStreamSessionAggOperatorInfo* pInfo = pOperator->info; bool masterScan = true; @@ -2776,14 +2835,14 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); - startTsCols = (int64_t*) pStartTsCol->pData; + startTsCols = (int64_t*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = NULL; if (hasEndTs) { pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->endTsIndex); } else { pEndTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); } - endTsCols = (int64_t*) pEndTsCol->pData; + endTsCols = (int64_t*)pEndTsCol->pData; } else { return; } @@ -2791,10 +2850,9 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData SStreamAggSupporter* pAggSup = &pInfo->streamAggSup; for (int32_t i = 0; i < pSDataBlock->info.rows;) { int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], - endTsCols[i], groupId, gap, &winIndex); - winRows = updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, - pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); + SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, startTsCols[i], endTsCols[i], groupId, gap, &winIndex); + winRows = + updateSessionWindowInfo(pCurWin, startTsCols, endTsCols, pSDataBlock->info.rows, i, pInfo->gap, pStDeleted); code = doOneWindowAgg(pInfo, pSDataBlock, pCurWin, &pResult, i, winRows, numOfOutput, pOperator); if (code != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -2817,14 +2875,15 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData } } -static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, - int32_t tsIndex, int32_t numOfOutput, int64_t gap, SArray* result) { +static void doClearSessionWindows(SStreamAggSupporter* pAggSup, SExprSupp* pSup, SSDataBlock* pBlock, int32_t tsIndex, + int32_t numOfOutput, int64_t gap, SArray* result) { SColumnInfoData* pColDataInfo = taosArrayGet(pBlock->pDataBlock, tsIndex); TSKEY* tsCols = (TSKEY*)pColDataInfo->pData; int32_t step = 0; for (int32_t i = 0; i < pBlock->info.rows; i += step) { int32_t winIndex = 0; - SResultWindowInfo* pCurWin = getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex); + SResultWindowInfo* pCurWin = + getSessionTimeWindow(pAggSup, tsCols[i], INT64_MIN, pBlock->info.groupId, gap, &winIndex); step = updateSessionWindowInfo(pCurWin, tsCols, NULL, pBlock->info.rows, i, gap, NULL); ASSERT(isInWindow(pCurWin, tsCols[i], gap)); doClearWindowImpl(&pCurWin->pos, pAggSup->pResultBuf, pSup, numOfOutput); @@ -2858,7 +2917,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It while (((*Ite) = taosHashIterate(pStDeleted, *Ite)) != NULL) { SColumnInfoData* pColInfoData = taosArrayGet(pBlock->pDataBlock, 0); colDataAppend(pColInfoData, pBlock->info.rows, *Ite, false); - for (int32_t i = 1; i < pBlock->info.numOfCols; i++) { + for (int32_t i = 1; i < taosArrayGetSize(pBlock->pDataBlock); i++) { pColInfoData = taosArrayGet(pBlock->pDataBlock, i); colDataAppendNULL(pColInfoData, pBlock->info.rows); } @@ -2874,7 +2933,7 @@ void doBuildDeleteDataBlock(SHashObj* pStDeleted, SSDataBlock* pBlock, void** It static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWinArray, int32_t groupId, int32_t numOfOutput, SOperatorInfo* pOperator) { - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; int32_t size = taosArrayGetSize(pWinArray); @@ -2893,7 +2952,7 @@ static void rebuildTimeWindow(SStreamSessionAggOperatorInfo* pInfo, SArray* pWin int32_t chWinSize = taosArrayGetSize(pChWins); int32_t index = binarySearch(pChWins, chWinSize, pParentWin->win.skey, TSDB_ORDER_DESC, getSessionWindowEndkey); if (index < 0) { - index = 0; + index = 0; } for (int32_t k = index; k < chWinSize; k++) { SResultWindowInfo* pcw = taosArrayGet(pChWins, k); @@ -2914,15 +2973,14 @@ typedef SResultWindowInfo* (*__get_win_info_)(void*); SResultWindowInfo* getResWinForSession(void* pData) { return (SResultWindowInfo*)pData; } SResultWindowInfo* getResWinForState(void* pData) { return &((SStateWindowInfo*)pData)->winInfo; } -int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, - __get_win_info_ fn) { +int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArray* pClosed, __get_win_info_ fn) { // Todo(liuyao) save window to tdb - void **pIte = NULL; + void** pIte = NULL; size_t keyLen = 0; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { uint64_t* pGroupId = taosHashGetKey(pIte, &keyLen); - SArray *pWins = (SArray *) (*pIte); - int32_t size = taosArrayGetSize(pWins); + SArray* pWins = (SArray*)(*pIte); + int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { void* pWin = taosArrayGet(pWins, i); SResultWindowInfo* pSeWin = fn(pWin); @@ -2943,9 +3001,9 @@ int32_t closeSessionWindow(SHashObj* pHashMap, STimeWindowAggSupp* pTwSup, SArra } int32_t getAllSessionWindow(SHashObj* pHashMap, SArray* pClosed, __get_win_info_ fn) { - void **pIte = NULL; + void** pIte = NULL; while ((pIte = taosHashIterate(pHashMap, pIte)) != NULL) { - SArray *pWins = (SArray *) (*pIte); + SArray* pWins = (SArray*)(*pIte); int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { void* pWin = taosArrayGet(pWins, i); @@ -2969,12 +3027,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { } else if (pOperator->status == OP_RES_TO_RETURN) { doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { + /*printDataBlock(pInfo->pDelRes, "session del");*/ return pInfo->pDelRes; } doBuildResultDatablock(pOperator, pBInfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); if (pBInfo->pRes->info.rows == 0 || !hasDataInGroupInfo(&pInfo->groupResInfo)) { doSetOperatorCompleted(pOperator); } + /*printDataBlock(pBInfo->pRes, "session insert");*/ return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } @@ -2990,7 +3050,8 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { if (pBlock->info.type == STREAM_REPROCESS) { SArray* pWins = taosArrayInit(16, sizeof(SResultWindowInfo)); - doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, pInfo->gap, pWins); + doClearSessionWindows(&pInfo->streamAggSup, &pOperator->exprSupp, pBlock, 0, pOperator->exprSupp.numOfExprs, + pInfo->gap, pWins); if (IS_FINAL_OP(pInfo)) { int32_t childIndex = getChildIndex(pBlock); SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, childIndex); @@ -3014,13 +3075,14 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { int32_t size = taosArrayGetSize(pInfo->pChildren); // if chIndex + 1 - size > 0, add new child for (int32_t i = 0; i < chIndex + 1 - size; i++) { - SOperatorInfo* pChildOp = createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); + SOperatorInfo* pChildOp = + createStreamFinalSessionAggOperatorInfo(NULL, pInfo->pPhyNode, pOperator->pTaskInfo, 0); if (!pChildOp) { longjmp(pOperator->pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } taosArrayPush(pInfo->pChildren, &pChildOp); } - SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); + SOperatorInfo* pChildOp = taosArrayGetP(pInfo->pChildren, chIndex); setInputDataBlock(pChildOp, pChildOp->exprSupp.pCtx, pBlock, TSDB_ORDER_ASC, MAIN_SCAN, true); doStreamSessionAggImpl(pChildOp, pBlock, NULL, NULL, true); } @@ -3031,27 +3093,27 @@ static SSDataBlock* doStreamSessionAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - getResWinForSession); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForSession); copyUpdateResult(pStUpdated, pUpdated); taosHashCleanup(pStUpdated); - finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, - pSup->rowEntryInfoOffset); + finalizeUpdatedResult(pSup->numOfExprs, pInfo->streamAggSup.pResultBuf, pUpdated, pSup->rowEntryInfoOffset); initMultiResInfoFromArrayList(&pInfo->groupResInfo, pUpdated); blockDataEnsureCapacity(pInfo->binfo.pRes, pOperator->resultInfo.capacity); doBuildDeleteDataBlock(pInfo->pStDeleted, pInfo->pDelRes, &pInfo->pDelIterator); if (pInfo->pDelRes->info.rows > 0) { + /*printDataBlock(pInfo->pDelRes, "session del");*/ return pInfo->pDelRes; } doBuildResultDatablock(pOperator, &pInfo->binfo, &pInfo->groupResInfo, pInfo->streamAggSup.pResultBuf); + /*printDataBlock(pBInfo->pRes, "session insert");*/ return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } static void clearStreamSessionOperator(SStreamSessionAggOperatorInfo* pInfo) { - void **pIte = NULL; + void** pIte = NULL; while ((pIte = taosHashIterate(pInfo->streamAggSup.pResultRows, pIte)) != NULL) { - SArray *pWins = (SArray *) (*pIte); + SArray* pWins = (SArray*)(*pIte); int32_t size = taosArrayGetSize(pWins); for (int32_t i = 0; i < size; i++) { SResultWindowInfo* pWin = (SResultWindowInfo*)taosArrayGet(pWins, i); @@ -3077,7 +3139,7 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { SOptrBasicInfo* pBInfo = &pInfo->binfo; TSKEY maxTs = INT64_MIN; SExprSupp* pSup = &pOperator->exprSupp; - + if (pOperator->status == OP_EXEC_DONE) { return NULL; } else if (pOperator->status == OP_RES_TO_RETURN) { @@ -3159,8 +3221,8 @@ static SSDataBlock* doStreamSessionSemiAgg(SOperatorInfo* pOperator) { return pBInfo->pRes->info.rows == 0 ? NULL : pBInfo->pRes; } -SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, - SPhysiNode* pPhyNode, SExecTaskInfo* pTaskInfo, int32_t numOfChild) { +SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream, SPhysiNode* pPhyNode, + SExecTaskInfo* pTaskInfo, int32_t numOfChild) { int32_t code = TSDB_CODE_OUT_OF_MEMORY; SOperatorInfo* pOperator = createStreamSessionAggOperatorInfo(downstream, pPhyNode, pTaskInfo); if (pOperator == NULL) { @@ -3178,15 +3240,14 @@ SOperatorInfo* createStreamFinalSessionAggOperatorInfo(SOperatorInfo* downstream blockDataEnsureCapacity(pInfo->pUpdateRes, 128); pOperator->name = "StreamSessionSemiAggOperator"; pOperator->fpSet = - createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, destroyStreamSessionAggOperatorInfo, - aggEncodeResultRow, aggDecodeResultRow, NULL); + createOperatorFpSet(operatorDummyOpenFn, doStreamSessionSemiAgg, NULL, NULL, + destroyStreamSessionAggOperatorInfo, aggEncodeResultRow, aggDecodeResultRow, NULL); } pOperator->operatorType = pPhyNode->type; if (numOfChild > 0) { pInfo->pChildren = taosArrayInit(numOfChild, sizeof(void*)); for (int32_t i = 0; i < numOfChild; i++) { - SOperatorInfo* pChild = - createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); + SOperatorInfo* pChild = createStreamFinalSessionAggOperatorInfo(NULL, pPhyNode, pTaskInfo, 0); if (pChild == NULL) { goto _error; } @@ -3304,8 +3365,8 @@ SStateWindowInfo* getStateWindowByTs(SStreamAggSupporter* pAggSup, TSKEY ts, uin return NULL; } -SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, - uint64_t groupId, char* pKeyData, SColumn* pCol, int32_t* pIndex) { +SStateWindowInfo* getStateWindow(SStreamAggSupporter* pAggSup, TSKEY ts, uint64_t groupId, char* pKeyData, + SColumn* pCol, int32_t* pIndex) { SArray* pWinInfos = getWinInfos(pAggSup, groupId); pAggSup->pCurWins = pWinInfos; int32_t size = taosArrayGetSize(pWinInfos); @@ -3437,11 +3498,10 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl char* pKeyData = colDataGetData(pKeyColInfo, i); int32_t winIndex = 0; bool allEqual = true; - SStateWindowInfo* pCurWin = - getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData, - &pInfo->stateCol, &winIndex); - winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, - pSDataBlock->info.rows, i, &allEqual, pInfo->pSeDeleted); + SStateWindowInfo* pCurWin = + getStateWindow(pAggSup, tsCols[i], pSDataBlock->info.groupId, pKeyData, &pInfo->stateCol, &winIndex); + winRows = updateStateWindowInfo(pAggSup->pCurWins, winIndex, tsCols, pKeyColInfo, pSDataBlock->info.rows, i, + &allEqual, pInfo->pSeDeleted); if (!allEqual) { taosArrayPush(pAggSup->pScanWindow, &pCurWin->winInfo.win); taosHashRemove(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition)); @@ -3455,8 +3515,7 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl pCurWin->winInfo.isClosed = false; if (pInfo->twAggSup.calTrigger == STREAM_TRIGGER_AT_ONCE) { SWinRes value = {.ts = pCurWin->winInfo.win.skey, .groupId = groupId}; - code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), - &value, sizeof(SWinRes)); + code = taosHashPut(pSeUpdated, &pCurWin->winInfo.pos, sizeof(SResultRowPosition), &value, sizeof(SWinRes)); if (code != TSDB_CODE_SUCCESS) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } @@ -3470,7 +3529,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { return NULL; } - SExprSupp* pSup = &pOperator->exprSupp; + SExprSupp* pSup = &pOperator->exprSupp; SStreamStateAggOperatorInfo* pInfo = pOperator->info; SOptrBasicInfo* pBInfo = &pInfo->binfo; if (pOperator->status == OP_RES_TO_RETURN) { @@ -3512,8 +3571,7 @@ static SSDataBlock* doStreamStateAgg(SOperatorInfo* pOperator) { // restore the value pOperator->status = OP_RES_TO_RETURN; - closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, - getResWinForState); + closeSessionWindow(pInfo->streamAggSup.pResultRows, &pInfo->twAggSup, pUpdated, getResWinForState); copyUpdateResult(pSeUpdated, pUpdated); taosHashCleanup(pSeUpdated); @@ -3584,6 +3642,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->pSeDeleted = taosHashInit(64, hashFn, true, HASH_NO_LOCK); pInfo->pDelIterator = NULL; pInfo->pDelRes = createOneDataBlock(pResBlock, false); + pInfo->pDelRes->info.type = STREAM_DELETE; blockDataEnsureCapacity(pInfo->pDelRes, 64); pInfo->pChildren = NULL; @@ -3612,48 +3671,48 @@ _error: return NULL; } -typedef struct SMergeIntervalAggOperatorInfo { +typedef struct SMergeAlignedIntervalAggOperatorInfo { SIntervalAggOperatorInfo intervalAggOperatorInfo; bool hasGroupId; uint64_t groupId; SSDataBlock* prefetchedBlock; bool inputBlocksFinished; -} SMergeIntervalAggOperatorInfo; +} SMergeAlignedIntervalAggOperatorInfo; -void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { - SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; +void destroyMergeAlignedIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SMergeAlignedIntervalAggOperatorInfo* miaInfo = (SMergeAlignedIntervalAggOperatorInfo*)param; destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); } -static int32_t outputMergeIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, TSKEY wstartTs) { - SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; - SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; - SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; +static int32_t outputMergeAlignedIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, + SSDataBlock* pResultBlock, TSKEY wstartTs) { + SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; SExprSupp* pSup = &pOperatorInfo->exprSupp; - bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &wstartTs, TSDB_KEYSIZE, tableGroupId); SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); ASSERT(p1 != NULL); - finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, - pSup->numOfExprs, pSup->rowEntryInfoOffset, pResultBlock, - pTaskInfo); + finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pSup->pCtx, pSup->pExprInfo, pSup->numOfExprs, + pSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); return 0; } -static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, - int32_t scanFlag, SSDataBlock* pResultBlock) { - SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; - SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; +static void doMergeAlignedIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, + SSDataBlock* pBlock, int32_t scanFlag, SSDataBlock* pResultBlock) { + SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; - SExprSupp* pSup = &pOperatorInfo->exprSupp; + SExprSupp* pSup = &pOperatorInfo->exprSupp; int32_t startPos = 0; int32_t numOfOutput = pSup->numOfExprs; @@ -3664,18 +3723,18 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* STimeWindow win; win.skey = blockStartTs; - win.ekey = taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; + win.ekey = + taosTimeAdd(win.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; - //TODO: remove the hash table usage (groupid + winkey => result row position) - int32_t ret = - setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); + // TODO: remove the hash table (groupid + winkey => result row position) + int32_t ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } - TSKEY currTs = blockStartTs; - TSKEY currPos = startPos; + TSKEY currTs = blockStartTs; + TSKEY currPos = startPos; STimeWindow currWin = win; while (1) { ++currPos; @@ -3686,27 +3745,298 @@ static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* continue; } else { updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, - currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); + doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, + tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); - outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); + outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); currTs = tsCols[currPos]; currWin.skey = currTs; - currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, iaInfo->interval.precision) - 1; + currWin.ekey = taosTimeAdd(currWin.skey, iaInfo->interval.interval, iaInfo->interval.intervalUnit, + iaInfo->interval.precision) - + 1; startPos = currPos; - ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pSup->pCtx, - numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); + ret = setTimeWindowOutputBuf(pResultRowInfo, &currWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pSup->pCtx, numOfOutput, pSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); } } } updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &currWin, true); - doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, - currPos - startPos, tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); + doApplyFunctions(pTaskInfo, pSup->pCtx, &currWin, &iaInfo->twAggSup.timeWindowData, startPos, currPos - startPos, + tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); + + outputMergeAlignedIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); +} + +static SSDataBlock* doMergeAlignedIntervalAgg(SOperatorInfo* pOperator) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + + SMergeAlignedIntervalAggOperatorInfo* miaInfo = pOperator->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + if (pOperator->status == OP_EXEC_DONE) { + return NULL; + } + + SExprSupp* pSup = &pOperator->exprSupp; + SSDataBlock* pRes = iaInfo->binfo.pRes; + blockDataCleanup(pRes); + blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); + + if (!miaInfo->inputBlocksFinished) { + SOperatorInfo* downstream = pOperator->pDownstream[0]; + int32_t scanFlag = MAIN_SCAN; + while (1) { + SSDataBlock* pBlock = NULL; + if (miaInfo->prefetchedBlock == NULL) { + pBlock = downstream->fpSet.getNextFn(downstream); + } else { + pBlock = miaInfo->prefetchedBlock; + miaInfo->groupId = pBlock->info.groupId; + miaInfo->prefetchedBlock = NULL; + } + + if (pBlock == NULL) { + miaInfo->inputBlocksFinished = true; + break; + } + + if (!miaInfo->hasGroupId) { + miaInfo->hasGroupId = true; + miaInfo->groupId = pBlock->info.groupId; + } else if (miaInfo->groupId != pBlock->info.groupId) { + miaInfo->prefetchedBlock = pBlock; + break; + } + + getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); + setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true); + doMergeAlignedIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); + + if (pRes->info.rows >= pOperator->resultInfo.threshold) { + break; + } + } + + pRes->info.groupId = miaInfo->groupId; + } + + if (pRes->info.rows == 0) { + doSetOperatorCompleted(pOperator); + } + + size_t rows = pRes->info.rows; + pOperator->resultInfo.totalRows += rows; + return (rows == 0) ? NULL : pRes; +} + +SOperatorInfo* createMergeAlignedIntervalOperatorInfo(SOperatorInfo* downstream, SExprInfo* pExprInfo, + int32_t numOfCols, SSDataBlock* pResBlock, SInterval* pInterval, + int32_t primaryTsSlotId, SExecTaskInfo* pTaskInfo) { + SMergeAlignedIntervalAggOperatorInfo* miaInfo = taosMemoryCalloc(1, sizeof(SMergeAlignedIntervalAggOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + if (miaInfo == NULL || pOperator == NULL) { + goto _error; + } + + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExprSupp* pSup = &pOperator->exprSupp; + + iaInfo->win = pTaskInfo->window; + iaInfo->order = TSDB_ORDER_ASC; + iaInfo->interval = *pInterval; + iaInfo->execModel = pTaskInfo->execModel; + iaInfo->primaryTsIndex = primaryTsSlotId; + + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; + initResultSizeInfo(pOperator, 4096); + + int32_t code = + initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + initBasicInfo(&iaInfo->binfo, pResBlock); + + initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win); + + iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, iaInfo); + if (iaInfo->timeWindowInterpo) { + iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); + } + + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + initResultRowInfo(&iaInfo->binfo.resultRowInfo); + + pOperator->name = "TimeMergeAlignedIntervalAggOperator"; + pOperator->operatorType = QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; + pOperator->blocking = false; + pOperator->status = OP_NOT_OPENED; + pOperator->exprSupp.pExprInfo = pExprInfo; + pOperator->pTaskInfo = pTaskInfo; + pOperator->exprSupp.numOfExprs = numOfCols; + pOperator->info = miaInfo; + + pOperator->fpSet = createOperatorFpSet(operatorDummyOpenFn, doMergeAlignedIntervalAgg, NULL, NULL, + destroyMergeAlignedIntervalOperatorInfo, NULL, NULL, NULL); + + code = appendDownstream(pOperator, &downstream, 1); + if (code != TSDB_CODE_SUCCESS) { + goto _error; + } + + return pOperator; + +_error: + destroyMergeAlignedIntervalOperatorInfo(miaInfo, numOfCols); + taosMemoryFreeClear(miaInfo); + taosMemoryFreeClear(pOperator); + pTaskInfo->code = code; + return NULL; +} + +//===================================================================================================================== +// merge interval operator +typedef struct SMergeIntervalAggOperatorInfo { + SIntervalAggOperatorInfo intervalAggOperatorInfo; + + SHashObj* groupIntervalHash; + bool hasGroupId; + uint64_t groupId; + SSDataBlock* prefetchedBlock; + bool inputBlocksFinished; +} SMergeIntervalAggOperatorInfo; + +void destroyMergeIntervalOperatorInfo(void* param, int32_t numOfOutput) { + SMergeIntervalAggOperatorInfo* miaInfo = (SMergeIntervalAggOperatorInfo*)param; + taosHashCleanup(miaInfo->groupIntervalHash); + destroyIntervalOperatorInfo(&miaInfo->intervalAggOperatorInfo, numOfOutput); +} + +static int32_t outputPrevIntervalResult(SOperatorInfo* pOperatorInfo, uint64_t tableGroupId, SSDataBlock* pResultBlock, + STimeWindow* newWin) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + SExprSupp* pExprSup = &pOperatorInfo->exprSupp; + + STimeWindow* prevWin = taosHashGet(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + if (prevWin == NULL) { + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + return 0; + } + + if (newWin == NULL || (ascScan && newWin->skey > prevWin->ekey || (!ascScan) && newWin->skey < prevWin->ekey)) { + SET_RES_WINDOW_KEY(iaInfo->aggSup.keyBuf, &prevWin->skey, TSDB_KEYSIZE, tableGroupId); + SResultRowPosition* p1 = (SResultRowPosition*)taosHashGet(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, + GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + ASSERT(p1 != NULL); + + finalizeResultRowIntoResultDataBlock(iaInfo->aggSup.pResultBuf, p1, pExprSup->pCtx, pExprSup->pExprInfo, + pExprSup->numOfExprs, pExprSup->rowEntryInfoOffset, pResultBlock, pTaskInfo); + taosHashRemove(iaInfo->aggSup.pResultRowHashTable, iaInfo->aggSup.keyBuf, GET_RES_WINDOW_KEY_LEN(TSDB_KEYSIZE)); + if (newWin == NULL) { + taosHashRemove(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId)); + } else { + taosHashPut(miaInfo->groupIntervalHash, &tableGroupId, sizeof(tableGroupId), newWin, sizeof(STimeWindow)); + } + } - outputMergeIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, currTs); + return 0; +} + +static void doMergeIntervalAggImpl(SOperatorInfo* pOperatorInfo, SResultRowInfo* pResultRowInfo, SSDataBlock* pBlock, + int32_t scanFlag, SSDataBlock* pResultBlock) { + SMergeIntervalAggOperatorInfo* miaInfo = pOperatorInfo->info; + SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + + SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; + SExprSupp* pExprSup = &pOperatorInfo->exprSupp; + + int32_t startPos = 0; + int32_t numOfOutput = pExprSup->numOfExprs; + int64_t* tsCols = extractTsCol(pBlock, iaInfo); + uint64_t tableGroupId = pBlock->info.groupId; + bool ascScan = (iaInfo->order == TSDB_ORDER_ASC); + TSKEY blockStartTs = getStartTsKey(&pBlock->info.window, tsCols); + SResultRow* pResult = NULL; + + STimeWindow win = getActiveTimeWindow(iaInfo->aggSup.pResultBuf, pResultRowInfo, blockStartTs, &iaInfo->interval, + iaInfo->interval.precision, &iaInfo->win); + + int32_t ret = + setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, + numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + TSKEY ekey = ascScan ? win.ekey : win.skey; + int32_t forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order); + ASSERT(forwardRows > 0); + + // prev time window not interpolation yet. + if (iaInfo->timeWindowInterpo) { + SResultRowPosition pos = addToOpenWindowList(pResultRowInfo, pResult); + doInterpUnclosedTimeWindow(pOperatorInfo, numOfOutput, pResultRowInfo, pBlock, scanFlag, tsCols, &pos); + + // restore current time window + ret = setTimeWindowOutputBuf(pResultRowInfo, &win, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, pExprSup->pCtx, + numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); + if (ret != TSDB_CODE_SUCCESS) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + // window start key interpolation + doWindowBorderInterpolation(iaInfo, pBlock, pResult, &win, startPos, forwardRows, pExprSup); + } + + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &win, true); + doApplyFunctions(pTaskInfo, pExprSup->pCtx, &win, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, tsCols, + pBlock->info.rows, numOfOutput, iaInfo->order); + doCloseWindow(pResultRowInfo, iaInfo, pResult); + + // output previous interval results after this interval (&win) is closed + outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &win); + + STimeWindow nextWin = win; + while (1) { + int32_t prevEndPos = forwardRows - 1 + startPos; + startPos = getNextQualifiedWindow(&iaInfo->interval, &nextWin, &pBlock->info, tsCols, prevEndPos, iaInfo->order); + if (startPos < 0) { + break; + } + + // null data, failed to allocate more memory buffer + int32_t code = + setTimeWindowOutputBuf(pResultRowInfo, &nextWin, (scanFlag == MAIN_SCAN), &pResult, tableGroupId, + pExprSup->pCtx, numOfOutput, pExprSup->rowEntryInfoOffset, &iaInfo->aggSup, pTaskInfo); + if (code != TSDB_CODE_SUCCESS || pResult == NULL) { + longjmp(pTaskInfo->env, TSDB_CODE_QRY_OUT_OF_MEMORY); + } + + ekey = ascScan ? nextWin.ekey : nextWin.skey; + forwardRows = + getNumOfRowsInTimeWindow(&pBlock->info, tsCols, startPos, ekey, binarySearchForKey, NULL, iaInfo->order); + + // window start(end) key interpolation + doWindowBorderInterpolation(iaInfo, pBlock, pResult, &nextWin, startPos, forwardRows, pExprSup); + + updateTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &nextWin, true); + doApplyFunctions(pTaskInfo, pExprSup->pCtx, &nextWin, &iaInfo->twAggSup.timeWindowData, startPos, forwardRows, + tsCols, pBlock->info.rows, numOfOutput, iaInfo->order); + doCloseWindow(pResultRowInfo, iaInfo, pResult); + + // output previous interval results after this interval (&nextWin) is closed + outputPrevIntervalResult(pOperatorInfo, tableGroupId, pResultBlock, &nextWin); + } + + if (iaInfo->timeWindowInterpo) { + saveDataBlockLastRow(iaInfo->pPrevValues, pBlock, iaInfo->pInterpCols); + } } static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { @@ -3714,11 +4044,12 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { SMergeIntervalAggOperatorInfo* miaInfo = pOperator->info; SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; + SExprSupp* pExpSupp = &pOperator->exprSupp; + if (pOperator->status == OP_EXEC_DONE) { return NULL; } - SExprSupp* pSup = &pOperator->exprSupp; SSDataBlock* pRes = iaInfo->binfo.pRes; blockDataCleanup(pRes); blockDataEnsureCapacity(pRes, pOperator->resultInfo.capacity); @@ -3733,6 +4064,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } else { pBlock = miaInfo->prefetchedBlock; miaInfo->groupId = pBlock->info.groupId; + miaInfo->prefetchedBlock = NULL; } if (pBlock == NULL) { @@ -3749,7 +4081,7 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } getTableScanInfo(pOperator, &iaInfo->order, &scanFlag); - setInputDataBlock(pOperator, pSup->pCtx, pBlock, iaInfo->order, scanFlag, true); + setInputDataBlock(pOperator, pExpSupp->pCtx, pBlock, iaInfo->order, scanFlag, true); doMergeIntervalAggImpl(pOperator, &iaInfo->binfo.resultRowInfo, pBlock, scanFlag, pRes); if (pRes->info.rows >= pOperator->resultInfo.threshold) { @@ -3758,6 +4090,13 @@ static SSDataBlock* doMergeIntervalAgg(SOperatorInfo* pOperator) { } pRes->info.groupId = miaInfo->groupId; + } else { + void* p = taosHashIterate(miaInfo->groupIntervalHash, NULL); + if (p != NULL) { + size_t len = 0; + uint64_t* pKey = taosHashGetKey(p, &len); + outputPrevIntervalResult(pOperator, *pKey, pRes, NULL); + } } if (pRes->info.rows == 0) { @@ -3779,29 +4118,30 @@ SOperatorInfo* createMergeIntervalOperatorInfo(SOperatorInfo* downstream, SExprI } SIntervalAggOperatorInfo* iaInfo = &miaInfo->intervalAggOperatorInfo; - SExprSupp* pSup = &pOperator->exprSupp; - iaInfo->win = pTaskInfo->window; - iaInfo->order = TSDB_ORDER_ASC; - iaInfo->interval = *pInterval; + iaInfo->win = pTaskInfo->window; + iaInfo->order = TSDB_ORDER_ASC; + iaInfo->interval = *pInterval; iaInfo->execModel = pTaskInfo->execModel; + iaInfo->primaryTsIndex = primaryTsSlotId; + SExprSupp* pExprSupp = &pOperator->exprSupp; + size_t keyBufSize = sizeof(int64_t) + sizeof(int64_t) + POINTER_BYTES; initResultSizeInfo(pOperator, 4096); - int32_t code = initAggInfo(&pOperator->exprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); + int32_t code = initAggInfo(pExprSupp, &iaInfo->aggSup, pExprInfo, numOfCols, keyBufSize, pTaskInfo->id.str); initBasicInfo(&iaInfo->binfo, pResBlock); initExecTimeWindowInfo(&iaInfo->twAggSup.timeWindowData, &iaInfo->win); - iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pSup->pCtx, numOfCols, iaInfo); + iaInfo->timeWindowInterpo = timeWindowinterpNeeded(pExprSupp->pCtx, numOfCols, iaInfo); if (iaInfo->timeWindowInterpo) { iaInfo->binfo.resultRowInfo.openWindow = tdListNew(sizeof(SResultRowPosition)); - } - - if (code != TSDB_CODE_SUCCESS) { - goto _error; + if (iaInfo->binfo.resultRowInfo.openWindow == NULL) { + goto _error; + } } initResultRowInfo(&iaInfo->binfo.resultRowInfo); @@ -3831,4 +4171,4 @@ _error: taosMemoryFreeClear(pOperator); pTaskInfo->code = code; return NULL; -} +} \ No newline at end of file diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index f21cad2dd62156d3acb87fdc811c747bd6b8ddad..50474ff62e006d1f48eb159805e0bc72c094ef97 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -86,6 +86,7 @@ SSortHandle* tsortCreateSortHandle(SArray* pSortInfo, int32_t type, int32_t page pSortHandle->pOrderedSource = taosArrayInit(4, POINTER_BYTES); pSortHandle->cmpParam.orderInfo = pSortInfo; + pSortHandle->cmpParam.cmpGroupId = false; tsortSetComparFp(pSortHandle, msortComparFn); @@ -146,7 +147,7 @@ static int32_t doAddNewExternalMemSource(SDiskbasedBuf *pBuf, SArray* pAllSource int32_t rowSize = blockDataGetSerialRowSize(pSource->src.pBlock); // The value of numOfRows must be greater than 0, which is guaranteed by the previous memory allocation - int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(pBlock->info.numOfCols))/rowSize; + int32_t numOfRows = (getBufPageSize(pBuf) - blockDataGetSerialMetaSize(taosArrayGetSize(pBlock->pDataBlock)))/rowSize; ASSERT(numOfRows > 0); return blockDataEnsureCapacity(pSource->src.pBlock, numOfRows); } @@ -177,7 +178,7 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(p) + sizeof(int32_t) + p->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(p) + sizeof(int32_t) + taosArrayGetSize(p->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, p); @@ -251,7 +252,7 @@ static int32_t sortComparInit(SMsortComparParam* cmpParam, SArray* pSources, int } static void appendOneRowToDataBlock(SSDataBlock *pBlock, const SSDataBlock* pSource, int32_t* rowIndex) { - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { SColumnInfoData* pColInfo = taosArrayGet(pBlock->pDataBlock, i); SColumnInfoData* pSrcColInfo = taosArrayGet(pSource->pDataBlock, i); @@ -374,9 +375,10 @@ int32_t msortComparFn(const void *pLeft, const void *pRight, void *param) { SSDataBlock* pLeftBlock = pLeftSource->src.pBlock; SSDataBlock* pRightBlock = pRightSource->src.pBlock; - // first sort by block groupId - if (pLeftBlock->info.groupId != pRightBlock->info.groupId) { - return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1; + if (pParam->cmpGroupId) { + if (pLeftBlock->info.groupId != pRightBlock->info.groupId) { + return pLeftBlock->info.groupId < pRightBlock->info.groupId ? -1 : 1; + } } for(int32_t i = 0; i < pInfo->size; ++i) { @@ -498,7 +500,7 @@ static int32_t doInternalMergeSort(SSortHandle* pHandle) { return terrno; } - int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + pDataBlock->info.numOfCols * sizeof(int32_t); + int32_t size = blockDataGetSize(pDataBlock) + sizeof(int32_t) + taosArrayGetSize(pDataBlock->pDataBlock) * sizeof(int32_t); assert(size <= getBufPageSize(pHandle->pBuf)); blockDataToBuf(pPage, pDataBlock); @@ -685,6 +687,11 @@ int32_t tsortSetComparFp(SSortHandle* pHandle, _sort_merge_compar_fn_t fp) { return TSDB_CODE_SUCCESS; } +int32_t tsortSetCompareGroupId(SSortHandle* pHandle, bool compareGroupId) { + pHandle->cmpParam.cmpGroupId = compareGroupId; + return TSDB_CODE_SUCCESS; +} + STupleHandle* tsortNextTuple(SSortHandle* pHandle) { if (pHandle->cmpParam.numOfSources == pHandle->numOfCompletedSources) { return NULL; diff --git a/source/libs/executor/test/executorTests.cpp b/source/libs/executor/test/executorTests.cpp index dba2fb897a1737c93777c8b0a468aa3d019977a9..3c12889acaf98f000867aac92a62b2a76d56bd3c 100644 --- a/source/libs/executor/test/executorTests.cpp +++ b/source/libs/executor/test/executorTests.cpp @@ -62,18 +62,11 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); + pInfo->pBlock = createDataBlock(); - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); // SColumnInfoData colInfo1 = {0}; // colInfo1.info.type = TSDB_DATA_TYPE_BINARY; @@ -115,7 +108,6 @@ SSDataBlock* getDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; return pBlock; @@ -128,28 +120,15 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } if (pInfo->pBlock == NULL) { - pInfo->pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - - pInfo->pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); - - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_TIMESTAMP; - colInfo.info.bytes = sizeof(int64_t); - colInfo.info.colId = 1; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int64_t))); - // colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); - - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo); + pInfo->pBlock = createDataBlock(); - SColumnInfoData colInfo1 = {0}; - colInfo1.info.type = TSDB_DATA_TYPE_INT; - colInfo1.info.bytes = 4; - colInfo1.info.colId = 2; + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_TIMESTAMP, sizeof(int64_t), 1); + blockDataAppendColInfo(pInfo->pBlock, &colInfo); - colInfo1.pData = static_cast(taosMemoryCalloc(pInfo->numOfRowsPerPage, sizeof(int32_t))); - colInfo1.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->numOfRowsPerPage + 7) / 8)); + SColumnInfoData colInfo1 = createColumnInfoData(TSDB_DATA_TYPE_INT, 4, 2); + blockDataAppendColInfo(pInfo->pBlock, &colInfo1); - taosArrayPush(pInfo->pBlock->pDataBlock, &colInfo1); + blockDataEnsureCapacity(pInfo->pBlock, pInfo->numOfRowsPerPage); } else { blockDataCleanup(pInfo->pBlock); } @@ -185,7 +164,6 @@ SSDataBlock* get2ColsDummyBlock(SOperatorInfo* pOperator) { } pBlock->info.rows = pInfo->numOfRowsPerPage; - pBlock->info.numOfCols = 1; pInfo->current += 1; diff --git a/source/libs/executor/test/sortTests.cpp b/source/libs/executor/test/sortTests.cpp index 66ed078bbe305d65906b8ab00ffbb47004cd311f..70def782632863de81498c06a494c3305efb98bb 100644 --- a/source/libs/executor/test/sortTests.cpp +++ b/source/libs/executor/test/sortTests.cpp @@ -62,25 +62,21 @@ SSDataBlock* getSingleColDummyBlock(void* param) { return NULL; } - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(4, sizeof(SColumnInfoData)); + SSDataBlock* pBlock = createDataBlock(); SColumnInfoData colInfo = {0}; colInfo.info.type = pInfo->type; if (pInfo->type == TSDB_DATA_TYPE_NCHAR){ colInfo.info.bytes = TSDB_NCHAR_SIZE * VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else if(pInfo->type == TSDB_DATA_TYPE_BINARY) { colInfo.info.bytes = VARCOUNT + VARSTR_HEADER_SIZE; - colInfo.varmeta.offset = static_cast(taosMemoryCalloc(pInfo->pageRows, sizeof(int32_t))); } else{ colInfo.info.bytes = tDataTypes[pInfo->type].bytes; - colInfo.pData = static_cast(taosMemoryCalloc(pInfo->pageRows, colInfo.info.bytes)); - colInfo.nullbitmap = static_cast(taosMemoryCalloc(1, (pInfo->pageRows + 7) / 8)); } colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + blockDataAppendColInfo(pBlock, &colInfo); + blockDataEnsureCapacity(pBlock, pInfo->pageRows); for (int32_t i = 0; i < pInfo->pageRows; ++i) { SColumnInfoData* pColInfo = static_cast(TARRAY_GET_ELEM(pBlock->pDataBlock, 0)); @@ -128,7 +124,6 @@ SSDataBlock* getSingleColDummyBlock(void* param) { } pBlock->info.rows = pInfo->pageRows; - pBlock->info.numOfCols = 1; return pBlock; } @@ -354,15 +349,10 @@ TEST(testCase, ordered_merge_sort_Test) { SArray* orderInfo = taosArrayInit(1, sizeof(SBlockOrderInfo)); taosArrayPush(orderInfo, &oi); - SSDataBlock* pBlock = static_cast(taosMemoryCalloc(1, sizeof(SSDataBlock))); - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - taosArrayPush(pBlock->pDataBlock, &colInfo); + SSDataBlock* pBlock = createDataBlock(); + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); } SSortHandle* phandle = tsortCreateSortHandle(orderInfo, SORT_MULTISOURCE_MERGE, 1024, 5, pBlock,"test_abc"); diff --git a/source/libs/function/inc/builtinsimpl.h b/source/libs/function/inc/builtinsimpl.h index e691d562c6b4426015a7a7d10906d6c4b307cb5f..4d2f926c125764c019ab8f2ca797bd3b230a683c 100644 --- a/source/libs/function/inc/builtinsimpl.h +++ b/source/libs/function/inc/builtinsimpl.h @@ -101,6 +101,11 @@ bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); bool derivativeFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); int32_t derivativeFunction(SqlFunctionCtx *pCtx); +bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); +bool irateFuncSetup(SqlFunctionCtx *pCtx, SResultRowEntryInfo* pResInfo); +int32_t irateFunction(SqlFunctionCtx *pCtx); +int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock); + bool getFirstLastFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv); int32_t firstFunction(SqlFunctionCtx *pCtx); int32_t firstFunctionMerge(SqlFunctionCtx *pCtx); diff --git a/source/libs/function/inc/taggfunction.h b/source/libs/function/inc/taggfunction.h index c3d61d426d889cecda0723b48c6c26eae16316ff..669c2635b5986eb3e47fee30b8a62b3a89fbc1be 100644 --- a/source/libs/function/inc/taggfunction.h +++ b/source/libs/function/inc/taggfunction.h @@ -28,19 +28,6 @@ extern "C" { #include "function.h" #include "tudf.h" - -extern SAggFunctionInfo aggFunc[35]; - -#define FUNCSTATE_SO 0x0u -#define FUNCSTATE_MO 0x1u // dynamic number of output, not multinumber of output e.g., TOP/BOTTOM -#define FUNCSTATE_STREAM 0x2u // function avail for stream -#define FUNCSTATE_STABLE 0x4u // function avail for super table -#define FUNCSTATE_NEED_TS 0x8u // timestamp is required during query processing -#define FUNCSTATE_SELECTIVITY 0x10u // selectivity functions, can exists along with tag columns - -#define BASIC_FUNC_SO FUNCSTATE_SO | FUNCSTATE_STREAM | FUNCSTATE_STABLE -#define BASIC_FUNC_MO FUNCSTATE_MO | FUNCSTATE_STREAM | FUNCSTATE_STABLE - #define AVG_FUNCTION_INTER_BUFFER_SIZE 50 #define DATA_SET_FLAG ',' // to denote the output area has data, not null value diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index cfad00f45898a60dc196edae522c67246d5afb69..f985cc324d809cd0937e2d886dfff55c1f29b2e3 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -978,6 +978,21 @@ static int32_t translateDerivative(SFunctionNode* pFunc, char* pErrBuf, int32_t return TSDB_CODE_SUCCESS; } +static int32_t translateIrate(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { + if (1 != LIST_LENGTH(pFunc->pParameterList)) { + return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); + } + + uint8_t colType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + + if (!IS_NUMERIC_TYPE(colType)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_DOUBLE].bytes, .type = TSDB_DATA_TYPE_DOUBLE}; + return TSDB_CODE_SUCCESS; +} + static int32_t translateFirstLast(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { // first(col_list) will be rewritten as first(col) if (1 != LIST_LENGTH(pFunc->pParameterList)) { @@ -1252,6 +1267,29 @@ static bool validateMinuteRange(int8_t hour, int8_t minute, char sign) { return false; } +static bool validateTimestampDigits(const SValueNode* pVal) { + if (!IS_INTEGER_TYPE(pVal->node.resType.type)) { + return false; + } + + int64_t tsVal = pVal->datum.i; + char fraction[20] = {0}; + NUM_TO_STRING(pVal->node.resType.type, &tsVal, sizeof(fraction), fraction); + int32_t tsDigits = (int32_t)strlen(fraction); + + if (tsDigits > TSDB_TIME_PRECISION_SEC_DIGITS) { + if (tsDigits == TSDB_TIME_PRECISION_MILLI_DIGITS || + tsDigits == TSDB_TIME_PRECISION_MICRO_DIGITS || + tsDigits == TSDB_TIME_PRECISION_NANO_DIGITS) { + return true; + } else { + return false; + } + } + + return true; +} + static bool validateTimezoneFormat(const SValueNode* pVal) { if (TSDB_DATA_TYPE_BINARY != pVal->node.resType.type) { return false; @@ -1364,6 +1402,15 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (QUERY_NODE_VALUE == nodeType(nodesListGetNode(pFunc->pParameterList, 0))) { + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 0); + + if (!validateTimestampDigits(pValue)) { + pFunc->node.resType = (SDataType){.bytes = 0, .type = TSDB_DATA_TYPE_BINARY}; + return TSDB_CODE_SUCCESS; + } + } + // param1 if (numOfParams == 2) { SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); @@ -1796,6 +1843,16 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { .processFunc = derivativeFunction, .finalizeFunc = functionFinalize }, + { + .name = "irate", + .type = FUNCTION_TYPE_IRATE, + .classification = FUNC_MGT_AGG_FUNC | FUNC_MGT_TIMELINE_FUNC, + .translateFunc = translateIrate, + .getEnvFunc = getIrateFuncEnv, + .initFunc = irateFuncSetup, + .processFunc = irateFunction, + .finalizeFunc = irateFinalize + }, { .name = "last_row", .type = FUNCTION_TYPE_LAST_ROW, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 076ae6146021b33179eb2271086c4b5a613176fd..b0de3b3364e4e07caafa425d0b523c64a28d1bb6 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -59,6 +59,12 @@ typedef struct STuplePos { int32_t offset; } STuplePos; +typedef struct SMinmaxResInfo { + bool assign; // assign the first value or not + int64_t v; + STuplePos tuplePos; +} SMinmaxResInfo; + typedef struct STopBotResItem { SVariant v; uint64_t uid; // it is a table uid, used to extract tag data during building of the final result for the tag data @@ -148,6 +154,12 @@ typedef struct SElapsedInfo { int64_t timeUnit; } SElapsedInfo; +typedef struct STwaInfo { + double dOutput; + SPoint1 p; + STimeWindow win; +} STwaInfo; + typedef struct SHistoFuncBin { double lower; double upper; @@ -195,13 +207,13 @@ typedef struct SMavgInfo { } SMavgInfo; typedef struct SSampleInfo { - int32_t samples; - int32_t totalPoints; - int32_t numSampled; - uint8_t colType; - int16_t colBytes; - char* data; - int64_t* timestamp; + int32_t samples; + int32_t totalPoints; + int32_t numSampled; + uint8_t colType; + int16_t colBytes; + char* data; + STuplePos* tuplePos; } SSampleInfo; typedef struct STailItem { @@ -234,6 +246,22 @@ typedef struct SUniqueInfo { char pItems[]; } SUniqueInfo; +typedef struct SDerivInfo { + double prevValue; // previous value + TSKEY prevTs; // previous timestamp + bool ignoreNegative; // ignore the negative value + int64_t tsWindow; // time window for derivative + bool valueSet; // the value has been set already +} SDerivInfo; + +typedef struct SRateInfo { + double firstValue; + TSKEY firstKey; + double lastValue; + TSKEY lastKey; + int8_t hasResult; // flag to denote has value +} SRateInfo; + #define SET_VAL(_info, numOfElem, res) \ do { \ if ((numOfElem) <= 0) { \ @@ -927,12 +955,6 @@ EFuncDataRequired statisDataRequired(SFunctionNode* pFunc, STimeWindow* pTimeWin return FUNC_DATA_REQUIRED_STATIS_LOAD; } -typedef struct SMinmaxResInfo { - bool assign; // assign the first value or not - int64_t v; - STuplePos tuplePos; -} SMinmaxResInfo; - bool minmaxFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) { if (!functionSetup(pCtx, pResultInfo)) { return false; // not initialized since it has been initialized @@ -1427,11 +1449,13 @@ void setNullSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, int32_t void setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STuplePos* pTuplePos, int32_t rowIndex) { int32_t pageId = pTuplePos->pageId; int32_t offset = pTuplePos->offset; + if (pTuplePos->pageId != -1) { + int32_t numOfCols = taosArrayGetSize(pCtx->pSrcBlock->pDataBlock); SFilePage* pPage = getBufPage(pCtx->pBuf, pageId); bool* nullList = (bool*)((char*)pPage + offset); - char* pStart = (char*)(nullList + pCtx->pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); // todo set the offset value to optimize the performance. for (int32_t j = 0; j < pCtx->subsidiaries.num; ++j) { @@ -1844,9 +1868,8 @@ int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { } numOfElem++; LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); - - break; } + break; } case TSDB_DATA_TYPE_SMALLINT: { int16_t* plist = (int16_t*)pCol->pData; @@ -1871,7 +1894,6 @@ int32_t leastSQRFunction(SqlFunctionCtx* pCtx) { numOfElem++; LEASTSQR_CAL(param, x, plist, i, pInfo->stepVal); } - break; } @@ -3018,7 +3040,7 @@ void doAddIntoResult(SqlFunctionCtx* pCtx, void* pData, int32_t rowIndex, SSData void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = NULL; - int32_t completeRowSize = pSrcBlock->info.rowSize + pSrcBlock->info.numOfCols * sizeof(bool); + int32_t completeRowSize = pSrcBlock->info.rowSize + (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock) * sizeof(bool); if (pCtx->curBufPage == -1) { pPage = getNewBufPage(pCtx->pBuf, 0, &pCtx->curBufPage); @@ -3036,8 +3058,8 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS // keep the current row data, extract method int32_t offset = 0; bool* nullList = (bool*)((char*)pPage + pPage->num); - char* pStart = (char*)(nullList + sizeof(bool) * pSrcBlock->info.numOfCols); - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + char* pStart = (char*)(nullList + sizeof(bool) * (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock)); + for (int32_t i = 0; i < (int32_t) taosArrayGetSize(pSrcBlock->pDataBlock); ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); bool isNull = colDataIsNull_s(pCol, rowIndex); if (isNull) { @@ -3066,11 +3088,13 @@ void saveTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pS void copyTupleData(SqlFunctionCtx* pCtx, int32_t rowIndex, const SSDataBlock* pSrcBlock, STuplePos* pPos) { SFilePage* pPage = getBufPage(pCtx->pBuf, pPos->pageId); + int32_t numOfCols = taosArrayGetSize(pSrcBlock->pDataBlock); + bool* nullList = (bool*)((char*)pPage + pPos->offset); - char* pStart = (char*)(nullList + pSrcBlock->info.numOfCols * sizeof(bool)); + char* pStart = (char*)(nullList + numOfCols * sizeof(bool)); int32_t offset = 0; - for (int32_t i = 0; i < pSrcBlock->info.numOfCols; ++i) { + for (int32_t i = 0; i < numOfCols; ++i) { SColumnInfoData* pCol = taosArrayGet(pSrcBlock->pDataBlock, i); if ((nullList[i] = colDataIsNull_s(pCol, rowIndex)) == true) { offset += pCol->info.bytes; @@ -4350,7 +4374,7 @@ bool getSampleFuncEnv(SFunctionNode* pFunc, SFuncExecEnv* pEnv) { SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pFunc->pParameterList, 0); SValueNode* pVal = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); int32_t numOfSamples = pVal->datum.i; - pEnv->calcMemSize = sizeof(SSampleInfo) + numOfSamples * (pCol->node.resType.bytes + sizeof(int64_t)); + pEnv->calcMemSize = sizeof(SSampleInfo) + numOfSamples * (pCol->node.resType.bytes + sizeof(STuplePos)); return true; } @@ -4371,25 +4395,30 @@ bool sampleFunctionSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResultInfo) return false; } pInfo->data = (char*)pInfo + sizeof(SSampleInfo); - pInfo->timestamp = (int64_t*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); + pInfo->tuplePos = (STuplePos*)((char*)pInfo + sizeof(SSampleInfo) + pInfo->samples * pInfo->colBytes); return true; } -static void sampleAssignResult(SSampleInfo* pInfo, char* data, TSKEY ts, int32_t index) { +static void sampleAssignResult(SSampleInfo* pInfo, char* data, int32_t index) { assignVal(pInfo->data + index * pInfo->colBytes, data, pInfo->colBytes, pInfo->colType); - *(pInfo->timestamp + index) = ts; } -static void doReservoirSample(SSampleInfo* pInfo, char* data, TSKEY ts, int32_t index) { +static void doReservoirSample(SqlFunctionCtx* pCtx, SSampleInfo* pInfo, char* data, int32_t index) { pInfo->totalPoints++; if (pInfo->numSampled < pInfo->samples) { - sampleAssignResult(pInfo, data, ts, pInfo->numSampled); + sampleAssignResult(pInfo, data, pInfo->numSampled); + if (pCtx->subsidiaries.num > 0) { + saveTupleData(pCtx, index, pCtx->pSrcBlock, pInfo->tuplePos + pInfo->numSampled * sizeof(STuplePos)); + } pInfo->numSampled++; } else { int32_t j = taosRand() % (pInfo->totalPoints); if (j < pInfo->samples) { - sampleAssignResult(pInfo, data, ts, j); + sampleAssignResult(pInfo, data, j); + if (pCtx->subsidiaries.num > 0) { + copyTupleData(pCtx, index, pCtx->pSrcBlock, pInfo->tuplePos + j * sizeof(STuplePos)); + } } } } @@ -4400,11 +4429,6 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { SInputColumnInfoData* pInput = &pCtx->input; - TSKEY* tsList = NULL; - if (pInput->pPTS != NULL) { - tsList = (int64_t*)pInput->pPTS->pData; - } - SColumnInfoData* pInputCol = pInput->pData[0]; for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { if (colDataIsNull_s(pInputCol, i)) { @@ -4412,7 +4436,7 @@ int32_t sampleFunction(SqlFunctionCtx* pCtx) { } char* data = colDataGetData(pInputCol, i); - doReservoirSample(pInfo, data, /*tsList[i]*/ 0, i); + doReservoirSample(pCtx, pInfo, data, i); } SET_VAL(pResInfo, pInfo->numSampled, pInfo->numSampled); @@ -4431,6 +4455,7 @@ int32_t sampleFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { int32_t currentRow = pBlock->info.rows; for (int32_t i = 0; i < pInfo->numSampled; ++i) { colDataAppend(pCol, currentRow + i, pInfo->data + i * pInfo->colBytes, false); + setSelectivityValue(pCtx, pBlock, pInfo->tuplePos + i * sizeof(STuplePos), currentRow + i); } return pInfo->numSampled; @@ -4666,12 +4691,6 @@ int32_t uniqueFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return pResInfo->numOfRes; } -typedef struct STwaInfo { - double dOutput; - SPoint1 p; - STimeWindow win; -} STwaInfo; - bool getTwaFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(STwaInfo); return true; @@ -5120,14 +5139,6 @@ int32_t blockDistFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { return TSDB_CODE_SUCCESS; } -typedef struct SDerivInfo { - double prevValue; // previous value - TSKEY prevTs; // previous timestamp - bool ignoreNegative; // ignore the negative value - int64_t tsWindow; // time window for derivative - bool valueSet; // the value has been set already -} SDerivInfo; - bool getDerivativeFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { pEnv->calcMemSize = sizeof(SDerivInfo); return true; @@ -5222,6 +5233,117 @@ int32_t derivativeFunction(SqlFunctionCtx* pCtx) { return numOfElems; } +bool getIrateFuncEnv(struct SFunctionNode* pFunc, SFuncExecEnv* pEnv) { + pEnv->calcMemSize = sizeof(SRateInfo); + return true; +} + +bool irateFuncSetup(SqlFunctionCtx* pCtx, SResultRowEntryInfo* pResInfo) { + if (!functionSetup(pCtx, pResInfo)) { + return false; // not initialized since it has been initialized + } + + SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + + pInfo->firstKey = INT64_MIN; + pInfo->lastKey = INT64_MIN; + pInfo->firstValue = (double)INT64_MIN; + pInfo->lastValue = (double)INT64_MIN; + + pInfo->hasResult = 0; + return true; +} + +int32_t irateFunction(SqlFunctionCtx* pCtx) { + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + SRateInfo* pRateInfo = GET_ROWCELL_INTERBUF(pResInfo); + + SInputColumnInfoData* pInput = &pCtx->input; + SColumnInfoData* pInputCol = pInput->pData[0]; + + SColumnInfoData* pOutput = (SColumnInfoData*)pCtx->pOutput; + + TSKEY* tsList = (int64_t*)pInput->pPTS->pData; + + int32_t numOfElems = 0; + int32_t type = pInputCol->info.type; + + for (int32_t i = pInput->startRowIndex; i < pInput->numOfRows + pInput->startRowIndex; i += 1) { + if (colDataIsNull_f(pInputCol->nullbitmap, i)) { + continue; + } + + numOfElems++; + + char* data = colDataGetData(pInputCol, i); + double v = 0; + GET_TYPED_DATA(v, double, type, data); + + if (INT64_MIN == pRateInfo->lastKey) { + pRateInfo->lastValue = v; + pRateInfo->lastKey = tsList[i]; + continue; + } + + if (tsList[i] > pRateInfo->lastKey) { + if ((INT64_MIN == pRateInfo->firstKey) || pRateInfo->lastKey > pRateInfo->firstKey) { + pRateInfo->firstValue = pRateInfo->lastValue; + pRateInfo->firstKey = pRateInfo->lastKey; + } + + pRateInfo->lastValue = v; + pRateInfo->lastKey = tsList[i]; + + continue; + } + + if ((INT64_MIN == pRateInfo->firstKey) || tsList[i] > pRateInfo->firstKey) { + pRateInfo->firstValue = v; + pRateInfo->firstKey = tsList[i]; + } + + } + + SET_VAL(pResInfo, numOfElems, 1); + return TSDB_CODE_SUCCESS; +} + +static double doCalcRate(const SRateInfo* pRateInfo, double tickPerSec) { + if ((INT64_MIN == pRateInfo->lastKey) || (INT64_MIN == pRateInfo->firstKey) || + (pRateInfo->firstKey >= pRateInfo->lastKey)) { + return 0.0; + } + + double diff = 0; + // If the previous value of the last is greater than the last value, only keep the last point instead of the delta + // value between two values. + diff = pRateInfo->lastValue; + if (diff >= pRateInfo->firstValue) { + diff -= pRateInfo->firstValue; + } + + int64_t duration = pRateInfo->lastKey - pRateInfo->firstKey; + if (duration == 0) { + return 0; + } + + return (duration > 0)? ((double)diff) / (duration/tickPerSec):0.0; +} + +int32_t irateFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { + int32_t slotId = pCtx->pExpr->base.resSchema.slotId; + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, slotId); + + SResultRowEntryInfo* pResInfo = GET_RES_INFO(pCtx); + pResInfo->isNullRes = (pResInfo->numOfRes == 0) ? 1 : 0; + + SRateInfo* pInfo = GET_ROWCELL_INTERBUF(pResInfo); + double result = doCalcRate(pInfo, 1000); + colDataAppend(pCol, pBlock->info.rows, (const char*)&result, pResInfo->isNullRes); + + return pResInfo->numOfRes; +} + int32_t interpFunction(SqlFunctionCtx* pCtx) { #if 0 int32_t fillType = (int32_t) pCtx->param[2].i64; diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 710b01ce59eb04573ed3ec9befe3046f1876df4e..5fcf5e239cc28d639eb2655d8d121c158d2cdec7 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -186,6 +186,13 @@ bool fmIsInterpFunc(int32_t funcId) { return FUNCTION_TYPE_INTERP == funcMgtBuiltins[funcId].type; } +bool fmIsLastRowFunc(int32_t funcId) { + if (funcId < 0 || funcId >= funcMgtBuiltinsNum) { + return false; + } + return FUNCTION_TYPE_LAST_ROW == funcMgtBuiltins[funcId].type; +} + void fmFuncMgtDestroy() { void* m = gFunMgtService.pFuncNameHashTable; if (m != NULL && atomic_val_compare_exchange_ptr((void**)&gFunMgtService.pFuncNameHashTable, m, 0) == m) { diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index f6ae027e48cc01a3d098d25156de168365abd04a..da9474ede0ff75010f7d23dbac0d18f6d29144e4 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -467,7 +467,7 @@ int32_t getUdfdPipeName(char* pipeName, int32_t size) { dnodeId[0] = '1'; } #ifdef _WIN32 - snprintf(pipeName, size, "%s%s", UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); + snprintf(pipeName, size, "%s.%x.%s", UDF_LISTEN_PIPE_NAME_PREFIX,MurmurHash3_32(tsDataDir, strlen(tsDataDir)), dnodeId); #else snprintf(pipeName, size, "%s/%s%s", tsDataDir, UDF_LISTEN_PIPE_NAME_PREFIX, dnodeId); #endif @@ -763,8 +763,8 @@ void freeUdfInterBuf(SUdfInterBuf *buf) { int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlock) { udfBlock->numOfRows = block->info.rows; - udfBlock->numOfCols = block->info.numOfCols; - udfBlock->udfCols = taosMemoryCalloc(udfBlock->numOfCols, sizeof(SUdfColumn*)); + udfBlock->numOfCols = taosArrayGetSize(block->pDataBlock); + udfBlock->udfCols = taosMemoryCalloc(taosArrayGetSize(block->pDataBlock), sizeof(SUdfColumn*)); for (int32_t i = 0; i < udfBlock->numOfCols; ++i) { udfBlock->udfCols[i] = taosMemoryCalloc(1, sizeof(SUdfColumn)); SColumnInfoData *col= (SColumnInfoData*)taosArrayGet(block->pDataBlock, i); @@ -799,7 +799,6 @@ int32_t convertDataBlockToUdfDataBlock(SSDataBlock *block, SUdfDataBlock *udfBlo } int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { - block->info.numOfCols = 1; block->info.rows = udfCol->colData.numOfRows; block->info.hasVarCol = IS_VAR_DATA_TYPE(udfCol->colMeta.type); @@ -830,25 +829,19 @@ int32_t convertUdfColumnToDataBlock(SUdfColumn *udfCol, SSDataBlock *block) { int32_t convertScalarParamToDataBlock(SScalarParam *input, int32_t numOfCols, SSDataBlock *output) { output->info.rows = input->numOfRows; - output->info.numOfCols = numOfCols; - bool hasVarCol = false; - for (int32_t i = 0; i < numOfCols; ++i) { - if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { - hasVarCol = true; - break; - } - } - output->info.hasVarCol = hasVarCol; - output->pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); for (int32_t i = 0; i < numOfCols; ++i) { taosArrayPush(output->pDataBlock, (input + i)->columnData); + + if (IS_VAR_DATA_TYPE((input+i)->columnData->info.type)) { + output->info.hasVarCol = true; + } } return 0; } int32_t convertDataBlockToScalarParm(SSDataBlock *input, SScalarParam *output) { - if (input->info.numOfCols != 1) { + if (taosArrayGetSize(input->pDataBlock) != 1) { fnError("scalar function only support one column"); return -1; } @@ -1089,24 +1082,14 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { int32_t start = pInput->startRowIndex; int32_t numOfRows = pInput->numOfRows; - - SSDataBlock tempBlock = {0}; - tempBlock.info.numOfCols = numOfCols; - tempBlock.info.rows = pInput->totalRows; - tempBlock.info.uid = pInput->uid; - bool hasVarCol = false; - tempBlock.pDataBlock = taosArrayInit(numOfCols, sizeof(SColumnInfoData)); - + SSDataBlock* pTempBlock = createDataBlock(); + pTempBlock->info.rows = pInput->totalRows; + pTempBlock->info.uid = pInput->uid; for (int32_t i = 0; i < numOfCols; ++i) { - SColumnInfoData *col = pInput->pData[i]; - if (IS_VAR_DATA_TYPE(col->info.type)) { - hasVarCol = true; - } - taosArrayPush(tempBlock.pDataBlock, col); + blockDataAppendColInfo(pTempBlock, pInput->pData[i]); } - tempBlock.info.hasVarCol = hasVarCol; - SSDataBlock *inputBlock = blockDataExtractBlock(&tempBlock, start, numOfRows); + SSDataBlock *inputBlock = blockDataExtractBlock(pTempBlock, start, numOfRows); SUdfInterBuf state = {.buf = udfRes->interResBuf, .bufLen = session->bufSize, @@ -1131,7 +1114,9 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { } blockDataDestroy(inputBlock); - taosArrayDestroy(tempBlock.pDataBlock); + + taosArrayDestroy(pTempBlock->pDataBlock); + taosMemoryFree(pTempBlock); releaseUdfFuncHandle(pCtx->udfName); freeUdfInterBuf(&newState); diff --git a/source/libs/function/test/runUdf.c b/source/libs/function/test/runUdf.c index 15109db22064ad8d761821510db930b58db0f51d..7395f1e14cfba22e497e8bc760a5883bbc4e11cd 100644 --- a/source/libs/function/test/runUdf.c +++ b/source/libs/function/test/runUdf.c @@ -45,27 +45,27 @@ int scalarFuncTest() { for (int k = 0; k < 1; ++k) { SSDataBlock block = {0}; SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + for (int32_t i = 0; i < 1; ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pCol = taosArrayGet(pBlock->pDataBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pCol, j, &j); } SScalarParam input = {0}; input.numOfRows = pBlock->info.rows; input.columnData = taosArrayGet(pBlock->pDataBlock, 0); + SScalarParam output = {0}; doCallUdfScalarFunc(handle, &input, 1, &output); taosArrayDestroy(pBlock->pDataBlock); + SColumnInfoData *col = output.columnData; for (int32_t i = 0; i < output.numOfRows; ++i) { if (i % 100 == 0) @@ -89,21 +89,18 @@ int aggregateFuncTest() { return -1; } - SSDataBlock block = {0}; - SSDataBlock *pBlock = █ - pBlock->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); - pBlock->info.numOfCols = 1; + SSDataBlock *pBlock = createDataBlock(); + for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); ++i) { + SColumnInfoData colInfo = createColumnInfoData(TSDB_DATA_TYPE_INT, sizeof(int32_t), 1); + blockDataAppendColInfo(pBlock, &colInfo); + } + + blockDataEnsureCapacity(pBlock, 1024); pBlock->info.rows = 1024; - for (int32_t i = 0; i < pBlock->info.numOfCols; ++i) { - SColumnInfoData colInfo = {0}; - colInfo.info.type = TSDB_DATA_TYPE_INT; - colInfo.info.bytes = sizeof(int32_t); - colInfo.info.colId = 1; - colInfoDataEnsureCapacity(&colInfo, 0, pBlock->info.rows); - for (int32_t j = 0; j < pBlock->info.rows; ++j) { - colDataAppendInt32(&colInfo, j, &j); - } - taosArrayPush(pBlock->pDataBlock, &colInfo); + + SColumnInfoData* pColInfo = bdGetColumnInfoData(pBlock, 0); + for (int32_t j = 0; j < pBlock->info.rows; ++j) { + colDataAppendInt32(pColInfo, j, &j); } SUdfInterBuf buf = {0}; @@ -121,6 +118,7 @@ int aggregateFuncTest() { freeUdfInterBuf(&resultBuf); doTeardownUdf(handle); + blockDataDestroy(pBlock); return 0; } diff --git a/source/libs/index/src/index.c b/source/libs/index/src/index.c index 04d7e04b30848329c9edfda3d569d7aa0d225f5d..d6d55c6be0ce50bb148450cba3be999ec2ae9f49 100644 --- a/source/libs/index/src/index.c +++ b/source/libs/index/src/index.c @@ -35,12 +35,12 @@ #define INDEX_DATA_BOOL_NULL 0x02 #define INDEX_DATA_TINYINT_NULL 0x80 #define INDEX_DATA_SMALLINT_NULL 0x8000 -#define INDEX_DATA_INT_NULL 0x80000000L -#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_INT_NULL 0x80000000LL +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000LL #define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN #define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF #define INDEX_DATA_BINARY_NULL 0xFF #define INDEX_DATA_JSON_NULL 0xFFFFFFFF diff --git a/source/libs/index/src/indexComm.c b/source/libs/index/src/indexComm.c index 99b49f97bd13b57a47a0e1160fdbfd0b4f9e1717..383d47e9c1ee053831e593f5994b560c55931073 100644 --- a/source/libs/index/src/indexComm.c +++ b/source/libs/index/src/indexComm.c @@ -25,12 +25,12 @@ #define INDEX_DATA_BOOL_NULL 0x02 #define INDEX_DATA_TINYINT_NULL 0x80 #define INDEX_DATA_SMALLINT_NULL 0x8000 -#define INDEX_DATA_INT_NULL 0x80000000L -#define INDEX_DATA_BIGINT_NULL 0x8000000000000000L +#define INDEX_DATA_INT_NULL 0x80000000LL +#define INDEX_DATA_BIGINT_NULL 0x8000000000000000LL #define INDEX_DATA_TIMESTAMP_NULL TSDB_DATA_BIGINT_NULL #define INDEX_DATA_FLOAT_NULL 0x7FF00000 // it is an NAN -#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000L // an NAN +#define INDEX_DATA_DOUBLE_NULL 0x7FFFFF0000000000LL // an NAN #define INDEX_DATA_NCHAR_NULL 0xFFFFFFFF #define INDEX_DATA_BINARY_NULL 0xFF #define INDEX_DATA_JSON_NULL 0xFFFFFFFF diff --git a/source/libs/index/test/index_executor_tests.cpp b/source/libs/index/test/index_executor_tests.cpp index b88ffe5b8bdb2058a66d1e56020206643c246e42..a0ac79a2976243a2097f3e4290951322b2b65f9a 100644 --- a/source/libs/index/test/index_executor_tests.cpp +++ b/source/libs/index/test/index_executor_tests.cpp @@ -86,7 +86,7 @@ void sifAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *sl SColumnInfoData idata = {0}; idata.info = *colInfo; - colInfoDataEnsureCapacity(&idata, 0, rows); + colInfoDataEnsureCapacity(&idata, rows); taosArrayPush(res->pDataBlock, &idata); diff --git a/source/libs/nodes/src/nodesCloneFuncs.c b/source/libs/nodes/src/nodesCloneFuncs.c index 5eaca60ea5d8c9f9439a7b9139789622ecad7dbf..05eaa88a83c505efdd9ec6de3b5c553935f6241c 100644 --- a/source/libs/nodes/src/nodesCloneFuncs.c +++ b/source/libs/nodes/src/nodesCloneFuncs.c @@ -351,6 +351,7 @@ static SNode* logicScanCopy(const SScanLogicNode* pSrc, SScanLogicNode* pDst) { COPY_SCALAR_FIELD(intervalUnit); COPY_SCALAR_FIELD(slidingUnit); CLONE_NODE_FIELD(pTagCond); + CLONE_NODE_FIELD(pTagIndexCond); COPY_SCALAR_FIELD(triggerType); COPY_SCALAR_FIELD(watermark); COPY_SCALAR_FIELD(tsColId); @@ -557,6 +558,14 @@ static SNode* physiSessionCopy(const SSessionWinodwPhysiNode* pSrc, SSessionWino return (SNode*)pDst; } +static SNode* physiPartitionCopy(const SPartitionPhysiNode* pSrc, SPartitionPhysiNode* pDst) { + COPY_BASE_OBJECT_FIELD(node, physiNodeCopy); + CLONE_NODE_LIST_FIELD(pExprs); + CLONE_NODE_LIST_FIELD(pPartitionKeys); + CLONE_NODE_LIST_FIELD(pTargets); + return (SNode*)pDst; +} + static SNode* dataBlockDescCopy(const SDataBlockDescNode* pSrc, SDataBlockDescNode* pDst) { COPY_SCALAR_FIELD(dataBlockId); CLONE_NODE_LIST_FIELD(pSlots); @@ -694,7 +703,7 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: return physiSysTableScanCopy((const SSystemTableScanPhysiNode*)pNode, (SSystemTableScanPhysiNode*)pDst); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -702,6 +711,8 @@ SNode* nodesCloneNode(const SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_SESSION: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_SESSION: return physiSessionCopy((const SSessionWinodwPhysiNode*)pNode, (SSessionWinodwPhysiNode*)pDst); + case QUERY_NODE_PHYSICAL_PLAN_PARTITION: + return physiPartitionCopy((const SPartitionPhysiNode*)pNode, (SPartitionPhysiNode*)pDst); default: break; } diff --git a/source/libs/nodes/src/nodesCodeFuncs.c b/source/libs/nodes/src/nodesCodeFuncs.c index bde5159087634a2ad06b6b4a1e38ff06f7a32948..5c5c62d915f1b6cc2fad4aafca4b2f1c3746e44a 100644 --- a/source/libs/nodes/src/nodesCodeFuncs.c +++ b/source/libs/nodes/src/nodesCodeFuncs.c @@ -220,6 +220,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSystemTableScan"; case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: return "PhysiBlockDistScan"; + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + return "PhysiLastRowScan"; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return "PhysiProject"; case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -234,8 +236,8 @@ const char* nodesNodeName(ENodeType type) { return "PhysiSort"; case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return "PhysiHashInterval"; - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: - return "PhysiMergeInterval"; + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + return "PhysiMergeAlignedInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return "PhysiStreamInterval"; case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -2324,6 +2326,7 @@ static const char* jkSubplanNodeAddr = "NodeAddr"; static const char* jkSubplanRootNode = "RootNode"; static const char* jkSubplanDataSink = "DataSink"; static const char* jkSubplanTagCond = "TagCond"; +static const char* jkSubplanTagIndexCond = "TagIndexCond"; static int32_t subplanToJson(const void* pObj, SJson* pJson) { const SSubplan* pNode = (const SSubplan*)pObj; @@ -2353,6 +2356,9 @@ static int32_t subplanToJson(const void* pObj, SJson* pJson) { if (TSDB_CODE_SUCCESS == code) { code = tjsonAddObject(pJson, jkSubplanTagCond, nodeToJson, pNode->pTagCond); } + if (TSDB_CODE_SUCCESS == code) { + code = tjsonAddObject(pJson, jkSubplanTagIndexCond, nodeToJson, pNode->pTagIndexCond); + } return code; } @@ -2386,6 +2392,9 @@ static int32_t jsonToSubplan(const SJson* pJson, void* pObj) { if (TSDB_CODE_SUCCESS == code) { code = jsonToNodeObject(pJson, jkSubplanTagCond, (SNode**)&pNode->pTagCond); } + if (TSDB_CODE_SUCCESS == code) { + code = jsonToNodeObject(pJson, jkSubplanTagIndexCond, (SNode**)&pNode->pTagIndexCond); + } return code; } @@ -3952,7 +3961,7 @@ static int32_t deleteStmtToJson(const void* pObj, SJson* pJson) { code = tjsonAddObject(pJson, jkDeleteStmtCountFunc, nodeToJson, pNode->pCountFunc); } if (TSDB_CODE_SUCCESS == code) { - code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagIndexCond); + code = tjsonAddObject(pJson, jkDeleteStmtTagIndexCond, nodeToJson, pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { code = tjsonAddIntegerToObject(pJson, jkDeleteStmtTimeRangeStartKey, pNode->timeRange.skey); @@ -3981,7 +3990,7 @@ static int32_t jsonToDeleteStmt(const SJson* pJson, void* pObj) { code = jsonToNodeObject(pJson, jkDeleteStmtCountFunc, &pNode->pCountFunc); } if (TSDB_CODE_SUCCESS == code) { - code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagIndexCond); + code = jsonToNodeObject(pJson, jkDeleteStmtTagIndexCond, &pNode->pTagCond); } if (TSDB_CODE_SUCCESS == code) { code = tjsonGetBigIntValue(pJson, jkDeleteStmtTimeRangeStartKey, &pNode->timeRange.skey); @@ -4105,6 +4114,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { return logicPlanToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return physiTagScanNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: @@ -4125,7 +4135,7 @@ static int32_t specificNodeToJson(const void* pObj, SJson* pJson) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return physiSortNodeToJson(pObj, pJson); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -4245,6 +4255,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { return jsonToLogicPlan(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: return jsonToPhysiTagScanNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN: @@ -4265,7 +4276,7 @@ static int32_t jsonToSpecificNode(const SJson* pJson, void* pObj) { case QUERY_NODE_PHYSICAL_PLAN_SORT: return jsonToPhysiSortNode(pJson, pObj); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index a48071ef52397450d3160def9d4f38adbf920d24..13628f85db611bca1be75170980ecce0d05a868f 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -207,9 +207,12 @@ SNode* nodesMakeNode(ENodeType type) { case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_APPS_STMT: case QUERY_NODE_SHOW_SCORES_STMT: - case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_VARIABLES_STMT: + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return makeNode(type, sizeof(SShowStmt)); + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return makeNode(type, sizeof(SShowDnodeVariablesStmt)); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return makeNode(type, sizeof(SShowCreateDatabaseStmt)); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: @@ -270,6 +273,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSystemTableScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: return makeNode(type, sizeof(SBlockDistScanPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: + return makeNode(type, sizeof(SLastRowScanPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_PROJECT: return makeNode(type, sizeof(SProjectPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_MERGE_JOIN: @@ -284,8 +289,8 @@ SNode* nodesMakeNode(ENodeType type) { return makeNode(type, sizeof(SSortPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: return makeNode(type, sizeof(SIntervalPhysiNode)); - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: - return makeNode(type, sizeof(SMergeIntervalPhysiNode)); + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: + return makeNode(type, sizeof(SMergeAlignedIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: return makeNode(type, sizeof(SStreamIntervalPhysiNode)); case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: @@ -637,19 +642,22 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_SHOW_VNODES_STMT: case QUERY_NODE_SHOW_APPS_STMT: case QUERY_NODE_SHOW_SCORES_STMT: - case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_VARIABLES_STMT: + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: { SShowStmt* pStmt = (SShowStmt*)pNode; nodesDestroyNode(pStmt->pDbName); nodesDestroyNode(pStmt->pTbName); break; } + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: // no pointer field + break; case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: taosMemoryFreeClear(((SShowCreateDatabaseStmt*)pNode)->pCfg); break; case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: - taosMemoryFreeClear(((SShowCreateTableStmt*)pNode)->pMeta); + taosMemoryFreeClear(((SShowCreateTableStmt*)pNode)->pCfg); break; case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: // no pointer field case QUERY_NODE_KILL_CONNECTION_STMT: // no pointer field @@ -661,7 +669,7 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pFromTable); nodesDestroyNode(pStmt->pWhere); nodesDestroyNode(pStmt->pCountFunc); - nodesDestroyNode(pStmt->pTagIndexCond); + nodesDestroyNode(pStmt->pTagCond); break; } case QUERY_NODE_QUERY: { @@ -680,7 +688,13 @@ void nodesDestroyNode(SNode* pNode) { SScanLogicNode* pLogicNode = (SScanLogicNode*)pNode; destroyLogicNode((SLogicNode*)pLogicNode); nodesDestroyList(pLogicNode->pScanCols); + nodesDestroyList(pLogicNode->pScanPseudoCols); taosMemoryFreeClear(pLogicNode->pVgroupList); + nodesDestroyList(pLogicNode->pDynamicScanFuncs); + nodesDestroyNode(pLogicNode->pTagCond); + nodesDestroyNode(pLogicNode->pTagIndexCond); + taosArrayDestroy(pLogicNode->pSmaIndexes); + nodesDestroyList(pLogicNode->pPartTags); break; } case QUERY_NODE_LOGIC_PLAN_JOIN: { @@ -775,6 +789,7 @@ void nodesDestroyNode(SNode* pNode) { case QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN: case QUERY_NODE_PHYSICAL_PLAN_SYSTABLE_SCAN: case QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN: + case QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN: destroyScanPhysiNode((SScanPhysiNode*)pNode); break; case QUERY_NODE_PHYSICAL_PLAN_PROJECT: { @@ -819,7 +834,7 @@ void nodesDestroyNode(SNode* pNode) { break; } case QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL: - case QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL: + case QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL: case QUERY_NODE_PHYSICAL_PLAN_STREAM_SEMI_INTERVAL: @@ -888,6 +903,8 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyList(pSubplan->pChildren); nodesDestroyNode((SNode*)pSubplan->pNode); nodesDestroyNode((SNode*)pSubplan->pDataSink); + nodesDestroyNode((SNode*)pSubplan->pTagCond); + nodesDestroyNode((SNode*)pSubplan->pTagIndexCond); nodesClearList(pSubplan->pParents); break; } @@ -1121,6 +1138,7 @@ void* nodesGetValueFromNode(SValueNode* pNode) { case TSDB_DATA_TYPE_NCHAR: case TSDB_DATA_TYPE_VARCHAR: case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_JSON: return (void*)pNode->datum.p; default: break; @@ -1348,6 +1366,17 @@ bool nodesIsRegularOp(const SOperatorNode* pOp) { return false; } +bool nodesIsBitwiseOp(const SOperatorNode* pOp) { + switch (pOp->opType) { + case OP_TYPE_BIT_AND: + case OP_TYPE_BIT_OR: + return true; + default: + break; + } + return false; +} + typedef struct SCollectColumnsCxt { int32_t errCode; const char* pTableAlias; @@ -1639,6 +1668,7 @@ int32_t nodesMergeConds(SNode** pDst, SNodeList** pSrc) { typedef struct SClassifyConditionCxt { bool hasPrimaryKey; bool hasTagIndexCol; + bool hasTagCol; bool hasOtherCol; } SClassifyConditionCxt; @@ -1650,6 +1680,9 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) { pCxt->hasPrimaryKey = true; } else if (pCol->hasIndex) { pCxt->hasTagIndexCol = true; + pCxt->hasTagCol = true; + } else if (COLUMN_TYPE_TAG == pCol->colType) { + pCxt->hasTagCol = true; } else { pCxt->hasOtherCol = true; } @@ -1658,23 +1691,31 @@ static EDealRes classifyConditionImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -typedef enum EConditionType { COND_TYPE_PRIMARY_KEY = 1, COND_TYPE_TAG_INDEX, COND_TYPE_NORMAL } EConditionType; +typedef enum EConditionType { + COND_TYPE_PRIMARY_KEY = 1, + COND_TYPE_TAG_INDEX, + COND_TYPE_TAG, + COND_TYPE_NORMAL +} EConditionType; static EConditionType classifyCondition(SNode* pNode) { SClassifyConditionCxt cxt = {.hasPrimaryKey = false, .hasTagIndexCol = false, .hasOtherCol = false}; nodesWalkExpr(pNode, classifyConditionImpl, &cxt); return cxt.hasOtherCol ? COND_TYPE_NORMAL - : (cxt.hasPrimaryKey && cxt.hasTagIndexCol + : (cxt.hasPrimaryKey && cxt.hasTagCol ? COND_TYPE_NORMAL - : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY : COND_TYPE_TAG_INDEX)); + : (cxt.hasPrimaryKey ? COND_TYPE_PRIMARY_KEY + : (cxt.hasTagIndexCol ? COND_TYPE_TAG_INDEX : COND_TYPE_TAG))); } -static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) { +static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond, + SNode** pOtherCond) { SLogicConditionNode* pLogicCond = (SLogicConditionNode*)(*pCondition); int32_t code = TSDB_CODE_SUCCESS; SNodeList* pPrimaryKeyConds = NULL; + SNodeList* pTagIndexConds = NULL; SNodeList* pTagConds = NULL; SNodeList* pOtherConds = NULL; SNode* pCond = NULL; @@ -1686,6 +1727,14 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S } break; case COND_TYPE_TAG_INDEX: + if (NULL != pTagIndexCond) { + code = nodesListMakeAppend(&pTagIndexConds, nodesCloneNode(pCond)); + } + if (NULL != pTagCond) { + code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); + } + break; + case COND_TYPE_TAG: if (NULL != pTagCond) { code = nodesListMakeAppend(&pTagConds, nodesCloneNode(pCond)); } @@ -1703,11 +1752,15 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S } SNode* pTempPrimaryKeyCond = NULL; + SNode* pTempTagIndexCond = NULL; SNode* pTempTagCond = NULL; SNode* pTempOtherCond = NULL; if (TSDB_CODE_SUCCESS == code) { code = nodesMergeConds(&pTempPrimaryKeyCond, &pPrimaryKeyConds); } + if (TSDB_CODE_SUCCESS == code) { + code = nodesMergeConds(&pTempTagIndexCond, &pTagIndexConds); + } if (TSDB_CODE_SUCCESS == code) { code = nodesMergeConds(&pTempTagCond, &pTagConds); } @@ -1719,6 +1772,9 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S if (NULL != pPrimaryKeyCond) { *pPrimaryKeyCond = pTempPrimaryKeyCond; } + if (NULL != pTagIndexCond) { + *pTagIndexCond = pTempTagIndexCond; + } if (NULL != pTagCond) { *pTagCond = pTempTagCond; } @@ -1729,9 +1785,11 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S *pCondition = NULL; } else { nodesDestroyList(pPrimaryKeyConds); + nodesDestroyList(pTagIndexConds); nodesDestroyList(pTagConds); nodesDestroyList(pOtherConds); nodesDestroyNode(pTempPrimaryKeyCond); + nodesDestroyNode(pTempTagIndexCond); nodesDestroyNode(pTempTagCond); nodesDestroyNode(pTempOtherCond); } @@ -1739,10 +1797,11 @@ static int32_t partitionLogicCond(SNode** pCondition, SNode** pPrimaryKeyCond, S return code; } -int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagCond, SNode** pOtherCond) { +int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** pTagIndexCond, SNode** pTagCond, + SNode** pOtherCond) { if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pCondition) && LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pCondition)->condType) { - return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagCond, pOtherCond); + return partitionLogicCond(pCondition, pPrimaryKeyCond, pTagIndexCond, pTagCond, pOtherCond); } switch (classifyCondition(*pCondition)) { @@ -1752,6 +1811,21 @@ int32_t nodesPartitionCond(SNode** pCondition, SNode** pPrimaryKeyCond, SNode** } break; case COND_TYPE_TAG_INDEX: + if (NULL != pTagIndexCond) { + *pTagIndexCond = *pCondition; + } + if (NULL != pTagCond) { + SNode* pTempCond = *pCondition; + if (NULL != pTagIndexCond) { + pTempCond = nodesCloneNode(*pCondition); + if (NULL == pTempCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + } + *pTagCond = pTempCond; + } + break; + case COND_TYPE_TAG: if (NULL != pTagCond) { *pTagCond = *pCondition; } diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 64fcccc827a93b2571f52ae5d3e5219a29c13043..213e21cdcf81fb4aaed4d9f4278df9d25cf1678b 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -161,7 +161,8 @@ SNode* createShowStmtWithCond(SAstCreateContext* pCxt, ENodeType type, SNode* pD SNode* createShowCreateDatabaseStmt(SAstCreateContext* pCxt, SToken* pDbName); SNode* createShowCreateTableStmt(SAstCreateContext* pCxt, ENodeType type, SNode* pRealTable); SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable); -SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword); +SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId); +SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo); SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t alterType, const SToken* pVal); SNode* createDropUserStmt(SAstCreateContext* pCxt, SToken* pUserName); SNode* createCreateDnodeStmt(SAstCreateContext* pCxt, const SToken* pFqdn, const SToken* pPort); @@ -170,12 +171,15 @@ SNode* createAlterDnodeStmt(SAstCreateContext* pCxt, const SToken* pDnode, const SNode* createCreateIndexStmt(SAstCreateContext* pCxt, EIndexType type, bool ignoreExists, SToken* pIndexName, SToken* pTableName, SNodeList* pCols, SNode* pOptions); SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInterval, SNode* pOffset, SNode* pSliding); -SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName); +SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName); SNode* createCreateComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, const SToken* pDnodeId); -SNode* createTopicOptions(SAstCreateContext* pCxt); -SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, - const SToken* pSubDbName, SNode* pRealTable); +SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, + SNode* pQuery); +SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, + const SToken* pSubDbName, bool withMeta); +SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, + SNode* pRealTable, bool withMeta); SNode* createDropTopicStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pTopicName); SNode* createDropCGroupStmt(SAstCreateContext* pCxt, bool ignoreNotExists, const SToken* pCGroupId, const SToken* pTopicName); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index 862e3e2cc03789cf18f648b44d40ea625e9f6fc2..d06d6ec52724a397f3c279f1685f3b96c96dfdfa 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -47,6 +47,9 @@ typedef struct SParseMetaCache { SHashObj* pUserAuth; // key is SUserAuthInfo serialized string, element is bool indicating whether or not to pass SHashObj* pUdf; // key is funcName, element is SFuncInfo* SHashObj* pTableIndex; // key is tbFName, element is SArray* + SHashObj* pTableCfg; // key is tbFName, element is STableCfg* + SArray* pDnodes; // element is SEpSet + bool dnodeRequired; } SParseMetaCache; int32_t generateSyntaxErrMsg(SMsgBuf* pBuf, int32_t errCode, ...); @@ -77,6 +80,8 @@ int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pD int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); +int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); +int32_t reserveDnodeRequiredInCache(SParseMetaCache* pMetaCache); int32_t getTableMetaFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableMeta** pMeta); int32_t getDbVgInfoFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SArray** pVgInfo); int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, SVgroupInfo* pVgroup); @@ -87,6 +92,8 @@ int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, con bool* pPass); int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo); int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); +int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); +int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes); void destoryParseMetaCache(SParseMetaCache* pMetaCache); #ifdef __cplusplus diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index 4c808fc60981f4c0a482759275503ad0b939465c..08199c4f3708592b60f4341ca801c1e18d0def08 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -80,11 +80,18 @@ alter_account_option ::= CONNS literal. alter_account_option ::= STATE literal. { } /************************************************ create/alter/drop user **********************************************/ -cmd ::= CREATE USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B); } +cmd ::= CREATE USER user_name(A) PASS NK_STRING(B) sysinfo_opt(C). { pCxt->pRootNode = createCreateUserStmt(pCxt, &A, &B, C); } cmd ::= ALTER USER user_name(A) PASS NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PASSWD, &B); } -cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); } +//cmd ::= ALTER USER user_name(A) PRIVILEGE NK_STRING(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_PRIVILEGES, &B); } +cmd ::= ALTER USER user_name(A) ENABLE NK_INTEGER(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_ENABLE, &B); } +cmd ::= ALTER USER user_name(A) SYSINFO NK_INTEGER(B). { pCxt->pRootNode = createAlterUserStmt(pCxt, &A, TSDB_ALTER_USER_SYSINFO, &B); } cmd ::= DROP USER user_name(A). { pCxt->pRootNode = createDropUserStmt(pCxt, &A); } +%type sysinfo_opt { int8_t } +%destructor sysinfo_opt { } +sysinfo_opt(A) ::= . { A = 1; } +sysinfo_opt(A) ::= SYSINFO NK_INTEGER(B). { A = taosStr2Int8(B.z, NULL, 10); } + /************************************************ grant/revoke ********************************************************/ cmd ::= GRANT privileges(A) ON priv_level(B) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C); } cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C); } @@ -373,7 +380,9 @@ cmd ::= SHOW CREATE STABLE full_table_name(A). cmd ::= SHOW QUERIES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } cmd ::= SHOW SCORES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } cmd ::= SHOW TOPICS. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } -cmd ::= SHOW VARIABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT); } +cmd ::= SHOW VARIABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } +cmd ::= SHOW LOCAL VARIABLES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } +cmd ::= SHOW DNODE NK_INTEGER(A) VARIABLES. { pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &A)); } cmd ::= SHOW BNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } cmd ::= SHOW SNODES. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } cmd ::= SHOW CLUSTER. { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } @@ -394,9 +403,9 @@ from_db_opt(A) ::= FROM db_name(B). /************************************************ create index ********************************************************/ cmd ::= CREATE SMA INDEX not_exists_opt(D) index_name(A) ON table_name(B) index_options(C). { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, D, &A, &B, NULL, C); } -cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D) - index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); } -cmd ::= DROP INDEX exists_opt(C) index_name(A) ON table_name(B). { pCxt->pRootNode = createDropIndexStmt(pCxt, C, &A, &B); } +//cmd ::= CREATE FULLTEXT INDEX not_exists_opt(D) +// index_name(A) ON table_name(B) NK_LP col_name_list(C) NK_RP. { pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, D, &A, &B, C, NULL); } +cmd ::= DROP INDEX exists_opt(B) index_name(A). { pCxt->pRootNode = createDropIndexStmt(pCxt, B, &A); } index_options(A) ::= . { A = NULL; } index_options(A) ::= FUNCTION NK_LP func_list(B) NK_RP INTERVAL @@ -412,10 +421,14 @@ func_list(A) ::= func_list(B) NK_COMMA func(C). func(A) ::= function_name(B) NK_LP expression_list(C) NK_RP. { A = createFunctionNode(pCxt, &B, C); } /************************************************ create/drop topic ***************************************************/ -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, C, NULL, NULL); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, &C, NULL); } -cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) - AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmt(pCxt, A, &B, NULL, NULL, C); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS query_expression(C). { pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, A, &B, C); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, false); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) + WITH META AS DATABASE db_name(C). { pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, A, &B, &C, true); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) + AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, false); } +cmd ::= CREATE TOPIC not_exists_opt(A) topic_name(B) + WITH META AS STABLE full_table_name(C). { pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, A, &B, C, true); } cmd ::= DROP TOPIC exists_opt(A) topic_name(B). { pCxt->pRootNode = createDropTopicStmt(pCxt, A, &B); } cmd ::= DROP CONSUMER GROUP exists_opt(A) cgroup_name(B) ON topic_name(C). { pCxt->pRootNode = createDropCGroupStmt(pCxt, A, &B, &C); } @@ -625,6 +638,16 @@ expression(A) ::= column_reference(B) NK_ARROW NK_STRING(C). SToken s = getTokenFromRawExprNode(pCxt, B); A = createRawExprNodeExt(pCxt, &s, &C, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, B), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &C))); } +expression(A) ::= expression(B) NK_BITAND expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } +expression(A) ::= expression(B) NK_BITOR expression(C). { + SToken s = getTokenFromRawExprNode(pCxt, B); + SToken e = getTokenFromRawExprNode(pCxt, C); + A = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, B), releaseRawExprNode(pCxt, C))); + } %type expression_list { SNodeList* } %destructor expression_list { nodesDestroyList($$); } @@ -755,8 +778,9 @@ boolean_primary(A) ::= NK_LP(C) boolean_value_expression(B) NK_RP(D). common_expression(A) ::= expression(B). { A = B; } common_expression(A) ::= boolean_value_expression(B). { A = B; } -/************************************************ from_clause *********************************************************/ -from_clause(A) ::= FROM table_reference_list(B). { A = B; } +/************************************************ from_clause_opt *********************************************************/ +from_clause_opt(A) ::= . { A = NULL; } +from_clause_opt(A) ::= FROM table_reference_list(B). { A = B; } table_reference_list(A) ::= table_reference(B). { A = B; } table_reference_list(A) ::= table_reference_list(B) NK_COMMA table_reference(C). { A = createJoinTableNode(pCxt, JOIN_TYPE_INNER, B, C, NULL); } @@ -790,9 +814,9 @@ join_type(A) ::= INNER. /************************************************ query_specification *************************************************/ query_specification(A) ::= - SELECT set_quantifier_opt(B) select_list(C) from_clause(D) where_clause_opt(E) - partition_by_clause_opt(F) range_opt(J) every_opt(K) fill_opt(L) twindow_clause_opt(G) - group_by_clause_opt(H) having_clause_opt(I). { + SELECT set_quantifier_opt(B) select_list(C) from_clause_opt(D) + where_clause_opt(E) partition_by_clause_opt(F) range_opt(J) every_opt(K) + fill_opt(L) twindow_clause_opt(G) group_by_clause_opt(H) having_clause_opt(I). { A = createSelectStmt(pCxt, B, C, D); A = addWhereClause(pCxt, A, E); A = addPartitionByClause(pCxt, A, F); diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index ccb0d37da65a6be158dfb0295815151bb3ce36f1..ef7f17a4608db27859c4483088688c79b97db29d 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1197,7 +1197,15 @@ SNode* createShowTableDistributedStmt(SAstCreateContext* pCxt, SNode* pRealTable return (SNode*)pStmt; } -SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword) { +SNode* createShowDnodeVariablesStmt(SAstCreateContext* pCxt, SNode* pDnodeId) { + CHECK_PARSER_STATUS(pCxt); + SShowDnodeVariablesStmt* pStmt = (SShowDnodeVariablesStmt*)nodesMakeNode(QUERY_NODE_SHOW_DNODE_VARIABLES_STMT); + CHECK_OUT_OF_MEM(pStmt); + pStmt->pDnodeId = pDnodeId; + return (SNode*)pStmt; +} + +SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const SToken* pPassword, int8_t sysinfo) { CHECK_PARSER_STATUS(pCxt); char password[TSDB_USET_PASSWORD_LEN] = {0}; if (!checkUserName(pCxt, pUserName) || !checkPassword(pCxt, pPassword, password)) { @@ -1207,6 +1215,7 @@ SNode* createCreateUserStmt(SAstCreateContext* pCxt, SToken* pUserName, const ST CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); strcpy(pStmt->password, password); + pStmt->sysinfo = sysinfo; return (SNode*)pStmt; } @@ -1218,15 +1227,26 @@ SNode* createAlterUserStmt(SAstCreateContext* pCxt, SToken* pUserName, int8_t al SAlterUserStmt* pStmt = (SAlterUserStmt*)nodesMakeNode(QUERY_NODE_ALTER_USER_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->useName, pUserName); - if (TSDB_ALTER_USER_PASSWD == alterType) { - char password[TSDB_USET_PASSWORD_LEN] = {0}; - if (!checkPassword(pCxt, pVal, password)) { - nodesDestroyNode((SNode*)pStmt); - return NULL; + pStmt->alterType = alterType; + switch (alterType) { + case TSDB_ALTER_USER_PASSWD: { + char password[TSDB_USET_PASSWORD_LEN] = {0}; + if (!checkPassword(pCxt, pVal, password)) { + nodesDestroyNode((SNode*)pStmt); + return NULL; + } + strcpy(pStmt->password, password); + break; } - strcpy(pStmt->password, password); + case TSDB_ALTER_USER_ENABLE: + pStmt->enable = taosStr2Int8(pVal->z, NULL, 10); + break; + case TSDB_ALTER_USER_SYSINFO: + pStmt->sysinfo = taosStr2Int8(pVal->z, NULL, 10); + break; + default: + break; } - pStmt->alterType = alterType; return (SNode*)pStmt; } @@ -1309,16 +1329,15 @@ SNode* createIndexOption(SAstCreateContext* pCxt, SNodeList* pFuncs, SNode* pInt return (SNode*)pOptions; } -SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName, SToken* pTableName) { +SNode* createDropIndexStmt(SAstCreateContext* pCxt, bool ignoreNotExists, SToken* pIndexName) { CHECK_PARSER_STATUS(pCxt); - if (!checkIndexName(pCxt, pIndexName) || !checkTableName(pCxt, pTableName)) { + if (!checkDbName(pCxt, NULL, true) || !checkIndexName(pCxt, pIndexName)) { return NULL; } SDropIndexStmt* pStmt = (SDropIndexStmt*)nodesMakeNode(QUERY_NODE_DROP_INDEX_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->ignoreNotExists = ignoreNotExists; COPY_STRING_FORM_ID_TOKEN(pStmt->indexName, pIndexName); - COPY_STRING_FORM_ID_TOKEN(pStmt->tableName, pTableName); return (SNode*)pStmt; } @@ -1338,22 +1357,40 @@ SNode* createDropComponentNodeStmt(SAstCreateContext* pCxt, ENodeType type, cons return (SNode*)pStmt; } -SNode* createCreateTopicStmt(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, SNode* pQuery, - const SToken* pSubDbName, SNode* pRealTable) { +SNode* createCreateTopicStmtUseQuery(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, + SNode* pQuery) { CHECK_PARSER_STATUS(pCxt); SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); CHECK_OUT_OF_MEM(pStmt); COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); pStmt->ignoreExists = ignoreExists; - if (NULL != pRealTable) { - strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); - strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); - nodesDestroyNode(pRealTable); - } else if (NULL != pSubDbName) { - COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName); - } else { - pStmt->pQuery = pQuery; - } + pStmt->pQuery = pQuery; + return (SNode*)pStmt; +} + +SNode* createCreateTopicStmtUseDb(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, + const SToken* pSubDbName, bool withMeta) { + CHECK_PARSER_STATUS(pCxt); + SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); + CHECK_OUT_OF_MEM(pStmt); + COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); + pStmt->ignoreExists = ignoreExists; + COPY_STRING_FORM_ID_TOKEN(pStmt->subDbName, pSubDbName); + pStmt->withMeta = withMeta; + return (SNode*)pStmt; +} + +SNode* createCreateTopicStmtUseTable(SAstCreateContext* pCxt, bool ignoreExists, const SToken* pTopicName, + SNode* pRealTable, bool withMeta) { + CHECK_PARSER_STATUS(pCxt); + SCreateTopicStmt* pStmt = (SCreateTopicStmt*)nodesMakeNode(QUERY_NODE_CREATE_TOPIC_STMT); + CHECK_OUT_OF_MEM(pStmt); + COPY_STRING_FORM_ID_TOKEN(pStmt->topicName, pTopicName); + pStmt->ignoreExists = ignoreExists; + pStmt->withMeta = withMeta; + strcpy(pStmt->subDbName, ((SRealTableNode*)pRealTable)->table.dbName); + strcpy(pStmt->subSTbName, ((SRealTableNode*)pRealTable)->table.tableName); + nodesDestroyNode(pRealTable); return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index 60589784511dbe2eab5ab16906f5149e7e2bd5da..d9c4551400fd261a144e467dd29e433c52405f02 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -137,6 +137,9 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, SRealTa code = reserveTableIndexInCache(pCxt->pParseCxt->acctId, pRealTable->table.dbName, pRealTable->table.tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code && (0 == strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_DNODE_VARIABLES))) { + code = reserveDnodeRequiredInCache(pCxt->pMetaCache); + } return code; } @@ -397,12 +400,28 @@ static int32_t collectMetaKeyFromShowVariables(SCollectMetaKeyCxt* pCxt, SShowSt pCxt->pMetaCache); } +static int32_t collectMetaKeyFromShowDnodeVariables(SCollectMetaKeyCxt* pCxt, SShowDnodeVariablesStmt* pStmt) { + int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, + TSDB_INS_TABLE_DNODE_VARIABLES, pCxt->pMetaCache); + if (TSDB_CODE_SUCCESS == code) { + code = reserveDnodeRequiredInCache(pCxt->pMetaCache); + } + return code; +} + static int32_t collectMetaKeyFromShowCreateDatabase(SCollectMetaKeyCxt* pCxt, SShowCreateDatabaseStmt* pStmt) { return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShowCreateTableStmt* pStmt) { - return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); + SName name = {.type = TSDB_TABLE_NAME_T, .acctId = pCxt->pParseCxt->acctId}; + strcpy(name.dbname, pStmt->dbName); + strcpy(name.tname, pStmt->tableName); + int32_t code = catalogRemoveTableMeta(pCxt->pParseCxt->pCatalog, &name); + if (TSDB_CODE_SUCCESS == code) { + code = reserveTableCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); + } + return code; } static int32_t collectMetaKeyFromShowApps(SCollectMetaKeyCxt* pCxt, SShowStmt* pStmt) { @@ -502,8 +521,10 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromShowConnections(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_QUERIES_STMT: return collectMetaKeyFromShowQueries(pCxt, (SShowStmt*)pStmt); - case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_VARIABLES_STMT: return collectMetaKeyFromShowVariables(pCxt, (SShowStmt*)pStmt); + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return collectMetaKeyFromShowDnodeVariables(pCxt, (SShowDnodeVariablesStmt*)pStmt); case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: return collectMetaKeyFromShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pStmt); case QUERY_NODE_SHOW_CREATE_TABLE_STMT: diff --git a/source/libs/parser/src/parCalcConst.c b/source/libs/parser/src/parCalcConst.c index 22d7afd6424a291e397a5a206f822ddadd1e2bc8..0dfc79269e9a47bf7af13521a36c1e0073015f20 100644 --- a/source/libs/parser/src/parCalcConst.c +++ b/source/libs/parser/src/parCalcConst.c @@ -227,12 +227,16 @@ static int32_t calcConstGroupBy(SCalcConstContext* pCxt, SSelectStmt* pSelect) { } } } - DESTORY_LIST(pSelect->pGroupByList); + NODES_DESTORY_LIST(pSelect->pGroupByList); } return code; } -static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { +static int32_t calcConstSelectWithoutFrom(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { + return calcConstProjections(pCxt, pSelect, subquery); +} + +static int32_t calcConstSelectFrom(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { int32_t code = calcConstFromTable(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { code = calcConstProjections(pCxt, pSelect, subquery); @@ -258,6 +262,14 @@ static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bo return code; } +static int32_t calcConstSelect(SCalcConstContext* pCxt, SSelectStmt* pSelect, bool subquery) { + if (NULL == pSelect->pFromTable) { + return calcConstSelectWithoutFrom(pCxt, pSelect, subquery); + } else { + return calcConstSelectFrom(pCxt, pSelect, subquery); + } +} + static int32_t calcConstDelete(SCalcConstContext* pCxt, SDeleteStmt* pDelete) { int32_t code = calcConstFromTable(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { diff --git a/source/libs/parser/src/parInsert.c b/source/libs/parser/src/parInsert.c index c8b78fcc8bf6acfc242a975014ef2d11b38c4d23..5d34250444d88f64df4de049e77dd33dfb709558 100644 --- a/source/libs/parser/src/parInsert.c +++ b/source/libs/parser/src/parInsert.c @@ -257,7 +257,7 @@ static int32_t checkAuth(SInsertParseContext* pCxt, char* pDbFname, bool* pPass) if (pBasicCtx->async) { return getUserAuthFromCache(pCxt->pMetaCache, pBasicCtx->pUser, pDbFname, AUTH_TYPE_WRITE, pPass); } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, + SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, .requestId = pBasicCtx->requestId, .requestObjRefId = pBasicCtx->requestRid, .mgmtEps = pBasicCtx->mgmtEpSet}; @@ -270,11 +270,11 @@ static int32_t getTableSchema(SInsertParseContext* pCxt, SName* pTbName, bool is if (pBasicCtx->async) { return getTableMetaFromCache(pCxt->pMetaCache, pTbName, pTableMeta); } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, + SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, .requestId = pBasicCtx->requestId, .requestObjRefId = pBasicCtx->requestRid, .mgmtEps = pBasicCtx->mgmtEpSet}; - + if (isStb) { return catalogGetSTableMeta(pBasicCtx->pCatalog, &conn, pTbName, pTableMeta); } @@ -286,7 +286,7 @@ static int32_t getTableVgroup(SInsertParseContext* pCxt, SName* pTbName, SVgroup if (pBasicCtx->async) { return getTableVgroupFromCache(pCxt->pMetaCache, pTbName, pVg); } - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, + SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, .requestId = pBasicCtx->requestId, .requestObjRefId = pBasicCtx->requestRid, .mgmtEps = pBasicCtx->mgmtEpSet}; @@ -322,7 +322,7 @@ static int32_t getDBCfg(SInsertParseContext* pCxt, const char* pDbFName, SDbCfgI if (pBasicCtx->async) { CHECK_CODE(getDbCfgFromCache(pCxt->pMetaCache, pDbFName, pInfo)); } else { - SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, + SRequestConnInfo conn = {.pTrans = pBasicCtx->pTransporter, .requestId = pBasicCtx->requestId, .requestObjRefId = pBasicCtx->requestRid, .mgmtEps = pBasicCtx->mgmtEpSet}; @@ -1081,15 +1081,6 @@ end: return code; } -static int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { - *pDst = taosMemoryMalloc(TABLE_META_SIZE(pSrc)); - if (NULL == *pDst) { - return TSDB_CODE_TSC_OUT_OF_MEMORY; - } - memcpy(*pDst, pSrc, TABLE_META_SIZE(pSrc)); - return TSDB_CODE_SUCCESS; -} - static int32_t storeTableMeta(SInsertParseContext* pCxt, SHashObj* pHash, SName* pTableName, const char* pName, int32_t len, STableMeta* pMeta) { SVgroupInfo vg; @@ -1315,15 +1306,6 @@ static void destroyInsertParseContext(SInsertParseContext* pCxt) { destroyBlockArrayList(pCxt->pVgDataBlocks); } -static int32_t checkSchemalessDb(SInsertParseContext* pCxt, char* pDbName) { -// SDbCfgInfo pInfo = {0}; -// char fullName[TSDB_TABLE_FNAME_LEN]; -// snprintf(fullName, sizeof(fullName), "%d.%s", pCxt->pComCxt->acctId, pDbName); -// CHECK_CODE(getDBCfg(pCxt, fullName, &pInfo)); -// return pInfo.schemaless ? TSDB_CODE_SML_INVALID_DB_CONF : TSDB_CODE_SUCCESS; - return TSDB_CODE_SUCCESS; -} - // tb_name // [USING stb_name [(tag1_name, ...)] TAGS (tag1_value, ...)] // [(field1_name, ...)] @@ -1377,8 +1359,6 @@ static int32_t parseInsertBody(SInsertParseContext* pCxt) { SName name; CHECK_CODE(createSName(&name, &tbnameToken, pCxt->pComCxt->acctId, pCxt->pComCxt->db, &pCxt->msg)); - CHECK_CODE(checkSchemalessDb(pCxt, name.dbname)); - tNameExtractFullName(&name, tbFName); CHECK_CODE(taosHashPut(pCxt->pTableNameHashObj, tbFName, strlen(tbFName), &name, sizeof(SName))); char dbFName[TSDB_DB_FNAME_LEN]; diff --git a/source/libs/parser/src/parTokenizer.c b/source/libs/parser/src/parTokenizer.c index 21f227b7eac945d5e2ff6dafc5c90fa211df3200..12b717e56a0648223c5320d5943532124f31bef1 100644 --- a/source/libs/parser/src/parTokenizer.c +++ b/source/libs/parser/src/parTokenizer.c @@ -79,6 +79,7 @@ static SKeyword keywordTable[] = { {"DOUBLE", TK_DOUBLE}, {"DROP", TK_DROP}, {"DURATION", TK_DURATION}, + {"ENABLE", TK_ENABLE}, {"EXISTS", TK_EXISTS}, {"EXPLAIN", TK_EXPLAIN}, {"EVERY", TK_EVERY}, @@ -120,6 +121,7 @@ static SKeyword keywordTable[] = { {"MAXROWS", TK_MAXROWS}, {"MAX_DELAY", TK_MAX_DELAY}, {"MERGE", TK_MERGE}, + {"META", TK_META}, {"MINROWS", TK_MINROWS}, {"MINUS", TK_MINUS}, {"MNODE", TK_MNODE}, @@ -146,7 +148,7 @@ static SKeyword keywordTable[] = { {"PORT", TK_PORT}, {"PPS", TK_PPS}, {"PRECISION", TK_PRECISION}, - {"PRIVILEGE", TK_PRIVILEGE}, + // {"PRIVILEGE", TK_PRIVILEGE}, {"PREV", TK_PREV}, {"QNODE", TK_QNODE}, {"QNODES", TK_QNODES}, @@ -187,6 +189,7 @@ static SKeyword keywordTable[] = { {"STREAMS", TK_STREAMS}, {"STRICT", TK_STRICT}, {"SYNCDB", TK_SYNCDB}, + {"SYSINFO", TK_SYSINFO}, {"TABLE", TK_TABLE}, {"TABLES", TK_TABLES}, {"TAG", TK_TAG}, @@ -222,6 +225,7 @@ static SKeyword keywordTable[] = { {"WATERMARK", TK_WATERMARK}, {"WHERE", TK_WHERE}, {"WINDOW_CLOSE", TK_WINDOW_CLOSE}, + {"WITH", TK_WITH}, {"WRITE", TK_WRITE}, {"_C0", TK_ROWTS}, {"_QENDTS", TK_QENDTS}, diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index a60dba3a9a46a81a0a20dd98397dc83593e59468..7b5c30d3cb25467d007644eceb3cb5ba740a6403 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -133,6 +133,30 @@ static int32_t getTableMeta(STranslateContext* pCxt, const char* pDbName, const return getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pDbName, pTableName, &name), pMeta); } +static int32_t getTableCfg(STranslateContext* pCxt, const SName* pName, STableCfg** pCfg) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = collectUseDatabase(pName, pCxt->pDbs); + if (TSDB_CODE_SUCCESS == code) { + code = collectUseTable(pName, pCxt->pTables); + } + if (TSDB_CODE_SUCCESS == code) { + if (pParCxt->async) { + code = getTableCfgFromCache(pCxt->pMetaCache, pName, pCfg); + } else { + SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; + code = catalogRefreshGetTableCfg(pParCxt->pCatalog, &conn, pName, pCfg); + } + } + if (TSDB_CODE_SUCCESS != code) { + parserError("catalogRefreshGetTableCfg error, code:%s, dbName:%s, tbName:%s", tstrerror(code), pName->dbname, + pName->tname); + } + return code; +} + static int32_t refreshGetTableMeta(STranslateContext* pCxt, const char* pDbName, const char* pTableName, STableMeta** pMeta) { SParseContext* pParCxt = pCxt->pParseCxt; @@ -305,6 +329,24 @@ static int32_t getTableIndex(STranslateContext* pCxt, const SName* pName, SArray return code; } +static int32_t getDnodeList(STranslateContext* pCxt, SArray** pDnodes) { + SParseContext* pParCxt = pCxt->pParseCxt; + int32_t code = TSDB_CODE_SUCCESS; + if (pParCxt->async) { + code = getDnodeListFromCache(pCxt->pMetaCache, pDnodes); + } else { + SRequestConnInfo conn = {.pTrans = pParCxt->pTransporter, + .requestId = pParCxt->requestId, + .requestObjRefId = pParCxt->requestRid, + .mgmtEps = pParCxt->mgmtEpSet}; + code = catalogGetDnodeList(pParCxt->pCatalog, &conn, pDnodes); + } + if (TSDB_CODE_SUCCESS != code) { + parserError("getDnodeList error, code:%s", tstrerror(code)); + } + return code; +} + static int32_t initTranslateContext(SParseContext* pParseCxt, SParseMetaCache* pMetaCache, STranslateContext* pCxt) { pCxt->pParseCxt = pParseCxt; pCxt->errCode = TSDB_CODE_SUCCESS; @@ -647,6 +689,10 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p } static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { + if (NULL != pCxt->pCurrSelectStmt && NULL == pCxt->pCurrSelectStmt->pFromTable) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName); + } + // count(*)/first(*)/last(*) and so on if (0 == strcmp((*pCol)->colName, "*")) { return DEAL_RES_CONTINUE; @@ -966,6 +1012,17 @@ static EDealRes translateJsonOperator(STranslateContext* pCxt, SOperatorNode* pO return DEAL_RES_CONTINUE; } +static EDealRes translateBitwiseOperator(STranslateContext* pCxt, SOperatorNode* pOp) { + SDataType ldt = ((SExprNode*)(pOp->pLeft))->resType; + SDataType rdt = ((SExprNode*)(pOp->pRight))->resType; + if (TSDB_DATA_TYPE_BLOB == ldt.type || TSDB_DATA_TYPE_BLOB == rdt.type) { + return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_WRONG_VALUE_TYPE, ((SExprNode*)(pOp->pRight))->aliasName); + } + pOp->node.resType.type = TSDB_DATA_TYPE_BIGINT; + pOp->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes; + return DEAL_RES_CONTINUE; +} + static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRef) { SOperatorNode* pOp = *pOpRef; @@ -984,6 +1041,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode** pOpRe return translateComparisonOperator(pCxt, pOp); } else if (nodesIsJsonOp(pOp)) { return translateJsonOperator(pCxt, pOp); + } else if (nodesIsBitwiseOp(pOp)) { + return translateBitwiseOperator(pCxt, pOp); } return DEAL_RES_CONTINUE; } @@ -1158,6 +1217,7 @@ static void setFuncClassification(SSelectStmt* pSelect, SFunctionNode* pFunc) { pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); + pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); } } @@ -1450,12 +1510,27 @@ static int32_t addMnodeToVgroupList(const SEpSet* pEpSet, SArray** pVgroupList) return TSDB_CODE_SUCCESS; } -static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { - if (0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLES) && - 0 != strcmp(pRealTable->table.tableName, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)) { - return TSDB_CODE_SUCCESS; +static int32_t dnodeToVgroupsInfo(SArray* pDnodes, SVgroupsInfo** pVgsInfo) { + size_t ndnode = taosArrayGetSize(pDnodes); + *pVgsInfo = taosMemoryCalloc(1, sizeof(SVgroupsInfo) + sizeof(SVgroupInfo) * ndnode); + if (NULL == *pVgsInfo) { + return TSDB_CODE_OUT_OF_MEMORY; + } + (*pVgsInfo)->numOfVgroups = ndnode; + for (int32_t i = 0; i < ndnode; ++i) { + memcpy(&((*pVgsInfo)->vgroups[i].epSet), taosArrayGet(pDnodes, i), sizeof(SEpSet)); } + return TSDB_CODE_SUCCESS; +} + +static bool sysTableFromVnode(const char* pTable) { + return (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLES)) || + (0 == strcmp(pTable, TSDB_INS_TABLE_USER_TABLE_DISTRIBUTED)); +} +static bool sysTableFromDnode(const char* pTable) { return 0 == strcmp(pTable, TSDB_INS_TABLE_DNODE_VARIABLES); } + +static int32_t setVnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { int32_t code = TSDB_CODE_SUCCESS; SArray* vgroupList = NULL; if ('\0' != pRealTable->qualDbName[0]) { @@ -1478,6 +1553,26 @@ static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRea return code; } +static int32_t setDnodeSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + SArray* pDnodes = NULL; + int32_t code = getDnodeList(pCxt, &pDnodes); + if (TSDB_CODE_SUCCESS == code) { + code = dnodeToVgroupsInfo(pDnodes, &pRealTable->pVgroupList); + } + taosArrayDestroy(pDnodes); + return code; +} + +static int32_t setSysTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { + if (sysTableFromVnode(pRealTable->table.tableName)) { + return setVnodeSysTableVgroupList(pCxt, pName, pRealTable); + } else if (sysTableFromDnode(pRealTable->table.tableName)) { + return setDnodeSysTableVgroupList(pCxt, pName, pRealTable); + } else { + return TSDB_CODE_SUCCESS; + } +} + static int32_t setTableVgroupList(STranslateContext* pCxt, SName* pName, SRealTableNode* pRealTable) { if (pCxt->pParseCxt->topicQuery) { return TSDB_CODE_SUCCESS; @@ -1918,7 +2013,7 @@ static int32_t getFillTimeRange(STranslateContext* pCxt, SNode* pWhere, STimeWin } SNode* pPrimaryKeyCond = NULL; - nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL); + nodesPartitionCond(&pCond, &pPrimaryKeyCond, NULL, NULL, NULL); int32_t code = TSDB_CODE_SUCCESS; if (NULL != pPrimaryKeyCond) { @@ -1952,9 +2047,9 @@ static int32_t checkFill(STranslateContext* pCxt, SFillNode* pFill, SValueNode* if (TIME_IS_VAR_DURATION(pInterval->unit)) { int64_t f = 1; if (pInterval->unit == 'n') { - f = 30L * MILLISECOND_PER_DAY; + f = 30LL * MILLISECOND_PER_DAY; } else if (pInterval->unit == 'y') { - f = 365L * MILLISECOND_PER_DAY; + f = 365LL * MILLISECOND_PER_DAY; } intervalRange = pInterval->datum.i * f; } else { @@ -2066,9 +2161,6 @@ static EDealRes checkStateExpr(SNode* pNode, void* pContext) { if (COLUMN_TYPE_TAG == pCol->colType) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_COL); } - if (TSDB_SUPER_TABLE == pCol->tableType) { - return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_STATE_WIN_TABLE); - } } return DEAL_RES_CONTINUE; } @@ -2299,6 +2391,8 @@ static SNode* createGroupingSet(SNode* pExpr) { return (SNode*)pGroupingSet; } +// from: select unique(expr), col1 + col2 from t where_clause partition_by_clause order_by_clause ... +// to: select expr, first(col1) + first(col2) from t where_clause partition_by_clause group by expr order_by_clause ... static int32_t rewriteUniqueStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { if (!pSelect->hasUniqueFunc) { return TSDB_CODE_SUCCESS; @@ -2364,6 +2458,8 @@ static SNode* createOrderByExpr(STranslateContext* pCxt) { return (SNode*)pOrder; } +// from: select tail(expr, k, f) from t where_clause partition_by_clause order_by_clause ... +// to: select expr from t where_clause order by _rowts desc limit k offset f static int32_t rewriteTailStmt(STranslateContext* pCxt, SSelectStmt* pSelect) { if (!pSelect->hasTailFunc) { return TSDB_CODE_SUCCESS; @@ -2412,7 +2508,13 @@ static int32_t replaceOrderByAlias(STranslateContext* pCxt, SNodeList* pProjecti return pCxt->errCode; } -static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { +static int32_t translateSelectWithoutFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { + pCxt->pCurrSelectStmt = pSelect; + pCxt->currClause = SQL_CLAUSE_SELECT; + return translateExprList(pCxt, pSelect->pProjectionList); +} + +static int32_t translateSelectFrom(STranslateContext* pCxt, SSelectStmt* pSelect) { pCxt->pCurrSelectStmt = pSelect; int32_t code = translateFrom(pCxt, pSelect->pFromTable); if (TSDB_CODE_SUCCESS == code) { @@ -2461,6 +2563,14 @@ static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { return code; } +static int32_t translateSelect(STranslateContext* pCxt, SSelectStmt* pSelect) { + if (NULL == pSelect->pFromTable) { + return translateSelectWithoutFrom(pCxt, pSelect); + } else { + return translateSelectFrom(pCxt, pSelect); + } +} + static SNode* createSetOperProject(const char* pTableAlias, SNode* pNode) { SColumnNode* pCol = (SColumnNode*)nodesMakeNode(QUERY_NODE_COLUMN); if (NULL == pCol) { @@ -2572,7 +2682,7 @@ static int32_t partitionDeleteWhere(STranslateContext* pCxt, SDeleteStmt* pDelet SNode* pPrimaryKeyCond = NULL; SNode* pOtherCond = NULL; - int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, &pDelete->pTagIndexCond, &pOtherCond); + int32_t code = nodesPartitionCond(&pDelete->pWhere, &pPrimaryKeyCond, NULL, &pDelete->pTagCond, &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pOtherCond) { code = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_DELETE_WHERE); } @@ -2613,7 +2723,7 @@ static int64_t getUnitPerMinute(uint8_t precision) { case TSDB_TIME_PRECISION_MILLI: return MILLISECOND_PER_MINUTE; case TSDB_TIME_PRECISION_MICRO: - return MILLISECOND_PER_MINUTE * 1000L; + return MILLISECOND_PER_MINUTE * 1000LL; case TSDB_TIME_PRECISION_NANO: return NANOSECOND_PER_MINUTE; default: @@ -3450,13 +3560,24 @@ static int32_t buildRollupAst(STranslateContext* pCxt, SCreateTableStmt* pStmt, return code; } +static int32_t buildRollupFuncs(SNodeList* pFuncs, SArray** pArray) { + if (NULL == pFuncs) { + return TSDB_CODE_SUCCESS; + } + *pArray = taosArrayInit(LIST_LENGTH(pFuncs), TSDB_FUNC_NAME_LEN); + SNode* pNode; + FOREACH(pNode, pFuncs) { + taosArrayPush(*pArray, ((SFunctionNode*)pNode)->functionName); + } + return TSDB_CODE_SUCCESS; +} + static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStmt, SMCreateStbReq* pReq) { pReq->igExists = pStmt->ignoreExists; pReq->delay1 = pStmt->pOptions->maxDelay1; pReq->delay2 = pStmt->pOptions->maxDelay2; pReq->watermark1 = pStmt->pOptions->watermark1; pReq->watermark2 = pStmt->pOptions->watermark2; - // pReq->ttl = pStmt->pOptions->ttl; columnDefNodeToField(pStmt->pCols, &pReq->pColumns); columnDefNodeToField(pStmt->pTags, &pReq->pTags); pReq->numOfColumns = LIST_LENGTH(pStmt->pCols); @@ -3470,6 +3591,7 @@ static int32_t buildCreateStbReq(STranslateContext* pCxt, SCreateTableStmt* pStm } else { pReq->commentLen = -1; } + buildRollupFuncs(pStmt->pOptions->pRollupFuncs, &pReq->pFuncs); SName tableName; tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &tableName), pReq->name); @@ -3632,7 +3754,7 @@ static int32_t translateCreateUser(STranslateContext* pCxt, SCreateUserStmt* pSt strcpy(createReq.user, pStmt->useName); createReq.createType = 0; createReq.superUser = 0; - createReq.sysInfo = 1; + createReq.sysInfo = pStmt->sysinfo; createReq.enable = 1; strcpy(createReq.pass, pStmt->password); @@ -3644,6 +3766,8 @@ static int32_t translateAlterUser(STranslateContext* pCxt, SAlterUserStmt* pStmt strcpy(alterReq.user, pStmt->useName); alterReq.alterType = pStmt->alterType; alterReq.superUser = 0; + alterReq.enable = pStmt->enable; + alterReq.sysInfo = pStmt->sysinfo; strcpy(alterReq.pass, pStmt->password); if (NULL != pCxt->pParseCxt->db) { strcpy(alterReq.dbname, pCxt->pParseCxt->db); @@ -3798,36 +3922,11 @@ static int32_t translateCreateIndex(STranslateContext* pCxt, SCreateIndexStmt* p } static int32_t translateDropIndex(STranslateContext* pCxt, SDropIndexStmt* pStmt) { - SEncoder encoder = {0}; - int32_t contLen = 0; - SVDropTSmaReq dropSmaReq = {0}; - strcpy(dropSmaReq.indexName, pStmt->indexName); - - pCxt->pCmdMsg = taosMemoryMalloc(sizeof(SCmdMsgInfo)); - if (NULL == pCxt->pCmdMsg) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - int32_t ret = 0; - tEncodeSize(tEncodeSVDropTSmaReq, &dropSmaReq, contLen, ret); - if (ret < 0) { - return TSDB_CODE_OUT_OF_MEMORY; - } - - pCxt->pCmdMsg->epSet = pCxt->pParseCxt->mgmtEpSet; - pCxt->pCmdMsg->msgType = TDMT_VND_DROP_SMA; - pCxt->pCmdMsg->msgLen = contLen; - pCxt->pCmdMsg->pMsg = taosMemoryMalloc(pCxt->pCmdMsg->msgLen); - if (NULL == pCxt->pCmdMsg->pMsg) { - return TSDB_CODE_OUT_OF_MEMORY; - } - void* pBuf = pCxt->pCmdMsg->pMsg; - if (tEncodeSVDropTSmaReq(&encoder, &dropSmaReq) < 0) { - tEncoderClear(&encoder); - return TSDB_CODE_OUT_OF_MEMORY; - } - tEncoderClear(&encoder); - return TSDB_CODE_SUCCESS; + SMDropSmaReq dropSmaReq = {0}; + SName name; + tNameExtractFullName(toName(pCxt->pParseCxt->acctId, pCxt->pParseCxt->db, pStmt->indexName, &name), dropSmaReq.name); + dropSmaReq.igNotExists = pStmt->ignoreNotExists; + return buildCmdMsg(pCxt, TDMT_MND_DROP_SMA, (FSerializeFunc)tSerializeSMDropSmaReq, &dropSmaReq); } static int16_t getCreateComponentNodeMsgType(ENodeType type) { @@ -3879,6 +3978,7 @@ static int32_t buildCreateTopicReq(STranslateContext* pCxt, SCreateTopicStmt* pS tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->topicName, strlen(pStmt->topicName)); tNameGetFullDbName(&name, pReq->name); pReq->igExists = pStmt->ignoreExists; + pReq->withMeta = pStmt->withMeta; pReq->sql = strdup(pCxt->pParseCxt->pSql); if (NULL == pReq->sql) { @@ -4081,8 +4181,12 @@ static int32_t translateCreateStream(STranslateContext* pCxt, SCreateStreamStmt* } static int32_t translateDropStream(STranslateContext* pCxt, SDropStreamStmt* pStmt) { - // todo - return TSDB_CODE_SUCCESS; + SMDropStreamReq dropReq = {0}; + SName name; + tNameSetDbName(&name, pCxt->pParseCxt->acctId, pStmt->streamName, strlen(pStmt->streamName)); + tNameGetFullDbName(&name, dropReq.name); + dropReq.igNotExists = pStmt->ignoreNotExists; + return buildCmdMsg(pCxt, TDMT_MND_DROP_STREAM, (FSerializeFunc)tSerializeSMDropStreamReq, &dropReq); } static int32_t readFromFile(char* pName, int32_t* len, char** buf) { @@ -4230,6 +4334,11 @@ static int32_t translateSplitVgroup(STranslateContext* pCxt, SSplitVgroupStmt* p return buildCmdMsg(pCxt, TDMT_MND_SPLIT_VGROUP, (FSerializeFunc)tSerializeSSplitVgroupReq, &req); } +static int32_t translateShowVariables(STranslateContext* pCxt, SShowStmt* pStmt) { + SShowVariablesReq req = {0}; + return buildCmdMsg(pCxt, TDMT_MND_SHOW_VARIABLES, (FSerializeFunc)tSerializeSShowVariablesReq, &req); +} + static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateDatabaseStmt* pStmt) { pStmt->pCfg = taosMemoryCalloc(1, sizeof(SDbCfgInfo)); if (NULL == pStmt->pCfg) { @@ -4239,7 +4348,10 @@ static int32_t translateShowCreateDatabase(STranslateContext* pCxt, SShowCreateD } static int32_t translateShowCreateTable(STranslateContext* pCxt, SShowCreateTableStmt* pStmt) { - return getTableMeta(pCxt, pStmt->dbName, pStmt->tableName, &pStmt->pMeta); + SName name; + toName(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, &name); + + return getTableCfg(pCxt, &name, (STableCfg**)&pStmt->pCfg); } static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { @@ -4371,6 +4483,9 @@ static int32_t translateQuery(STranslateContext* pCxt, SNode* pNode) { case QUERY_NODE_SPLIT_VGROUP_STMT: code = translateSplitVgroup(pCxt, (SSplitVgroupStmt*)pNode); break; + case QUERY_NODE_SHOW_VARIABLES_STMT: + code = translateShowVariables(pCxt, (SShowStmt*)pNode); + break; case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: code = translateShowCreateDatabase(pCxt, (SShowCreateDatabaseStmt*)pNode); break; @@ -4496,6 +4611,25 @@ static int32_t extractShowCreateTableResultSchema(int32_t* numOfCols, SSchema** return TSDB_CODE_SUCCESS; } +static int32_t extractShowLocalVariablesResultSchema(int32_t* numOfCols, SSchema** pSchema) { + *numOfCols = 2; + *pSchema = taosMemoryCalloc((*numOfCols), sizeof(SSchema)); + if (NULL == (*pSchema)) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + (*pSchema)[0].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[0].bytes = TSDB_CONFIG_OPTION_LEN; + strcpy((*pSchema)[0].name, "name"); + + (*pSchema)[1].type = TSDB_DATA_TYPE_BINARY; + (*pSchema)[1].bytes = TSDB_CONFIG_VALUE_LEN; + strcpy((*pSchema)[1].name, "value"); + + return TSDB_CODE_SUCCESS; +} + + int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pSchema) { if (NULL == pRoot) { return TSDB_CODE_SUCCESS; @@ -4514,6 +4648,8 @@ int32_t extractResultSchema(const SNode* pRoot, int32_t* numOfCols, SSchema** pS case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: return extractShowCreateTableResultSchema(numOfCols, pSchema); + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: + return extractShowLocalVariablesResultSchema(numOfCols, pSchema); default: break; } @@ -4538,7 +4674,8 @@ static const char* getSysDbName(ENodeType type) { case QUERY_NODE_SHOW_SNODES_STMT: case QUERY_NODE_SHOW_LICENCE_STMT: case QUERY_NODE_SHOW_CLUSTER_STMT: - case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_VARIABLES_STMT: + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: return TSDB_INFORMATION_SCHEMA_DB; case QUERY_NODE_SHOW_CONNECTIONS_STMT: case QUERY_NODE_SHOW_QUERIES_STMT: @@ -4595,10 +4732,12 @@ static const char* getSysTableName(ENodeType type) { return TSDB_PERFS_TABLE_TOPICS; case QUERY_NODE_SHOW_TRANSACTIONS_STMT: return TSDB_PERFS_TABLE_TRANS; - case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_VARIABLES_STMT: return TSDB_INS_TABLE_CONFIGS; case QUERY_NODE_SHOW_APPS_STMT: return TSDB_PERFS_TABLE_APPS; + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + return TSDB_INS_TABLE_DNODE_VARIABLES; default: break; } @@ -4725,6 +4864,21 @@ static int32_t rewriteShow(STranslateContext* pCxt, SQuery* pQuery) { return code; } +static int32_t rewriteShowDnodeVariables(STranslateContext* pCxt, SQuery* pQuery) { + SSelectStmt* pStmt = NULL; + int32_t code = createSelectStmtForShow(nodeType(pQuery->pRoot), &pStmt); + if (TSDB_CODE_SUCCESS == code) { + code = createOperatorNode(OP_TYPE_EQUAL, "dnode_id", ((SShowDnodeVariablesStmt*)pQuery->pRoot)->pDnodeId, + &pStmt->pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + pQuery->showRewrite = true; + nodesDestroyNode(pQuery->pRoot); + pQuery->pRoot = (SNode*)pStmt; + } + return code; +} + static SNode* createBlockDistInfoFunc() { SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); if (NULL == pFunc) { @@ -5720,10 +5874,13 @@ static int32_t rewriteQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CLUSTER_STMT: case QUERY_NODE_SHOW_TOPICS_STMT: case QUERY_NODE_SHOW_TRANSACTIONS_STMT: - case QUERY_NODE_SHOW_VARIABLE_STMT: + case QUERY_NODE_SHOW_VARIABLES_STMT: case QUERY_NODE_SHOW_APPS_STMT: code = rewriteShow(pCxt, pQuery); break; + case QUERY_NODE_SHOW_DNODE_VARIABLES_STMT: + code = rewriteShowDnodeVariables(pCxt, pQuery); + break; case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: code = rewriteShowTableDist(pCxt, pQuery); break; @@ -5809,6 +5966,7 @@ static int32_t setQuery(STranslateContext* pCxt, SQuery* pQuery) { case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_CREATE_TABLE_STMT: case QUERY_NODE_SHOW_CREATE_STABLE_STMT: + case QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT: pQuery->execMode = QUERY_EXEC_MODE_LOCAL; pQuery->haveResultSet = true; break; diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index d94b430c452a0e131a1ef91f836615dad90bddaa..689ff08ab1731b04562aefaf17e19ddbd625c5ad 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -561,6 +561,10 @@ int32_t buildCatalogReq(const SParseMetaCache* pMetaCache, SCatalogReq* pCatalog if (TSDB_CODE_SUCCESS == code) { code = buildTableReq(pMetaCache->pTableIndex, &pCatalogReq->pTableIndex); } + if (TSDB_CODE_SUCCESS == code) { + code = buildTableReq(pMetaCache->pTableCfg, &pCatalogReq->pTableCfg); + } + pCatalogReq->dNodeRequired = pMetaCache->dnodeRequired; return code; } @@ -656,6 +660,10 @@ int32_t putMetaDataToCache(const SCatalogReq* pCatalogReq, const SMetaData* pMet if (TSDB_CODE_SUCCESS == code) { code = putTableDataToCache(pCatalogReq->pTableIndex, pMetaData->pTableIndex, &pMetaCache->pTableIndex); } + if (TSDB_CODE_SUCCESS == code) { + code = putTableDataToCache(pCatalogReq->pTableCfg, pMetaData->pTableCfg, &pMetaCache->pTableCfg); + } + pMetaCache->pDnodes = pMetaData->pDnodeList; return code; } @@ -861,6 +869,10 @@ int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pT return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableIndex); } +int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache) { + return reserveTableReqInCache(acctId, pDb, pTable, &pMetaCache->pTableCfg); +} + int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes) { char fullName[TSDB_TABLE_FNAME_LEN]; tNameExtractFullName(pName, fullName); @@ -875,6 +887,59 @@ int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, return code; } +STableCfg* tableCfgDup(STableCfg* pCfg) { + STableCfg* pNew = taosMemoryMalloc(sizeof(*pNew)); + + memcpy(pNew, pCfg, sizeof(*pNew)); + if (pNew->pComment) { + pNew->pComment = strdup(pNew->pComment); + } + if (pNew->pFuncs) { + pNew->pFuncs = taosArrayDup(pNew->pFuncs); + } + + int32_t schemaSize = (pCfg->numOfColumns + pCfg->numOfTags) * sizeof(SSchema); + + SSchema* pSchema = taosMemoryMalloc(schemaSize); + memcpy(pSchema, pCfg->pSchemas, schemaSize); + + pNew->pSchemas = pSchema; + + return pNew; +} + +int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput) { + char fullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(pName, fullName); + STableCfg* pCfg = NULL; + int32_t code = getMetaDataFromHash(fullName, strlen(fullName), pMetaCache->pTableCfg, (void**)&pCfg); + if (TSDB_CODE_SUCCESS == code) { + *pOutput = tableCfgDup(pCfg); + if (NULL == *pOutput) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + return code; +} + +int32_t reserveDnodeRequiredInCache(SParseMetaCache* pMetaCache) { + pMetaCache->dnodeRequired = true; + return TSDB_CODE_SUCCESS; +} + +int32_t getDnodeListFromCache(SParseMetaCache* pMetaCache, SArray** pDnodes) { + SMetaRes* pRes = taosArrayGet(pMetaCache->pDnodes, 0); + if (pRes->code) { + return pRes->code; + } + + *pDnodes = taosArrayDup((SArray*)pRes->pRes); + if (NULL == *pDnodes) { + return TSDB_CODE_OUT_OF_MEMORY; + } + return TSDB_CODE_SUCCESS; +} + void destoryParseMetaCache(SParseMetaCache* pMetaCache) { taosHashCleanup(pMetaCache->pTableMeta); taosHashCleanup(pMetaCache->pDbVgroup); @@ -884,4 +949,5 @@ void destoryParseMetaCache(SParseMetaCache* pMetaCache) { taosHashCleanup(pMetaCache->pUserAuth); taosHashCleanup(pMetaCache->pUdf); taosHashCleanup(pMetaCache->pTableIndex); + taosHashCleanup(pMetaCache->pTableCfg); } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index 422661219e9d0b4ca8d0bc1d2ef11f480e48633b..b632f2eb32cf6c2256e0e231b5fccab70acf6711 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,25 +104,26 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 364 +#define YYNOCODE 367 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - SToken yy57; - EOrder yy162; - EJoinType yy204; - SAlterOption yy221; - int32_t yy228; - EFillMode yy270; - EOperatorType yy324; - int64_t yy389; - SNode* yy392; - SDataType yy448; - bool yy481; - ENullOrder yy529; - SNodeList* yy600; + bool yy197; + EJoinType yy208; + SToken yy241; + SAlterOption yy301; + EOrder yy362; + SNodeList* yy368; + EFillMode yy388; + SDataType yy392; + EOperatorType yy412; + ENullOrder yy417; + SNode* yy442; + int32_t yy578; + int8_t yy629; + int64_t yy683; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -138,17 +139,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 636 -#define YYNRULE 464 -#define YYNTOKEN 242 -#define YY_MAX_SHIFT 635 -#define YY_MIN_SHIFTREDUCE 928 -#define YY_MAX_SHIFTREDUCE 1391 -#define YY_ERROR_ACTION 1392 -#define YY_ACCEPT_ACTION 1393 -#define YY_NO_ACTION 1394 -#define YY_MIN_REDUCE 1395 -#define YY_MAX_REDUCE 1858 +#define YYNSTATE 645 +#define YYNRULE 473 +#define YYNTOKEN 244 +#define YY_MAX_SHIFT 644 +#define YY_MIN_SHIFTREDUCE 938 +#define YY_MAX_SHIFTREDUCE 1410 +#define YY_ERROR_ACTION 1411 +#define YY_ACCEPT_ACTION 1412 +#define YY_NO_ACTION 1413 +#define YY_MIN_REDUCE 1414 +#define YY_MAX_REDUCE 1886 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -215,603 +216,631 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (2123) +#define YY_ACTTAB_COUNT (2260) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1688, 1836, 1515, 309, 407, 1688, 408, 1427, 326, 1393, - /* 10 */ 137, 1685, 37, 35, 1483, 1835, 1685, 1701, 412, 1833, - /* 20 */ 318, 551, 1201, 551, 1223, 38, 36, 34, 33, 32, - /* 30 */ 325, 324, 109, 415, 109, 408, 1427, 1681, 1687, 446, - /* 40 */ 1215, 451, 1681, 1687, 1717, 30, 240, 1199, 554, 1526, - /* 50 */ 1784, 1526, 535, 554, 22, 551, 550, 1671, 14, 534, - /* 60 */ 37, 35, 1328, 322, 1207, 1208, 157, 335, 318, 1701, - /* 70 */ 1201, 134, 515, 365, 1781, 38, 36, 34, 33, 32, - /* 80 */ 1528, 1, 1207, 1526, 1730, 67, 550, 85, 1702, 537, - /* 90 */ 1704, 1705, 533, 58, 554, 1199, 1717, 1770, 113, 76, - /* 100 */ 371, 292, 1766, 632, 514, 551, 14, 1521, 1836, 1671, - /* 110 */ 1418, 534, 1207, 1836, 1836, 538, 369, 1270, 1271, 958, - /* 120 */ 1519, 555, 152, 465, 550, 1622, 1833, 154, 152, 2, - /* 130 */ 425, 1833, 1833, 1526, 206, 82, 1730, 290, 473, 86, - /* 140 */ 1702, 537, 1704, 1705, 533, 1362, 554, 40, 118, 1770, - /* 150 */ 1671, 632, 198, 311, 1766, 148, 1518, 962, 963, 1202, - /* 160 */ 58, 1200, 70, 328, 468, 1270, 1271, 232, 56, 462, - /* 170 */ 364, 134, 363, 1226, 197, 490, 1797, 1216, 330, 1211, - /* 180 */ 1528, 1571, 1573, 1205, 1206, 41, 1252, 1253, 1255, 1256, - /* 190 */ 1257, 1258, 1259, 530, 552, 1267, 1268, 1269, 1272, 53, - /* 200 */ 1225, 1219, 52, 1572, 1573, 26, 59, 1202, 1836, 1200, - /* 210 */ 505, 155, 552, 1267, 1268, 38, 36, 34, 33, 32, - /* 220 */ 37, 35, 153, 155, 1417, 1701, 1833, 397, 318, 67, - /* 230 */ 1201, 1205, 1206, 302, 1252, 1253, 1255, 1256, 1257, 1258, - /* 240 */ 1259, 530, 552, 1267, 1268, 1269, 1272, 511, 1717, 460, - /* 250 */ 459, 1522, 1717, 484, 458, 1199, 504, 114, 455, 1659, - /* 260 */ 535, 454, 453, 452, 1671, 1671, 14, 534, 37, 35, - /* 270 */ 1107, 1108, 1207, 166, 165, 117, 318, 155, 1201, 1292, - /* 280 */ 515, 1504, 303, 58, 301, 300, 356, 448, 58, 2, - /* 290 */ 155, 450, 1730, 503, 1836, 85, 1702, 537, 1704, 1705, - /* 300 */ 533, 1297, 554, 1199, 344, 1770, 358, 354, 152, 292, - /* 310 */ 1766, 632, 1833, 449, 115, 38, 36, 34, 33, 32, - /* 320 */ 1207, 1836, 136, 1517, 1407, 1270, 1271, 1502, 513, 149, - /* 330 */ 1777, 1778, 1327, 1782, 1685, 152, 27, 8, 425, 1833, - /* 340 */ 1065, 577, 576, 575, 1069, 574, 1071, 1072, 573, 1074, - /* 350 */ 570, 1396, 1080, 567, 1082, 1083, 564, 561, 81, 632, - /* 360 */ 1681, 1687, 38, 36, 34, 33, 32, 1202, 78, 1200, - /* 370 */ 1352, 554, 99, 1270, 1271, 98, 97, 96, 95, 94, - /* 380 */ 93, 92, 91, 90, 588, 1342, 38, 36, 34, 33, - /* 390 */ 32, 1205, 1206, 517, 1252, 1253, 1255, 1256, 1257, 1258, - /* 400 */ 1259, 530, 552, 1267, 1268, 1269, 1272, 497, 1350, 1351, - /* 410 */ 1353, 1354, 975, 155, 974, 1202, 99, 1200, 155, 98, - /* 420 */ 97, 96, 95, 94, 93, 92, 91, 90, 37, 35, - /* 430 */ 1273, 406, 414, 1701, 410, 410, 318, 162, 1201, 1205, - /* 440 */ 1206, 976, 1252, 1253, 1255, 1256, 1257, 1258, 1259, 530, - /* 450 */ 552, 1267, 1268, 1269, 1272, 38, 36, 34, 33, 32, - /* 460 */ 1717, 474, 65, 1199, 1836, 64, 1027, 155, 535, 975, - /* 470 */ 199, 974, 1578, 1671, 1416, 534, 37, 35, 1834, 308, - /* 480 */ 1207, 1454, 1833, 1029, 318, 1784, 1201, 592, 1576, 1498, - /* 490 */ 38, 36, 34, 33, 32, 1415, 444, 9, 976, 500, - /* 500 */ 1730, 1388, 1836, 87, 1702, 537, 1704, 1705, 533, 1780, - /* 510 */ 554, 1199, 200, 1770, 1671, 1226, 152, 1769, 1766, 632, - /* 520 */ 1833, 538, 133, 1578, 147, 1239, 69, 291, 1207, 321, - /* 530 */ 323, 1621, 588, 1270, 1271, 1671, 293, 1565, 1414, 1576, - /* 540 */ 1326, 610, 609, 608, 333, 9, 607, 606, 605, 119, - /* 550 */ 600, 599, 598, 597, 596, 595, 594, 593, 127, 589, - /* 560 */ 1239, 1395, 38, 36, 34, 33, 32, 632, 1290, 506, - /* 570 */ 501, 1183, 1184, 1227, 1335, 1202, 1413, 1200, 1671, 1412, - /* 580 */ 1225, 1270, 1271, 1387, 28, 108, 107, 106, 105, 104, - /* 590 */ 103, 102, 101, 100, 38, 36, 34, 33, 32, 1205, - /* 600 */ 1206, 1411, 1252, 1253, 1255, 1256, 1257, 1258, 1259, 530, - /* 610 */ 552, 1267, 1268, 1269, 1272, 551, 1671, 1578, 134, 1671, - /* 620 */ 1291, 1224, 331, 1202, 329, 1200, 370, 1529, 1278, 1410, - /* 630 */ 134, 457, 456, 1576, 1225, 1207, 37, 35, 1457, 1528, - /* 640 */ 1228, 1671, 1296, 1526, 318, 1503, 1201, 1205, 1206, 527, - /* 650 */ 1252, 1253, 1255, 1256, 1257, 1258, 1259, 530, 552, 1267, - /* 660 */ 1268, 1269, 1272, 11, 10, 34, 33, 32, 1784, 1671, - /* 670 */ 580, 1199, 38, 36, 34, 33, 32, 29, 316, 1285, - /* 680 */ 1286, 1287, 1288, 1289, 1293, 1294, 1295, 584, 1207, 511, - /* 690 */ 1569, 1689, 1779, 604, 602, 1409, 7, 551, 460, 459, - /* 700 */ 586, 231, 1685, 458, 551, 2, 114, 455, 372, 450, - /* 710 */ 454, 453, 452, 1406, 1578, 382, 1225, 117, 293, 125, - /* 720 */ 124, 583, 582, 581, 289, 1526, 1223, 632, 1681, 1687, - /* 730 */ 1577, 449, 1526, 390, 551, 1671, 402, 515, 586, 554, - /* 740 */ 585, 1270, 1271, 1569, 1304, 383, 135, 1254, 1405, 1404, - /* 750 */ 1290, 269, 263, 1671, 403, 1556, 115, 125, 124, 583, - /* 760 */ 582, 581, 1526, 267, 55, 1403, 551, 54, 212, 551, - /* 770 */ 551, 229, 1777, 510, 1282, 509, 551, 424, 1836, 1612, - /* 780 */ 1523, 1648, 522, 1202, 167, 1200, 373, 253, 1671, 1671, - /* 790 */ 164, 591, 154, 519, 1526, 551, 1833, 1526, 1526, 962, - /* 800 */ 963, 1254, 1291, 1511, 1526, 1671, 482, 1205, 1206, 58, - /* 810 */ 1252, 1253, 1255, 1256, 1257, 1258, 1259, 530, 552, 1267, - /* 820 */ 1268, 1269, 1272, 1526, 1296, 1513, 401, 1789, 1323, 396, - /* 830 */ 395, 394, 393, 392, 389, 388, 387, 386, 385, 381, - /* 840 */ 380, 379, 378, 377, 376, 375, 374, 84, 1402, 1509, - /* 850 */ 190, 1701, 1201, 188, 203, 1401, 1400, 1399, 1398, 29, - /* 860 */ 316, 1285, 1286, 1287, 1288, 1289, 1293, 1294, 1295, 1323, - /* 870 */ 603, 192, 194, 196, 191, 193, 195, 1199, 1717, 62, - /* 880 */ 61, 368, 529, 1254, 161, 999, 514, 1501, 1671, 551, - /* 890 */ 362, 1671, 1701, 534, 1207, 1671, 1671, 1671, 1671, 472, - /* 900 */ 548, 288, 1000, 123, 352, 48, 350, 346, 342, 158, - /* 910 */ 337, 359, 470, 216, 11, 10, 1210, 1526, 1730, 1717, - /* 920 */ 1209, 86, 1702, 537, 1704, 1705, 533, 535, 554, 551, - /* 930 */ 1444, 1770, 1671, 632, 534, 311, 1766, 148, 39, 155, - /* 940 */ 549, 1390, 1391, 579, 486, 39, 1349, 1433, 481, 39, - /* 950 */ 1408, 1484, 461, 1439, 218, 1691, 235, 1526, 1798, 1730, - /* 960 */ 551, 1437, 86, 1702, 537, 1704, 1705, 533, 242, 554, - /* 970 */ 1701, 332, 1770, 523, 121, 463, 311, 1766, 1849, 1298, - /* 980 */ 586, 122, 520, 466, 123, 498, 1260, 1804, 1526, 1202, - /* 990 */ 1156, 1200, 1693, 628, 475, 443, 1701, 1717, 223, 125, - /* 1000 */ 124, 583, 582, 581, 1718, 535, 334, 1428, 48, 244, - /* 1010 */ 1671, 559, 534, 1205, 1206, 543, 1800, 122, 123, 110, - /* 1020 */ 122, 1566, 250, 1717, 512, 1058, 3, 234, 237, 239, - /* 1030 */ 1213, 535, 5, 336, 1212, 1223, 1671, 1730, 534, 343, - /* 1040 */ 86, 1702, 537, 1704, 1705, 533, 339, 554, 1701, 262, - /* 1050 */ 1770, 1027, 1086, 298, 311, 1766, 1849, 1701, 1090, 1097, - /* 1060 */ 1095, 126, 299, 1730, 1167, 1827, 86, 1702, 537, 1704, - /* 1070 */ 1705, 533, 259, 554, 384, 1717, 1770, 1614, 163, 399, - /* 1080 */ 311, 1766, 1849, 535, 1717, 391, 398, 400, 1671, 404, - /* 1090 */ 534, 1788, 535, 1229, 405, 413, 1232, 1671, 416, 534, - /* 1100 */ 170, 417, 172, 515, 1231, 418, 1701, 1233, 419, 175, - /* 1110 */ 421, 177, 515, 422, 1230, 1730, 423, 426, 278, 1702, - /* 1120 */ 537, 1704, 1705, 533, 1730, 554, 180, 278, 1702, 537, - /* 1130 */ 1704, 1705, 533, 1717, 554, 66, 183, 445, 447, 1516, - /* 1140 */ 187, 535, 307, 1512, 1836, 89, 1671, 1701, 534, 189, - /* 1150 */ 201, 260, 476, 1836, 128, 129, 1514, 1510, 154, 1653, - /* 1160 */ 130, 131, 1833, 477, 204, 1701, 1652, 152, 483, 207, - /* 1170 */ 1228, 1833, 487, 1730, 1717, 485, 87, 1702, 537, 1704, - /* 1180 */ 1705, 533, 532, 554, 488, 480, 1770, 1671, 210, 534, - /* 1190 */ 526, 1766, 1717, 499, 494, 214, 1811, 496, 541, 1810, - /* 1200 */ 535, 217, 310, 502, 1791, 1671, 6, 534, 1801, 508, - /* 1210 */ 495, 492, 222, 493, 1730, 1323, 1701, 286, 1702, 537, - /* 1220 */ 1704, 1705, 533, 531, 554, 528, 1742, 1227, 116, 227, - /* 1230 */ 42, 524, 1730, 142, 635, 138, 1702, 537, 1704, 1705, - /* 1240 */ 533, 1785, 554, 1717, 312, 226, 224, 521, 258, 18, - /* 1250 */ 1620, 535, 1619, 228, 539, 544, 1671, 1751, 534, 225, - /* 1260 */ 145, 1832, 545, 233, 246, 546, 626, 622, 618, 614, - /* 1270 */ 257, 540, 1527, 320, 248, 261, 75, 1701, 77, 557, - /* 1280 */ 516, 1850, 518, 1730, 525, 1852, 87, 1702, 537, 1704, - /* 1290 */ 1705, 533, 236, 554, 1701, 83, 1770, 1570, 251, 238, - /* 1300 */ 264, 1767, 1499, 631, 1717, 256, 141, 511, 270, 49, - /* 1310 */ 279, 271, 535, 266, 268, 1665, 1664, 1671, 60, 534, - /* 1320 */ 1663, 1717, 338, 1660, 340, 341, 1194, 1195, 159, 535, - /* 1330 */ 491, 547, 345, 1658, 1671, 117, 534, 347, 348, 349, - /* 1340 */ 1657, 351, 1656, 353, 1730, 1701, 355, 287, 1702, 537, - /* 1350 */ 1704, 1705, 533, 1655, 554, 515, 1654, 357, 1638, 489, - /* 1360 */ 360, 1730, 208, 361, 282, 1702, 537, 1704, 1705, 533, - /* 1370 */ 160, 554, 1717, 511, 115, 1170, 1632, 1169, 1631, 366, - /* 1380 */ 535, 1175, 367, 202, 1630, 1671, 1629, 534, 1142, 229, - /* 1390 */ 1777, 510, 1607, 509, 63, 1606, 1836, 1605, 1604, 1701, - /* 1400 */ 1603, 117, 1602, 507, 1601, 1600, 1599, 1598, 1701, 1597, - /* 1410 */ 152, 1596, 1730, 1595, 1833, 138, 1702, 537, 1704, 1705, - /* 1420 */ 533, 1594, 554, 1593, 1592, 1591, 1717, 1590, 1589, 120, - /* 1430 */ 1588, 1587, 1586, 1585, 535, 1717, 1144, 1582, 1581, 1671, - /* 1440 */ 115, 534, 1584, 532, 1583, 1580, 1579, 1456, 1671, 1424, - /* 1450 */ 534, 146, 315, 111, 965, 150, 1777, 1778, 168, 1782, - /* 1460 */ 1423, 1851, 1701, 409, 964, 1646, 1730, 169, 1640, 287, - /* 1470 */ 1702, 537, 1704, 1705, 533, 1730, 554, 511, 286, 1702, - /* 1480 */ 537, 1704, 1705, 533, 1628, 554, 176, 1743, 112, 1717, - /* 1490 */ 1627, 174, 411, 1617, 179, 1505, 1455, 535, 993, 1453, - /* 1500 */ 185, 429, 1671, 427, 534, 117, 428, 1451, 433, 1449, - /* 1510 */ 431, 432, 140, 1701, 437, 317, 435, 436, 442, 438, - /* 1520 */ 434, 430, 184, 1447, 440, 439, 441, 1701, 1436, 1730, - /* 1530 */ 1435, 1422, 287, 1702, 537, 1704, 1705, 533, 1101, 554, - /* 1540 */ 1717, 1507, 47, 186, 115, 1506, 1100, 68, 535, 1026, - /* 1550 */ 182, 1025, 1024, 1671, 1717, 534, 601, 1023, 603, 151, - /* 1560 */ 1777, 1778, 535, 1782, 1445, 1020, 319, 1671, 1019, 534, - /* 1570 */ 304, 1440, 1018, 305, 464, 1438, 306, 1421, 469, 467, - /* 1580 */ 1730, 1420, 1701, 287, 1702, 537, 1704, 1705, 533, 471, - /* 1590 */ 554, 1701, 88, 1645, 1730, 1639, 1177, 272, 1702, 537, - /* 1600 */ 1704, 1705, 533, 1626, 554, 1701, 51, 478, 132, 1717, - /* 1610 */ 181, 1625, 173, 1624, 178, 205, 420, 535, 1717, 1616, - /* 1620 */ 71, 479, 1671, 50, 534, 209, 535, 4, 211, 1364, - /* 1630 */ 39, 1671, 1717, 534, 23, 171, 221, 15, 213, 45, - /* 1640 */ 535, 215, 1348, 139, 219, 1671, 24, 534, 1341, 1730, - /* 1650 */ 220, 1691, 273, 1702, 537, 1704, 1705, 533, 1730, 554, - /* 1660 */ 72, 274, 1702, 537, 1704, 1705, 533, 25, 554, 1320, - /* 1670 */ 44, 230, 1730, 1319, 16, 281, 1702, 537, 1704, 1705, - /* 1680 */ 533, 1701, 554, 43, 143, 17, 1381, 1370, 1376, 1375, - /* 1690 */ 313, 13, 1380, 1379, 314, 10, 1283, 19, 1701, 144, - /* 1700 */ 1262, 31, 1247, 1261, 12, 20, 156, 1701, 1717, 21, - /* 1710 */ 1615, 243, 536, 241, 1346, 245, 535, 73, 247, 542, - /* 1720 */ 74, 1671, 1690, 534, 78, 1717, 249, 252, 1217, 1733, - /* 1730 */ 556, 1264, 553, 535, 1717, 46, 558, 327, 1671, 1087, - /* 1740 */ 534, 560, 535, 1084, 562, 563, 565, 1671, 1730, 534, - /* 1750 */ 1081, 283, 1702, 537, 1704, 1705, 533, 566, 554, 1701, - /* 1760 */ 568, 1075, 571, 569, 1079, 1730, 1073, 572, 275, 1702, - /* 1770 */ 537, 1704, 1705, 533, 1730, 554, 1078, 284, 1702, 537, - /* 1780 */ 1704, 1705, 533, 1701, 554, 1064, 1717, 578, 79, 80, - /* 1790 */ 1077, 1096, 1076, 57, 535, 254, 1092, 991, 587, 1671, - /* 1800 */ 1701, 534, 1015, 1033, 590, 255, 1013, 1012, 1011, 1010, - /* 1810 */ 1717, 1008, 1009, 1007, 1006, 1030, 1028, 1003, 535, 1002, - /* 1820 */ 1001, 1452, 998, 1671, 997, 534, 1730, 1717, 996, 276, - /* 1830 */ 1702, 537, 1704, 1705, 533, 535, 554, 611, 612, 613, - /* 1840 */ 1671, 1450, 534, 615, 616, 617, 1448, 621, 619, 620, - /* 1850 */ 1730, 1701, 1446, 285, 1702, 537, 1704, 1705, 533, 623, - /* 1860 */ 554, 624, 1701, 625, 1434, 627, 1419, 1730, 634, 629, - /* 1870 */ 277, 1702, 537, 1704, 1705, 533, 630, 554, 1717, 1203, - /* 1880 */ 265, 633, 1394, 1394, 1394, 1394, 535, 1394, 1394, 1717, - /* 1890 */ 1394, 1671, 1394, 534, 1394, 1394, 1394, 535, 1394, 1394, - /* 1900 */ 1394, 1394, 1671, 1701, 534, 1394, 1394, 1394, 1394, 1394, - /* 1910 */ 1394, 1394, 1394, 1394, 1701, 1394, 1394, 1394, 1730, 1394, - /* 1920 */ 1394, 1713, 1702, 537, 1704, 1705, 533, 1394, 554, 1730, - /* 1930 */ 1717, 1394, 1712, 1702, 537, 1704, 1705, 533, 535, 554, - /* 1940 */ 1394, 1717, 1394, 1671, 1394, 534, 1394, 1394, 1394, 535, - /* 1950 */ 1394, 1394, 1394, 1394, 1671, 1394, 534, 1394, 1394, 1394, - /* 1960 */ 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1701, 1394, 1394, - /* 1970 */ 1730, 1394, 1394, 1711, 1702, 537, 1704, 1705, 533, 1701, - /* 1980 */ 554, 1730, 1394, 1394, 296, 1702, 537, 1704, 1705, 533, - /* 1990 */ 1394, 554, 1394, 1701, 1717, 1394, 1394, 1394, 1394, 1394, - /* 2000 */ 1394, 1394, 535, 1394, 1394, 1394, 1717, 1671, 1394, 534, - /* 2010 */ 1394, 1394, 1394, 1394, 535, 1394, 1394, 1394, 1394, 1671, - /* 2020 */ 1717, 534, 1394, 1394, 1394, 1394, 1394, 1394, 535, 1394, - /* 2030 */ 1394, 1394, 1394, 1671, 1730, 534, 1394, 295, 1702, 537, - /* 2040 */ 1704, 1705, 533, 1394, 554, 1701, 1730, 1394, 1394, 297, - /* 2050 */ 1702, 537, 1704, 1705, 533, 1394, 554, 1394, 1394, 1394, - /* 2060 */ 1730, 1394, 1394, 294, 1702, 537, 1704, 1705, 533, 1394, - /* 2070 */ 554, 1394, 1717, 1394, 1394, 1394, 1394, 1394, 1394, 1394, - /* 2080 */ 535, 1394, 1394, 1394, 1394, 1671, 1394, 534, 1394, 1394, - /* 2090 */ 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, - /* 2100 */ 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, 1394, - /* 2110 */ 1394, 1394, 1730, 1394, 1394, 280, 1702, 537, 1704, 1705, - /* 2120 */ 533, 1394, 554, + /* 0 */ 78, 415, 1726, 416, 1449, 30, 239, 423, 61, 416, + /* 10 */ 1449, 1864, 39, 37, 1864, 40, 38, 36, 35, 34, + /* 20 */ 321, 1541, 1217, 333, 69, 1863, 1593, 1595, 155, 1861, + /* 30 */ 1744, 1713, 1861, 1600, 312, 1215, 558, 115, 522, 519, + /* 40 */ 326, 26, 1710, 1696, 971, 542, 1543, 33, 32, 1598, + /* 50 */ 28, 40, 38, 36, 35, 34, 33, 32, 1744, 14, + /* 60 */ 40, 38, 36, 35, 34, 1223, 512, 119, 1706, 1712, + /* 70 */ 1757, 368, 1726, 88, 1727, 545, 1729, 1730, 541, 562, + /* 80 */ 562, 84, 1, 1797, 975, 976, 60, 314, 1793, 150, + /* 90 */ 1684, 33, 32, 1864, 120, 40, 38, 36, 35, 34, + /* 100 */ 1744, 231, 1540, 511, 641, 432, 117, 1862, 543, 498, + /* 110 */ 1825, 1861, 1864, 1696, 558, 542, 149, 60, 1288, 1289, + /* 120 */ 521, 151, 1805, 1806, 559, 1810, 154, 559, 523, 1587, + /* 130 */ 1861, 39, 37, 1347, 1726, 159, 1217, 347, 372, 321, + /* 140 */ 1757, 1217, 296, 87, 1727, 545, 1729, 1730, 541, 1215, + /* 150 */ 562, 513, 1548, 1797, 1215, 1548, 508, 295, 1793, 467, + /* 160 */ 466, 1218, 1744, 1216, 465, 1242, 1255, 116, 462, 1864, + /* 170 */ 522, 461, 460, 459, 1308, 1696, 43, 542, 14, 1223, + /* 180 */ 1241, 11, 10, 156, 1223, 1221, 1222, 1861, 1270, 1271, + /* 190 */ 1273, 1274, 1275, 1276, 1277, 538, 560, 1285, 1286, 1287, + /* 200 */ 1290, 2, 1757, 1240, 305, 88, 1727, 545, 1729, 1730, + /* 210 */ 541, 186, 562, 157, 546, 1797, 374, 157, 641, 314, + /* 220 */ 1793, 150, 324, 641, 1644, 142, 1309, 1371, 514, 509, + /* 230 */ 449, 445, 441, 437, 185, 1354, 1223, 1288, 1289, 33, + /* 240 */ 32, 1241, 1826, 40, 38, 36, 35, 34, 157, 1314, + /* 250 */ 39, 37, 306, 292, 304, 303, 596, 455, 321, 70, + /* 260 */ 1217, 457, 183, 1322, 505, 1369, 1370, 1372, 1373, 1594, + /* 270 */ 1595, 39, 37, 1215, 22, 1218, 599, 1216, 519, 321, + /* 280 */ 1218, 1217, 1216, 456, 29, 319, 1303, 1304, 1305, 1306, + /* 290 */ 1307, 1311, 1312, 1313, 1215, 1242, 1296, 14, 414, 1221, + /* 300 */ 1222, 418, 1241, 1223, 1221, 1222, 119, 1270, 1271, 1273, + /* 310 */ 1274, 1275, 1276, 1277, 538, 560, 1285, 1286, 1287, 1290, + /* 320 */ 2, 519, 182, 175, 1223, 180, 523, 101, 1539, 428, + /* 330 */ 100, 99, 98, 97, 96, 95, 94, 93, 92, 1710, + /* 340 */ 139, 8, 641, 1726, 1505, 117, 33, 32, 173, 119, + /* 350 */ 40, 38, 36, 35, 34, 481, 1288, 1289, 1714, 588, + /* 360 */ 228, 1805, 518, 641, 517, 1706, 1712, 1864, 479, 1710, + /* 370 */ 477, 1744, 988, 138, 987, 1426, 562, 1288, 1289, 543, + /* 380 */ 600, 156, 1520, 420, 1696, 1861, 542, 451, 117, 1239, + /* 390 */ 39, 37, 1291, 1120, 1121, 1706, 1712, 499, 321, 1218, + /* 400 */ 1217, 1216, 989, 152, 1805, 1806, 562, 1810, 1526, 422, + /* 410 */ 1272, 1757, 418, 1215, 284, 1727, 545, 1729, 1730, 541, + /* 420 */ 1218, 562, 1216, 1221, 1222, 1537, 1270, 1271, 1273, 1274, + /* 430 */ 1275, 1276, 1277, 538, 560, 1285, 1286, 1287, 1290, 519, + /* 440 */ 1437, 558, 594, 1223, 1221, 1222, 69, 1270, 1271, 1273, + /* 450 */ 1274, 1275, 1276, 1277, 538, 560, 1285, 1286, 1287, 1290, + /* 460 */ 9, 127, 126, 591, 590, 589, 432, 119, 1544, 33, + /* 470 */ 32, 1272, 157, 40, 38, 36, 35, 34, 988, 200, + /* 480 */ 987, 1696, 641, 1436, 33, 32, 135, 523, 40, 38, + /* 490 */ 36, 35, 34, 359, 33, 32, 1288, 1289, 40, 38, + /* 500 */ 36, 35, 34, 42, 60, 1812, 117, 559, 989, 39, + /* 510 */ 37, 464, 463, 361, 357, 559, 559, 321, 373, 1217, + /* 520 */ 296, 228, 1805, 518, 1696, 517, 111, 111, 1864, 1809, + /* 530 */ 39, 37, 1215, 453, 458, 1548, 71, 294, 321, 1218, + /* 540 */ 1217, 1216, 154, 1548, 1548, 1243, 1861, 36, 35, 34, + /* 550 */ 33, 32, 1308, 1215, 40, 38, 36, 35, 34, 1381, + /* 560 */ 612, 610, 1223, 1221, 1222, 1415, 1270, 1271, 1273, 1274, + /* 570 */ 1275, 1276, 1277, 538, 560, 1285, 1286, 1287, 1290, 9, + /* 580 */ 1524, 490, 367, 1223, 366, 405, 101, 1533, 1636, 100, + /* 590 */ 99, 98, 97, 96, 95, 94, 93, 92, 1600, 166, + /* 600 */ 2, 641, 33, 32, 1309, 332, 40, 38, 36, 35, + /* 610 */ 34, 325, 559, 331, 1598, 1288, 1289, 1414, 1435, 136, + /* 620 */ 1427, 136, 641, 375, 1455, 1361, 164, 1314, 1550, 208, + /* 630 */ 1550, 168, 167, 1434, 157, 157, 1288, 1289, 596, 201, + /* 640 */ 1548, 110, 109, 108, 107, 106, 105, 104, 103, 102, + /* 650 */ 559, 67, 1812, 7, 66, 60, 519, 73, 1218, 1696, + /* 660 */ 1216, 390, 29, 319, 1303, 1304, 1305, 1306, 1307, 1311, + /* 670 */ 1312, 1313, 636, 230, 1696, 1310, 1808, 1255, 1548, 1218, + /* 680 */ 1433, 1216, 1221, 1222, 119, 1270, 1271, 1273, 1274, 1275, + /* 690 */ 1276, 1277, 538, 560, 1285, 1286, 1287, 1290, 1315, 1199, + /* 700 */ 1200, 1535, 497, 1221, 1222, 1476, 1270, 1271, 1273, 1274, + /* 710 */ 1275, 1276, 1277, 538, 560, 1285, 1286, 1287, 1290, 334, + /* 720 */ 136, 1696, 291, 117, 1239, 137, 1040, 136, 1812, 1551, + /* 730 */ 269, 398, 592, 27, 410, 1591, 1550, 58, 153, 1805, + /* 740 */ 1806, 457, 1810, 1864, 267, 57, 1600, 1432, 56, 1479, + /* 750 */ 546, 383, 1807, 411, 1412, 385, 1042, 154, 1241, 559, + /* 760 */ 1645, 1861, 1599, 456, 169, 527, 618, 617, 616, 336, + /* 770 */ 391, 615, 614, 613, 121, 608, 607, 606, 605, 604, + /* 780 */ 603, 602, 601, 129, 597, 376, 157, 1548, 1696, 60, + /* 790 */ 1078, 585, 584, 583, 1082, 582, 1084, 1085, 581, 1087, + /* 800 */ 578, 1525, 1093, 575, 1095, 1096, 572, 569, 1523, 593, + /* 810 */ 467, 466, 1591, 338, 483, 465, 1531, 263, 116, 462, + /* 820 */ 1578, 204, 461, 460, 459, 409, 1342, 86, 404, 403, + /* 830 */ 402, 401, 400, 397, 396, 395, 394, 393, 389, 388, + /* 840 */ 387, 386, 380, 379, 378, 377, 975, 976, 559, 1244, + /* 850 */ 537, 1431, 1430, 559, 1864, 1864, 1429, 328, 327, 431, + /* 860 */ 64, 63, 371, 1428, 1545, 163, 559, 1231, 154, 154, + /* 870 */ 1726, 365, 1861, 1861, 1346, 559, 1548, 1673, 1425, 1466, + /* 880 */ 1224, 1548, 290, 1817, 1342, 355, 556, 353, 349, 345, + /* 890 */ 160, 340, 1696, 1696, 1548, 594, 1424, 1696, 1744, 1423, + /* 900 */ 530, 468, 594, 1548, 1696, 191, 543, 1506, 189, 1422, + /* 910 */ 1223, 1696, 1421, 542, 127, 126, 591, 590, 589, 1696, + /* 920 */ 157, 127, 126, 591, 590, 589, 523, 1272, 1420, 1726, + /* 930 */ 1419, 207, 1418, 1417, 52, 494, 193, 1696, 1757, 192, + /* 940 */ 1696, 87, 1727, 545, 1729, 1730, 541, 472, 562, 563, + /* 950 */ 1696, 1797, 559, 1696, 528, 295, 1793, 1744, 195, 72, + /* 960 */ 535, 194, 482, 491, 525, 543, 1461, 1864, 559, 1696, + /* 970 */ 1696, 1696, 542, 1696, 1696, 1713, 199, 211, 329, 495, + /* 980 */ 1548, 154, 11, 10, 197, 1861, 1710, 196, 470, 1226, + /* 990 */ 475, 611, 1726, 587, 469, 1459, 1548, 1757, 559, 198, + /* 1000 */ 88, 1727, 545, 1729, 1730, 541, 1232, 562, 1227, 557, + /* 1010 */ 1797, 559, 1706, 1712, 314, 1793, 1877, 473, 50, 215, + /* 1020 */ 1744, 41, 252, 562, 55, 1832, 1548, 54, 543, 559, + /* 1030 */ 1235, 362, 41, 1696, 83, 542, 1409, 1410, 1716, 1548, + /* 1040 */ 335, 560, 1285, 1286, 80, 41, 1225, 241, 506, 234, + /* 1050 */ 123, 450, 124, 1726, 222, 125, 1745, 1548, 1368, 217, + /* 1060 */ 1757, 1316, 50, 88, 1727, 545, 1729, 1730, 541, 1012, + /* 1070 */ 562, 484, 1278, 1797, 1718, 1726, 337, 314, 1793, 1877, + /* 1080 */ 567, 1744, 124, 125, 1345, 1170, 1300, 243, 1855, 543, + /* 1090 */ 551, 531, 249, 1450, 1696, 1071, 542, 112, 1588, 1013, + /* 1100 */ 1828, 124, 262, 1744, 520, 233, 236, 238, 3, 5, + /* 1110 */ 339, 543, 1239, 342, 346, 301, 1696, 1229, 542, 1040, + /* 1120 */ 1099, 1757, 1103, 1110, 89, 1727, 545, 1729, 1730, 541, + /* 1130 */ 302, 562, 1183, 392, 1797, 259, 1726, 1108, 1796, 1793, + /* 1140 */ 1638, 128, 165, 1757, 399, 406, 88, 1727, 545, 1729, + /* 1150 */ 1730, 541, 408, 562, 412, 1245, 1797, 407, 413, 421, + /* 1160 */ 314, 1793, 1877, 1248, 1744, 424, 172, 425, 1247, 426, + /* 1170 */ 174, 1816, 543, 1249, 1228, 427, 177, 1696, 429, 542, + /* 1180 */ 179, 1246, 430, 181, 68, 433, 184, 452, 454, 1726, + /* 1190 */ 1538, 188, 523, 1678, 1534, 190, 293, 130, 131, 91, + /* 1200 */ 1677, 1536, 260, 1726, 1757, 485, 1532, 276, 1727, 545, + /* 1210 */ 1729, 1730, 541, 492, 562, 132, 133, 1744, 202, 489, + /* 1220 */ 205, 209, 486, 496, 502, 543, 1244, 507, 6, 516, + /* 1230 */ 1696, 1744, 542, 1864, 311, 549, 213, 1839, 504, 543, + /* 1240 */ 1838, 313, 144, 510, 1696, 523, 542, 156, 216, 1819, + /* 1250 */ 1829, 1861, 221, 503, 224, 1726, 501, 1757, 500, 226, + /* 1260 */ 276, 1727, 545, 1729, 1730, 541, 118, 562, 1342, 1243, + /* 1270 */ 44, 1757, 529, 1726, 89, 1727, 545, 1729, 1730, 541, + /* 1280 */ 532, 562, 225, 1744, 1797, 1813, 1864, 315, 534, 1793, + /* 1290 */ 18, 540, 227, 1643, 223, 1860, 1696, 1642, 542, 232, + /* 1300 */ 154, 1744, 1778, 552, 1861, 547, 548, 553, 323, 543, + /* 1310 */ 245, 79, 554, 1549, 1696, 261, 542, 1880, 247, 77, + /* 1320 */ 526, 533, 235, 1757, 565, 1726, 283, 1727, 545, 1729, + /* 1330 */ 1730, 541, 539, 562, 536, 1769, 1592, 237, 1521, 255, + /* 1340 */ 264, 1757, 1726, 637, 140, 1727, 545, 1729, 1730, 541, + /* 1350 */ 640, 562, 638, 1744, 266, 51, 143, 285, 287, 286, + /* 1360 */ 268, 543, 1690, 1689, 62, 1688, 1696, 341, 542, 1685, + /* 1370 */ 1744, 343, 344, 1210, 1211, 161, 348, 1683, 543, 350, + /* 1380 */ 351, 352, 1682, 1696, 354, 542, 1681, 356, 1680, 524, + /* 1390 */ 1878, 1726, 358, 1757, 1679, 360, 89, 1727, 545, 1729, + /* 1400 */ 1730, 541, 1663, 562, 363, 162, 1797, 364, 1186, 1185, + /* 1410 */ 1757, 1794, 1657, 279, 1727, 545, 1729, 1730, 541, 1744, + /* 1420 */ 562, 1656, 369, 370, 1655, 1654, 1157, 543, 1631, 65, + /* 1430 */ 1630, 1629, 1696, 1628, 542, 1627, 1626, 381, 382, 1625, + /* 1440 */ 384, 1624, 1623, 1622, 1621, 1620, 1619, 1618, 1617, 1616, + /* 1450 */ 1615, 1726, 515, 1614, 1613, 1612, 1611, 122, 1610, 1757, + /* 1460 */ 1726, 170, 140, 1727, 545, 1729, 1730, 541, 113, 562, + /* 1470 */ 1609, 1608, 1607, 1606, 1605, 1604, 33, 32, 1603, 1744, + /* 1480 */ 40, 38, 36, 35, 34, 1159, 1602, 543, 1744, 1601, + /* 1490 */ 1478, 1446, 1696, 978, 542, 977, 540, 417, 1445, 171, + /* 1500 */ 148, 1696, 1671, 542, 114, 318, 419, 1665, 1879, 1726, + /* 1510 */ 178, 1652, 1640, 1527, 1477, 176, 1475, 1473, 434, 1757, + /* 1520 */ 1653, 1726, 284, 1727, 545, 1729, 1730, 541, 1757, 562, + /* 1530 */ 438, 283, 1727, 545, 1729, 1730, 541, 1744, 562, 435, + /* 1540 */ 1770, 439, 1006, 436, 440, 543, 1471, 443, 444, 1744, + /* 1550 */ 1696, 1469, 542, 442, 446, 447, 1458, 543, 1457, 1442, + /* 1560 */ 448, 1529, 1696, 320, 542, 1114, 1113, 1528, 49, 187, + /* 1570 */ 1039, 1038, 644, 1037, 1726, 322, 1036, 1757, 1033, 1467, + /* 1580 */ 284, 1727, 545, 1729, 1730, 541, 258, 562, 609, 1757, + /* 1590 */ 1726, 611, 284, 1727, 545, 1729, 1730, 541, 1032, 562, + /* 1600 */ 147, 1031, 1744, 307, 1462, 634, 630, 626, 622, 256, + /* 1610 */ 543, 308, 1460, 471, 309, 1696, 1441, 542, 1744, 474, + /* 1620 */ 476, 1407, 1440, 478, 1439, 480, 543, 90, 1670, 1193, + /* 1630 */ 1664, 1696, 53, 542, 85, 487, 134, 250, 493, 1726, + /* 1640 */ 1651, 488, 1757, 1649, 206, 270, 1727, 545, 1729, 1730, + /* 1650 */ 541, 310, 562, 1650, 1648, 1647, 23, 4, 1757, 1383, + /* 1660 */ 41, 271, 1727, 545, 1729, 1730, 541, 1744, 562, 214, + /* 1670 */ 555, 220, 212, 141, 47, 543, 1367, 218, 24, 15, + /* 1680 */ 1696, 1360, 542, 1716, 219, 25, 10, 74, 46, 1339, + /* 1690 */ 229, 1338, 145, 1400, 17, 1395, 1394, 316, 45, 1726, + /* 1700 */ 210, 1389, 16, 1406, 1399, 1398, 317, 1757, 1726, 1301, + /* 1710 */ 272, 1727, 545, 1729, 1730, 541, 19, 562, 13, 1280, + /* 1720 */ 146, 1191, 1279, 203, 158, 1263, 544, 1744, 550, 31, + /* 1730 */ 12, 20, 1639, 246, 21, 543, 1744, 1715, 251, 240, + /* 1740 */ 1696, 1365, 542, 248, 543, 242, 1233, 244, 75, 1696, + /* 1750 */ 76, 542, 564, 566, 1760, 330, 570, 1726, 1282, 80, + /* 1760 */ 1100, 561, 48, 568, 573, 1097, 1094, 1757, 571, 1726, + /* 1770 */ 278, 1727, 545, 1729, 1730, 541, 1757, 562, 574, 280, + /* 1780 */ 1727, 545, 1729, 1730, 541, 1744, 562, 1088, 576, 577, + /* 1790 */ 579, 1086, 580, 543, 1092, 1091, 1090, 1744, 1696, 1089, + /* 1800 */ 542, 1077, 1109, 586, 81, 543, 253, 82, 59, 1105, + /* 1810 */ 1696, 1004, 542, 595, 1046, 1028, 598, 254, 1026, 1025, + /* 1820 */ 1726, 1024, 1021, 1023, 1022, 1757, 1020, 1019, 273, 1727, + /* 1830 */ 545, 1729, 1730, 541, 1726, 562, 1043, 1757, 1041, 1016, + /* 1840 */ 281, 1727, 545, 1729, 1730, 541, 1015, 562, 1744, 1014, + /* 1850 */ 1011, 1010, 1009, 1474, 619, 620, 543, 621, 1472, 623, + /* 1860 */ 624, 1696, 1744, 542, 625, 1470, 627, 628, 629, 1468, + /* 1870 */ 543, 631, 633, 632, 1456, 1696, 635, 542, 968, 1438, + /* 1880 */ 643, 257, 639, 1413, 1219, 1726, 265, 642, 1757, 1413, + /* 1890 */ 1413, 274, 1727, 545, 1729, 1730, 541, 1726, 562, 1413, + /* 1900 */ 1413, 1413, 1757, 1413, 1413, 282, 1727, 545, 1729, 1730, + /* 1910 */ 541, 1726, 562, 1744, 1413, 1413, 1413, 1413, 1413, 1413, + /* 1920 */ 1413, 543, 1413, 1413, 1413, 1744, 1696, 1413, 542, 1413, + /* 1930 */ 1413, 1413, 1413, 543, 1413, 1413, 1413, 1413, 1696, 1744, + /* 1940 */ 542, 1413, 1413, 1413, 1413, 1413, 1413, 543, 1413, 1413, + /* 1950 */ 1413, 1413, 1696, 1757, 542, 1413, 275, 1727, 545, 1729, + /* 1960 */ 1730, 541, 1726, 562, 1413, 1757, 1413, 1413, 288, 1727, + /* 1970 */ 545, 1729, 1730, 541, 1413, 562, 1413, 1413, 1413, 1757, + /* 1980 */ 1413, 1413, 289, 1727, 545, 1729, 1730, 541, 1726, 562, + /* 1990 */ 1744, 1413, 1413, 1413, 1413, 1413, 1413, 1413, 543, 1413, + /* 2000 */ 1413, 1413, 1413, 1696, 1413, 542, 1413, 1413, 1413, 1413, + /* 2010 */ 1413, 1413, 1413, 1413, 1413, 1413, 1744, 1413, 1413, 1413, + /* 2020 */ 1413, 1413, 1413, 1413, 543, 1413, 1413, 1413, 1413, 1696, + /* 2030 */ 1757, 542, 1413, 1738, 1727, 545, 1729, 1730, 541, 1413, + /* 2040 */ 562, 1726, 1413, 1413, 1413, 1413, 1413, 1413, 1413, 1413, + /* 2050 */ 1413, 1413, 1413, 1726, 1413, 1413, 1757, 1413, 1413, 1737, + /* 2060 */ 1727, 545, 1729, 1730, 541, 1413, 562, 1413, 1413, 1744, + /* 2070 */ 1413, 1413, 1413, 1413, 1413, 1413, 1413, 543, 1413, 1413, + /* 2080 */ 1413, 1744, 1696, 1413, 542, 1413, 1413, 1413, 1413, 543, + /* 2090 */ 1413, 1413, 1413, 1413, 1696, 1413, 542, 1413, 1413, 1413, + /* 2100 */ 1413, 1413, 1413, 1413, 1726, 1413, 1413, 1413, 1413, 1757, + /* 2110 */ 1413, 1413, 1736, 1727, 545, 1729, 1730, 541, 1726, 562, + /* 2120 */ 1413, 1757, 1413, 1413, 299, 1727, 545, 1729, 1730, 541, + /* 2130 */ 1413, 562, 1744, 1413, 1413, 1413, 1413, 1413, 1413, 1413, + /* 2140 */ 543, 1413, 1413, 1413, 1413, 1696, 1744, 542, 1413, 1413, + /* 2150 */ 1413, 1413, 1413, 1413, 543, 1413, 1413, 1413, 1413, 1696, + /* 2160 */ 1413, 542, 1413, 1413, 1413, 1413, 1413, 1413, 1413, 1726, + /* 2170 */ 1413, 1413, 1757, 1413, 1413, 298, 1727, 545, 1729, 1730, + /* 2180 */ 541, 1726, 562, 1413, 1413, 1413, 1757, 1413, 1413, 300, + /* 2190 */ 1727, 545, 1729, 1730, 541, 1413, 562, 1744, 1413, 1413, + /* 2200 */ 1413, 1413, 1413, 1413, 1413, 543, 1413, 1413, 1413, 1744, + /* 2210 */ 1696, 1413, 542, 1413, 1413, 1413, 1413, 543, 1413, 1413, + /* 2220 */ 1413, 1413, 1696, 1413, 542, 1413, 1413, 1413, 1413, 1413, + /* 2230 */ 1413, 1413, 1413, 1413, 1413, 1413, 1413, 1757, 1413, 1413, + /* 2240 */ 297, 1727, 545, 1729, 1730, 541, 1413, 562, 1413, 1757, + /* 2250 */ 1413, 1413, 277, 1727, 545, 1729, 1730, 541, 1413, 562, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 274, 341, 273, 277, 248, 274, 250, 251, 277, 242, - /* 10 */ 257, 285, 12, 13, 261, 355, 285, 245, 14, 359, - /* 20 */ 20, 252, 22, 252, 20, 12, 13, 14, 15, 16, - /* 30 */ 12, 13, 263, 248, 263, 250, 251, 311, 312, 270, - /* 40 */ 22, 270, 311, 312, 272, 326, 327, 47, 322, 280, - /* 50 */ 313, 280, 280, 322, 41, 252, 20, 285, 58, 287, - /* 60 */ 12, 13, 14, 264, 64, 47, 263, 300, 20, 245, - /* 70 */ 22, 272, 300, 300, 337, 12, 13, 14, 15, 16, - /* 80 */ 281, 81, 64, 280, 312, 256, 20, 315, 316, 317, - /* 90 */ 318, 319, 320, 81, 322, 47, 272, 325, 269, 254, - /* 100 */ 252, 329, 330, 103, 280, 252, 58, 278, 341, 285, - /* 110 */ 245, 287, 64, 341, 341, 287, 263, 117, 118, 4, - /* 120 */ 275, 103, 355, 4, 20, 297, 359, 355, 355, 81, - /* 130 */ 57, 359, 359, 280, 55, 254, 312, 289, 19, 315, - /* 140 */ 316, 317, 318, 319, 320, 82, 322, 81, 267, 325, - /* 150 */ 285, 103, 33, 329, 330, 331, 275, 42, 43, 159, - /* 160 */ 81, 161, 83, 264, 45, 117, 118, 343, 3, 50, - /* 170 */ 158, 272, 160, 20, 55, 351, 352, 159, 283, 161, - /* 180 */ 281, 286, 287, 183, 184, 81, 186, 187, 188, 189, - /* 190 */ 190, 191, 192, 193, 194, 195, 196, 197, 198, 80, - /* 200 */ 20, 183, 83, 286, 287, 2, 4, 159, 341, 161, - /* 210 */ 20, 211, 194, 195, 196, 12, 13, 14, 15, 16, - /* 220 */ 12, 13, 355, 211, 245, 245, 359, 75, 20, 256, - /* 230 */ 22, 183, 184, 35, 186, 187, 188, 189, 190, 191, - /* 240 */ 192, 193, 194, 195, 196, 197, 198, 252, 272, 60, - /* 250 */ 61, 278, 272, 300, 65, 47, 280, 68, 69, 0, - /* 260 */ 280, 72, 73, 74, 285, 285, 58, 287, 12, 13, - /* 270 */ 117, 118, 64, 121, 122, 280, 20, 211, 22, 142, - /* 280 */ 300, 0, 84, 81, 86, 87, 154, 89, 81, 81, - /* 290 */ 211, 93, 312, 317, 341, 315, 316, 317, 318, 319, - /* 300 */ 320, 164, 322, 47, 45, 325, 174, 175, 355, 329, - /* 310 */ 330, 103, 359, 115, 319, 12, 13, 14, 15, 16, - /* 320 */ 64, 341, 244, 274, 246, 117, 118, 0, 333, 334, - /* 330 */ 335, 336, 4, 338, 285, 355, 199, 81, 57, 359, - /* 340 */ 94, 95, 96, 97, 98, 99, 100, 101, 102, 103, - /* 350 */ 104, 0, 106, 107, 108, 109, 110, 111, 81, 103, - /* 360 */ 311, 312, 12, 13, 14, 15, 16, 159, 91, 161, - /* 370 */ 183, 322, 21, 117, 118, 24, 25, 26, 27, 28, - /* 380 */ 29, 30, 31, 32, 57, 82, 12, 13, 14, 15, - /* 390 */ 16, 183, 184, 228, 186, 187, 188, 189, 190, 191, - /* 400 */ 192, 193, 194, 195, 196, 197, 198, 220, 221, 222, - /* 410 */ 223, 224, 20, 211, 22, 159, 21, 161, 211, 24, - /* 420 */ 25, 26, 27, 28, 29, 30, 31, 32, 12, 13, - /* 430 */ 14, 249, 249, 245, 252, 252, 20, 55, 22, 183, - /* 440 */ 184, 49, 186, 187, 188, 189, 190, 191, 192, 193, - /* 450 */ 194, 195, 196, 197, 198, 12, 13, 14, 15, 16, - /* 460 */ 272, 300, 80, 47, 341, 83, 47, 211, 280, 20, - /* 470 */ 113, 22, 272, 285, 245, 287, 12, 13, 355, 279, - /* 480 */ 64, 0, 359, 64, 20, 313, 22, 260, 288, 262, - /* 490 */ 12, 13, 14, 15, 16, 245, 47, 81, 49, 146, - /* 500 */ 312, 151, 341, 315, 316, 317, 318, 319, 320, 337, - /* 510 */ 322, 47, 112, 325, 285, 20, 355, 329, 330, 103, - /* 520 */ 359, 287, 148, 272, 271, 82, 169, 170, 64, 295, - /* 530 */ 279, 297, 57, 117, 118, 285, 58, 284, 245, 288, - /* 540 */ 212, 60, 61, 62, 63, 81, 65, 66, 67, 68, - /* 550 */ 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, - /* 560 */ 82, 0, 12, 13, 14, 15, 16, 103, 90, 216, - /* 570 */ 217, 171, 172, 20, 14, 159, 245, 161, 285, 245, - /* 580 */ 20, 117, 118, 233, 2, 24, 25, 26, 27, 28, - /* 590 */ 29, 30, 31, 32, 12, 13, 14, 15, 16, 183, - /* 600 */ 184, 245, 186, 187, 188, 189, 190, 191, 192, 193, - /* 610 */ 194, 195, 196, 197, 198, 252, 285, 272, 272, 285, - /* 620 */ 142, 20, 264, 159, 279, 161, 263, 281, 14, 245, - /* 630 */ 272, 258, 259, 288, 20, 64, 12, 13, 0, 281, - /* 640 */ 20, 285, 164, 280, 20, 0, 22, 183, 184, 58, - /* 650 */ 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - /* 660 */ 196, 197, 198, 1, 2, 14, 15, 16, 313, 285, - /* 670 */ 92, 47, 12, 13, 14, 15, 16, 199, 200, 201, - /* 680 */ 202, 203, 204, 205, 206, 207, 208, 282, 64, 252, - /* 690 */ 285, 274, 337, 258, 259, 245, 37, 252, 60, 61, - /* 700 */ 93, 148, 285, 65, 252, 81, 68, 69, 263, 93, - /* 710 */ 72, 73, 74, 245, 272, 263, 20, 280, 58, 112, - /* 720 */ 113, 114, 115, 116, 18, 280, 20, 103, 311, 312, - /* 730 */ 288, 115, 280, 27, 252, 285, 30, 300, 93, 322, - /* 740 */ 282, 117, 118, 285, 82, 263, 18, 187, 245, 245, - /* 750 */ 90, 23, 265, 285, 48, 268, 319, 112, 113, 114, - /* 760 */ 115, 116, 280, 35, 36, 245, 252, 39, 148, 252, - /* 770 */ 252, 334, 335, 336, 183, 338, 252, 263, 341, 280, - /* 780 */ 263, 263, 41, 159, 56, 161, 80, 263, 285, 285, - /* 790 */ 291, 64, 355, 41, 280, 252, 359, 280, 280, 42, - /* 800 */ 43, 187, 142, 273, 280, 285, 263, 183, 184, 81, - /* 810 */ 186, 187, 188, 189, 190, 191, 192, 193, 194, 195, - /* 820 */ 196, 197, 198, 280, 164, 273, 120, 209, 210, 123, - /* 830 */ 124, 125, 126, 127, 128, 129, 130, 131, 132, 133, - /* 840 */ 134, 135, 136, 137, 138, 139, 140, 119, 245, 273, - /* 850 */ 85, 245, 22, 88, 273, 245, 245, 245, 245, 199, - /* 860 */ 200, 201, 202, 203, 204, 205, 206, 207, 208, 210, - /* 870 */ 41, 85, 85, 85, 88, 88, 88, 47, 272, 151, - /* 880 */ 152, 153, 273, 187, 156, 47, 280, 0, 285, 252, - /* 890 */ 162, 285, 245, 287, 64, 285, 285, 285, 285, 21, - /* 900 */ 263, 173, 64, 41, 176, 41, 178, 179, 180, 181, - /* 910 */ 182, 82, 34, 41, 1, 2, 47, 280, 312, 272, - /* 920 */ 47, 315, 316, 317, 318, 319, 320, 280, 322, 252, - /* 930 */ 0, 325, 285, 103, 287, 329, 330, 331, 41, 211, - /* 940 */ 263, 117, 118, 273, 82, 41, 82, 0, 304, 41, - /* 950 */ 246, 261, 22, 0, 82, 44, 362, 280, 352, 312, - /* 960 */ 252, 0, 315, 316, 317, 318, 319, 320, 41, 322, - /* 970 */ 245, 263, 325, 232, 41, 22, 329, 330, 331, 82, - /* 980 */ 93, 41, 230, 22, 41, 353, 82, 340, 280, 159, - /* 990 */ 82, 161, 81, 46, 308, 253, 245, 272, 348, 112, - /* 1000 */ 113, 114, 115, 116, 272, 280, 253, 251, 41, 82, - /* 1010 */ 285, 41, 287, 183, 184, 82, 314, 41, 41, 41, - /* 1020 */ 41, 284, 82, 272, 339, 82, 342, 356, 356, 356, - /* 1030 */ 161, 280, 213, 310, 161, 20, 285, 312, 287, 45, - /* 1040 */ 315, 316, 317, 318, 319, 320, 252, 322, 245, 82, - /* 1050 */ 325, 47, 82, 309, 329, 330, 331, 245, 82, 82, - /* 1060 */ 82, 82, 258, 312, 157, 340, 315, 316, 317, 318, - /* 1070 */ 319, 320, 302, 322, 252, 272, 325, 252, 40, 142, - /* 1080 */ 329, 330, 331, 280, 272, 292, 290, 290, 285, 252, - /* 1090 */ 287, 340, 280, 20, 247, 247, 20, 285, 306, 287, - /* 1100 */ 256, 287, 256, 300, 20, 299, 245, 20, 301, 256, - /* 1110 */ 299, 256, 300, 280, 20, 312, 293, 252, 315, 316, - /* 1120 */ 317, 318, 319, 320, 312, 322, 256, 315, 316, 317, - /* 1130 */ 318, 319, 320, 272, 322, 256, 256, 247, 272, 272, - /* 1140 */ 272, 280, 247, 272, 341, 252, 285, 245, 287, 272, - /* 1150 */ 254, 306, 167, 341, 272, 272, 272, 272, 355, 285, - /* 1160 */ 272, 272, 359, 305, 254, 245, 285, 355, 252, 254, - /* 1170 */ 20, 359, 280, 312, 272, 299, 315, 316, 317, 318, - /* 1180 */ 319, 320, 280, 322, 293, 287, 325, 285, 254, 287, - /* 1190 */ 329, 330, 272, 219, 285, 296, 347, 285, 218, 347, - /* 1200 */ 280, 296, 285, 285, 350, 285, 225, 287, 314, 150, - /* 1210 */ 227, 214, 349, 226, 312, 210, 245, 315, 316, 317, - /* 1220 */ 318, 319, 320, 321, 322, 323, 324, 20, 280, 310, - /* 1230 */ 40, 231, 312, 347, 19, 315, 316, 317, 318, 319, - /* 1240 */ 320, 313, 322, 272, 234, 344, 346, 229, 33, 81, - /* 1250 */ 296, 280, 296, 332, 285, 145, 285, 328, 287, 345, - /* 1260 */ 45, 358, 294, 357, 280, 293, 51, 52, 53, 54, - /* 1270 */ 55, 285, 280, 285, 254, 268, 254, 245, 81, 276, - /* 1280 */ 360, 361, 358, 312, 358, 363, 315, 316, 317, 318, - /* 1290 */ 319, 320, 357, 322, 245, 80, 325, 285, 83, 357, - /* 1300 */ 252, 330, 262, 247, 272, 254, 307, 252, 266, 303, - /* 1310 */ 266, 266, 280, 255, 243, 0, 0, 285, 40, 287, - /* 1320 */ 0, 272, 72, 0, 47, 177, 47, 47, 47, 280, - /* 1330 */ 298, 116, 177, 0, 285, 280, 287, 47, 47, 177, - /* 1340 */ 0, 177, 0, 47, 312, 245, 22, 315, 316, 317, - /* 1350 */ 318, 319, 320, 0, 322, 300, 0, 47, 0, 144, - /* 1360 */ 164, 312, 147, 163, 315, 316, 317, 318, 319, 320, - /* 1370 */ 81, 322, 272, 252, 319, 161, 0, 159, 0, 155, - /* 1380 */ 280, 166, 154, 168, 0, 285, 0, 287, 44, 334, - /* 1390 */ 335, 336, 0, 338, 141, 0, 341, 0, 0, 245, - /* 1400 */ 0, 280, 0, 354, 0, 0, 0, 0, 245, 0, - /* 1410 */ 355, 0, 312, 0, 359, 315, 316, 317, 318, 319, - /* 1420 */ 320, 0, 322, 0, 0, 0, 272, 0, 0, 40, - /* 1430 */ 0, 0, 0, 0, 280, 272, 22, 0, 0, 285, - /* 1440 */ 319, 287, 0, 280, 0, 0, 0, 0, 285, 0, - /* 1450 */ 287, 41, 298, 37, 14, 334, 335, 336, 40, 338, - /* 1460 */ 0, 361, 245, 44, 14, 0, 312, 38, 0, 315, - /* 1470 */ 316, 317, 318, 319, 320, 312, 322, 252, 315, 316, - /* 1480 */ 317, 318, 319, 320, 0, 322, 150, 324, 37, 272, - /* 1490 */ 0, 37, 44, 0, 37, 0, 0, 280, 59, 0, - /* 1500 */ 33, 37, 285, 47, 287, 280, 45, 0, 37, 0, - /* 1510 */ 47, 45, 45, 245, 37, 298, 47, 45, 51, 52, - /* 1520 */ 53, 54, 55, 0, 45, 47, 37, 245, 0, 312, - /* 1530 */ 0, 0, 315, 316, 317, 318, 319, 320, 47, 322, - /* 1540 */ 272, 0, 90, 88, 319, 0, 22, 80, 280, 47, - /* 1550 */ 83, 47, 47, 285, 272, 287, 41, 47, 41, 334, - /* 1560 */ 335, 336, 280, 338, 0, 47, 298, 285, 47, 287, - /* 1570 */ 22, 0, 47, 22, 48, 0, 22, 0, 22, 47, - /* 1580 */ 312, 0, 245, 315, 316, 317, 318, 319, 320, 22, - /* 1590 */ 322, 245, 20, 0, 312, 0, 47, 315, 316, 317, - /* 1600 */ 318, 319, 320, 0, 322, 245, 148, 22, 165, 272, - /* 1610 */ 143, 0, 145, 0, 147, 145, 149, 280, 272, 0, - /* 1620 */ 81, 148, 285, 148, 287, 37, 280, 41, 143, 82, - /* 1630 */ 41, 285, 272, 287, 81, 168, 44, 215, 81, 41, - /* 1640 */ 280, 82, 82, 81, 81, 285, 81, 287, 82, 312, - /* 1650 */ 41, 44, 315, 316, 317, 318, 319, 320, 312, 322, - /* 1660 */ 81, 315, 316, 317, 318, 319, 320, 41, 322, 82, - /* 1670 */ 41, 44, 312, 82, 215, 315, 316, 317, 318, 319, - /* 1680 */ 320, 245, 322, 209, 44, 41, 82, 82, 47, 47, - /* 1690 */ 47, 215, 47, 47, 47, 2, 183, 41, 245, 44, - /* 1700 */ 82, 81, 22, 82, 81, 81, 44, 245, 272, 81, - /* 1710 */ 0, 81, 185, 82, 82, 81, 280, 81, 37, 146, - /* 1720 */ 81, 285, 44, 287, 91, 272, 143, 44, 22, 81, - /* 1730 */ 92, 82, 81, 280, 272, 81, 47, 47, 285, 82, - /* 1740 */ 287, 81, 280, 82, 47, 81, 47, 285, 312, 287, - /* 1750 */ 82, 315, 316, 317, 318, 319, 320, 81, 322, 245, - /* 1760 */ 47, 82, 47, 81, 105, 312, 82, 81, 315, 316, - /* 1770 */ 317, 318, 319, 320, 312, 322, 105, 315, 316, 317, - /* 1780 */ 318, 319, 320, 245, 322, 22, 272, 93, 81, 81, - /* 1790 */ 105, 47, 105, 81, 280, 41, 22, 59, 58, 285, - /* 1800 */ 245, 287, 47, 64, 79, 41, 47, 47, 47, 47, - /* 1810 */ 272, 22, 47, 47, 47, 64, 47, 47, 280, 47, - /* 1820 */ 47, 0, 47, 285, 47, 287, 312, 272, 47, 315, - /* 1830 */ 316, 317, 318, 319, 320, 280, 322, 47, 45, 37, - /* 1840 */ 285, 0, 287, 47, 45, 37, 0, 37, 47, 45, - /* 1850 */ 312, 245, 0, 315, 316, 317, 318, 319, 320, 47, - /* 1860 */ 322, 45, 245, 37, 0, 47, 0, 312, 20, 22, - /* 1870 */ 315, 316, 317, 318, 319, 320, 21, 322, 272, 22, - /* 1880 */ 22, 21, 364, 364, 364, 364, 280, 364, 364, 272, - /* 1890 */ 364, 285, 364, 287, 364, 364, 364, 280, 364, 364, - /* 1900 */ 364, 364, 285, 245, 287, 364, 364, 364, 364, 364, - /* 1910 */ 364, 364, 364, 364, 245, 364, 364, 364, 312, 364, - /* 1920 */ 364, 315, 316, 317, 318, 319, 320, 364, 322, 312, - /* 1930 */ 272, 364, 315, 316, 317, 318, 319, 320, 280, 322, - /* 1940 */ 364, 272, 364, 285, 364, 287, 364, 364, 364, 280, - /* 1950 */ 364, 364, 364, 364, 285, 364, 287, 364, 364, 364, - /* 1960 */ 364, 364, 364, 364, 364, 364, 364, 245, 364, 364, - /* 1970 */ 312, 364, 364, 315, 316, 317, 318, 319, 320, 245, - /* 1980 */ 322, 312, 364, 364, 315, 316, 317, 318, 319, 320, - /* 1990 */ 364, 322, 364, 245, 272, 364, 364, 364, 364, 364, - /* 2000 */ 364, 364, 280, 364, 364, 364, 272, 285, 364, 287, - /* 2010 */ 364, 364, 364, 364, 280, 364, 364, 364, 364, 285, - /* 2020 */ 272, 287, 364, 364, 364, 364, 364, 364, 280, 364, - /* 2030 */ 364, 364, 364, 285, 312, 287, 364, 315, 316, 317, - /* 2040 */ 318, 319, 320, 364, 322, 245, 312, 364, 364, 315, - /* 2050 */ 316, 317, 318, 319, 320, 364, 322, 364, 364, 364, - /* 2060 */ 312, 364, 364, 315, 316, 317, 318, 319, 320, 364, - /* 2070 */ 322, 364, 272, 364, 364, 364, 364, 364, 364, 364, - /* 2080 */ 280, 364, 364, 364, 364, 285, 364, 287, 364, 364, - /* 2090 */ 364, 364, 364, 364, 364, 364, 364, 364, 364, 364, - /* 2100 */ 364, 364, 364, 364, 364, 364, 364, 364, 364, 364, - /* 2110 */ 364, 364, 312, 364, 364, 315, 316, 317, 318, 319, - /* 2120 */ 320, 364, 322, 364, + /* 0 */ 257, 251, 247, 253, 254, 329, 330, 251, 4, 253, + /* 10 */ 254, 344, 12, 13, 344, 12, 13, 14, 15, 16, + /* 20 */ 20, 278, 22, 286, 259, 358, 289, 290, 358, 362, + /* 30 */ 275, 277, 362, 275, 280, 35, 20, 272, 283, 255, + /* 40 */ 282, 2, 288, 288, 4, 290, 281, 8, 9, 291, + /* 50 */ 2, 12, 13, 14, 15, 16, 8, 9, 275, 59, + /* 60 */ 12, 13, 14, 15, 16, 65, 283, 283, 314, 315, + /* 70 */ 315, 303, 247, 318, 319, 320, 321, 322, 323, 325, + /* 80 */ 325, 257, 82, 328, 44, 45, 82, 332, 333, 334, + /* 90 */ 0, 8, 9, 344, 270, 12, 13, 14, 15, 16, + /* 100 */ 275, 346, 278, 320, 104, 58, 322, 358, 283, 354, + /* 110 */ 355, 362, 344, 288, 20, 290, 274, 82, 118, 119, + /* 120 */ 336, 337, 338, 339, 255, 341, 358, 255, 303, 287, + /* 130 */ 362, 12, 13, 14, 247, 266, 22, 47, 266, 20, + /* 140 */ 315, 22, 59, 318, 319, 320, 321, 322, 323, 35, + /* 150 */ 325, 20, 283, 328, 35, 283, 146, 332, 333, 61, + /* 160 */ 62, 161, 275, 163, 66, 20, 83, 69, 70, 344, + /* 170 */ 283, 73, 74, 75, 91, 288, 82, 290, 59, 65, + /* 180 */ 20, 1, 2, 358, 65, 185, 186, 362, 188, 189, + /* 190 */ 190, 191, 192, 193, 194, 195, 196, 197, 198, 199, + /* 200 */ 200, 82, 315, 20, 37, 318, 319, 320, 321, 322, + /* 210 */ 323, 33, 325, 213, 290, 328, 255, 213, 104, 332, + /* 220 */ 333, 334, 298, 104, 300, 47, 143, 185, 218, 219, + /* 230 */ 52, 53, 54, 55, 56, 14, 65, 118, 119, 8, + /* 240 */ 9, 20, 355, 12, 13, 14, 15, 16, 213, 166, + /* 250 */ 12, 13, 85, 292, 87, 88, 58, 90, 20, 81, + /* 260 */ 22, 94, 84, 83, 222, 223, 224, 225, 226, 289, + /* 270 */ 290, 12, 13, 35, 43, 161, 65, 163, 255, 20, + /* 280 */ 161, 22, 163, 116, 201, 202, 203, 204, 205, 206, + /* 290 */ 207, 208, 209, 210, 35, 20, 14, 59, 252, 185, + /* 300 */ 186, 255, 20, 65, 185, 186, 283, 188, 189, 190, + /* 310 */ 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, + /* 320 */ 82, 255, 144, 145, 65, 147, 303, 21, 277, 151, + /* 330 */ 24, 25, 26, 27, 28, 29, 30, 31, 32, 288, + /* 340 */ 260, 82, 104, 247, 264, 322, 8, 9, 170, 283, + /* 350 */ 12, 13, 14, 15, 16, 21, 118, 119, 277, 93, + /* 360 */ 337, 338, 339, 104, 341, 314, 315, 344, 34, 288, + /* 370 */ 36, 275, 20, 246, 22, 248, 325, 118, 119, 283, + /* 380 */ 263, 358, 265, 14, 288, 362, 290, 35, 322, 20, + /* 390 */ 12, 13, 14, 118, 119, 314, 315, 301, 20, 161, + /* 400 */ 22, 163, 50, 337, 338, 339, 325, 341, 0, 252, + /* 410 */ 189, 315, 255, 35, 318, 319, 320, 321, 322, 323, + /* 420 */ 161, 325, 163, 185, 186, 276, 188, 189, 190, 191, + /* 430 */ 192, 193, 194, 195, 196, 197, 198, 199, 200, 255, + /* 440 */ 247, 20, 94, 65, 185, 186, 259, 188, 189, 190, + /* 450 */ 191, 192, 193, 194, 195, 196, 197, 198, 199, 200, + /* 460 */ 82, 113, 114, 115, 116, 117, 58, 283, 281, 8, + /* 470 */ 9, 189, 213, 12, 13, 14, 15, 16, 20, 114, + /* 480 */ 22, 288, 104, 247, 8, 9, 148, 303, 12, 13, + /* 490 */ 14, 15, 16, 156, 8, 9, 118, 119, 12, 13, + /* 500 */ 14, 15, 16, 82, 82, 316, 322, 255, 50, 12, + /* 510 */ 13, 261, 262, 176, 177, 255, 255, 20, 266, 22, + /* 520 */ 59, 337, 338, 339, 288, 341, 266, 266, 344, 340, + /* 530 */ 12, 13, 35, 273, 273, 283, 171, 172, 20, 161, + /* 540 */ 22, 163, 358, 283, 283, 20, 362, 14, 15, 16, + /* 550 */ 8, 9, 91, 35, 12, 13, 14, 15, 16, 83, + /* 560 */ 261, 262, 65, 185, 186, 0, 188, 189, 190, 191, + /* 570 */ 192, 193, 194, 195, 196, 197, 198, 199, 200, 82, + /* 580 */ 0, 307, 160, 65, 162, 76, 21, 276, 283, 24, + /* 590 */ 25, 26, 27, 28, 29, 30, 31, 32, 275, 294, + /* 600 */ 82, 104, 8, 9, 143, 282, 12, 13, 14, 15, + /* 610 */ 16, 267, 255, 267, 291, 118, 119, 0, 247, 275, + /* 620 */ 248, 275, 104, 266, 0, 83, 56, 166, 284, 56, + /* 630 */ 284, 122, 123, 247, 213, 213, 118, 119, 58, 113, + /* 640 */ 283, 24, 25, 26, 27, 28, 29, 30, 31, 32, + /* 650 */ 255, 81, 316, 39, 84, 82, 255, 84, 161, 288, + /* 660 */ 163, 266, 201, 202, 203, 204, 205, 206, 207, 208, + /* 670 */ 209, 210, 48, 148, 288, 143, 340, 83, 283, 161, + /* 680 */ 247, 163, 185, 186, 283, 188, 189, 190, 191, 192, + /* 690 */ 193, 194, 195, 196, 197, 198, 199, 200, 166, 173, + /* 700 */ 174, 276, 303, 185, 186, 0, 188, 189, 190, 191, + /* 710 */ 192, 193, 194, 195, 196, 197, 198, 199, 200, 267, + /* 720 */ 275, 288, 18, 322, 20, 18, 35, 275, 316, 284, + /* 730 */ 23, 27, 285, 201, 30, 288, 284, 3, 337, 338, + /* 740 */ 339, 94, 341, 344, 37, 38, 275, 247, 41, 0, + /* 750 */ 290, 47, 340, 49, 244, 51, 65, 358, 20, 255, + /* 760 */ 300, 362, 291, 116, 57, 43, 61, 62, 63, 64, + /* 770 */ 266, 66, 67, 68, 69, 70, 71, 72, 73, 74, + /* 780 */ 75, 76, 77, 78, 79, 81, 213, 283, 288, 82, + /* 790 */ 95, 96, 97, 98, 99, 100, 101, 102, 103, 104, + /* 800 */ 105, 0, 107, 108, 109, 110, 111, 112, 0, 285, + /* 810 */ 61, 62, 288, 303, 303, 66, 276, 268, 69, 70, + /* 820 */ 271, 276, 73, 74, 75, 121, 212, 120, 124, 125, + /* 830 */ 126, 127, 128, 129, 130, 131, 132, 133, 134, 135, + /* 840 */ 136, 137, 138, 139, 140, 141, 44, 45, 255, 20, + /* 850 */ 276, 247, 247, 255, 344, 344, 247, 12, 13, 266, + /* 860 */ 153, 154, 155, 247, 266, 158, 255, 22, 358, 358, + /* 870 */ 247, 164, 362, 362, 4, 255, 283, 266, 247, 0, + /* 880 */ 35, 283, 175, 211, 212, 178, 266, 180, 181, 182, + /* 890 */ 183, 184, 288, 288, 283, 94, 247, 288, 275, 247, + /* 900 */ 43, 22, 94, 283, 288, 86, 283, 264, 89, 247, + /* 910 */ 65, 288, 247, 290, 113, 114, 115, 116, 117, 288, + /* 920 */ 213, 113, 114, 115, 116, 117, 303, 189, 247, 247, + /* 930 */ 247, 56, 247, 247, 148, 149, 86, 288, 315, 89, + /* 940 */ 288, 318, 319, 320, 321, 322, 323, 4, 325, 104, + /* 950 */ 288, 328, 255, 288, 232, 332, 333, 275, 86, 84, + /* 960 */ 59, 89, 19, 266, 230, 283, 0, 344, 255, 288, + /* 970 */ 288, 288, 290, 288, 288, 277, 33, 148, 280, 266, + /* 980 */ 283, 358, 1, 2, 86, 362, 288, 89, 22, 35, + /* 990 */ 47, 43, 247, 276, 51, 0, 283, 315, 255, 56, + /* 1000 */ 318, 319, 320, 321, 322, 323, 161, 325, 163, 266, + /* 1010 */ 328, 255, 314, 315, 332, 333, 334, 22, 43, 43, + /* 1020 */ 275, 43, 266, 325, 81, 343, 283, 84, 283, 255, + /* 1030 */ 185, 83, 43, 288, 82, 290, 118, 119, 46, 283, + /* 1040 */ 266, 196, 197, 198, 92, 43, 35, 43, 356, 365, + /* 1050 */ 43, 256, 43, 247, 351, 43, 275, 283, 83, 83, + /* 1060 */ 315, 83, 43, 318, 319, 320, 321, 322, 323, 35, + /* 1070 */ 325, 311, 83, 328, 82, 247, 256, 332, 333, 334, + /* 1080 */ 43, 275, 43, 43, 214, 83, 185, 83, 343, 283, + /* 1090 */ 83, 234, 83, 254, 288, 83, 290, 43, 287, 65, + /* 1100 */ 317, 43, 83, 275, 342, 359, 359, 359, 345, 215, + /* 1110 */ 313, 283, 20, 255, 47, 312, 288, 163, 290, 35, + /* 1120 */ 83, 315, 83, 83, 318, 319, 320, 321, 322, 323, + /* 1130 */ 261, 325, 159, 255, 328, 305, 247, 83, 332, 333, + /* 1140 */ 255, 83, 42, 315, 295, 293, 318, 319, 320, 321, + /* 1150 */ 322, 323, 293, 325, 255, 20, 328, 143, 249, 249, + /* 1160 */ 332, 333, 334, 20, 275, 309, 259, 290, 20, 302, + /* 1170 */ 259, 343, 283, 20, 163, 304, 259, 288, 302, 290, + /* 1180 */ 259, 20, 296, 259, 259, 255, 259, 249, 275, 247, + /* 1190 */ 275, 275, 303, 288, 275, 275, 249, 275, 275, 255, + /* 1200 */ 288, 275, 309, 247, 315, 169, 275, 318, 319, 320, + /* 1210 */ 321, 322, 323, 255, 325, 275, 275, 275, 257, 290, + /* 1220 */ 257, 257, 308, 255, 288, 283, 20, 221, 227, 152, + /* 1230 */ 288, 275, 290, 344, 302, 220, 299, 350, 288, 283, + /* 1240 */ 350, 288, 350, 288, 288, 303, 290, 358, 299, 353, + /* 1250 */ 317, 362, 352, 229, 348, 247, 228, 315, 216, 313, + /* 1260 */ 318, 319, 320, 321, 322, 323, 283, 325, 212, 20, + /* 1270 */ 42, 315, 231, 247, 318, 319, 320, 321, 322, 323, + /* 1280 */ 233, 325, 347, 275, 328, 316, 344, 236, 332, 333, + /* 1290 */ 82, 283, 335, 299, 349, 361, 288, 299, 290, 360, + /* 1300 */ 358, 275, 331, 145, 362, 288, 288, 297, 288, 283, + /* 1310 */ 283, 82, 296, 283, 288, 271, 290, 366, 257, 257, + /* 1320 */ 361, 361, 360, 315, 279, 247, 318, 319, 320, 321, + /* 1330 */ 322, 323, 324, 325, 326, 327, 288, 360, 265, 257, + /* 1340 */ 255, 315, 247, 36, 318, 319, 320, 321, 322, 323, + /* 1350 */ 249, 325, 250, 275, 258, 306, 310, 269, 269, 269, + /* 1360 */ 245, 283, 0, 0, 42, 0, 288, 73, 290, 0, + /* 1370 */ 275, 35, 179, 35, 35, 35, 179, 0, 283, 35, + /* 1380 */ 35, 179, 0, 288, 179, 290, 0, 35, 0, 363, + /* 1390 */ 364, 247, 22, 315, 0, 35, 318, 319, 320, 321, + /* 1400 */ 322, 323, 0, 325, 166, 82, 328, 165, 163, 161, + /* 1410 */ 315, 333, 0, 318, 319, 320, 321, 322, 323, 275, + /* 1420 */ 325, 0, 157, 156, 0, 0, 46, 283, 0, 142, + /* 1430 */ 0, 0, 288, 0, 290, 0, 0, 137, 35, 0, + /* 1440 */ 137, 0, 0, 0, 0, 0, 0, 0, 0, 0, + /* 1450 */ 0, 247, 357, 0, 0, 0, 0, 42, 0, 315, + /* 1460 */ 247, 42, 318, 319, 320, 321, 322, 323, 39, 325, + /* 1470 */ 0, 0, 0, 0, 0, 0, 8, 9, 0, 275, + /* 1480 */ 12, 13, 14, 15, 16, 22, 0, 283, 275, 0, + /* 1490 */ 0, 0, 288, 14, 290, 14, 283, 46, 0, 40, + /* 1500 */ 43, 288, 0, 290, 39, 301, 46, 0, 364, 247, + /* 1510 */ 152, 0, 0, 0, 0, 39, 0, 0, 35, 315, + /* 1520 */ 0, 247, 318, 319, 320, 321, 322, 323, 315, 325, + /* 1530 */ 35, 318, 319, 320, 321, 322, 323, 275, 325, 47, + /* 1540 */ 327, 47, 60, 39, 39, 283, 0, 47, 39, 275, + /* 1550 */ 288, 0, 290, 35, 35, 47, 0, 283, 0, 0, + /* 1560 */ 39, 0, 288, 301, 290, 35, 22, 0, 91, 89, + /* 1570 */ 35, 35, 19, 35, 247, 301, 35, 315, 35, 0, + /* 1580 */ 318, 319, 320, 321, 322, 323, 33, 325, 43, 315, + /* 1590 */ 247, 43, 318, 319, 320, 321, 322, 323, 35, 325, + /* 1600 */ 47, 35, 275, 22, 0, 52, 53, 54, 55, 56, + /* 1610 */ 283, 22, 0, 49, 22, 288, 0, 290, 275, 35, + /* 1620 */ 35, 153, 0, 35, 0, 22, 283, 20, 0, 35, + /* 1630 */ 0, 288, 148, 290, 81, 22, 167, 84, 150, 247, + /* 1640 */ 0, 148, 315, 0, 145, 318, 319, 320, 321, 322, + /* 1650 */ 323, 148, 325, 0, 0, 0, 82, 43, 315, 83, + /* 1660 */ 43, 318, 319, 320, 321, 322, 323, 275, 325, 83, + /* 1670 */ 117, 46, 82, 82, 43, 283, 83, 82, 82, 217, + /* 1680 */ 288, 83, 290, 46, 43, 43, 2, 82, 43, 83, + /* 1690 */ 46, 83, 46, 83, 43, 35, 35, 35, 211, 247, + /* 1700 */ 147, 83, 217, 235, 35, 35, 35, 315, 247, 185, + /* 1710 */ 318, 319, 320, 321, 322, 323, 43, 325, 217, 83, + /* 1720 */ 46, 168, 83, 170, 46, 22, 187, 275, 146, 82, + /* 1730 */ 82, 82, 0, 39, 82, 283, 275, 46, 46, 83, + /* 1740 */ 288, 83, 290, 144, 283, 82, 22, 82, 82, 288, + /* 1750 */ 82, 290, 93, 35, 82, 35, 35, 247, 83, 92, + /* 1760 */ 83, 82, 82, 82, 35, 83, 83, 315, 82, 247, + /* 1770 */ 318, 319, 320, 321, 322, 323, 315, 325, 82, 318, + /* 1780 */ 319, 320, 321, 322, 323, 275, 325, 83, 35, 82, + /* 1790 */ 35, 83, 82, 283, 106, 106, 106, 275, 288, 106, + /* 1800 */ 290, 22, 35, 94, 82, 283, 43, 82, 82, 22, + /* 1810 */ 288, 60, 290, 59, 65, 35, 80, 43, 35, 35, + /* 1820 */ 247, 35, 22, 35, 35, 315, 35, 35, 318, 319, + /* 1830 */ 320, 321, 322, 323, 247, 325, 65, 315, 35, 35, + /* 1840 */ 318, 319, 320, 321, 322, 323, 35, 325, 275, 35, + /* 1850 */ 35, 35, 35, 0, 35, 47, 283, 39, 0, 35, + /* 1860 */ 47, 288, 275, 290, 39, 0, 35, 47, 39, 0, + /* 1870 */ 283, 35, 39, 47, 0, 288, 35, 290, 35, 0, + /* 1880 */ 20, 22, 21, 367, 22, 247, 22, 21, 315, 367, + /* 1890 */ 367, 318, 319, 320, 321, 322, 323, 247, 325, 367, + /* 1900 */ 367, 367, 315, 367, 367, 318, 319, 320, 321, 322, + /* 1910 */ 323, 247, 325, 275, 367, 367, 367, 367, 367, 367, + /* 1920 */ 367, 283, 367, 367, 367, 275, 288, 367, 290, 367, + /* 1930 */ 367, 367, 367, 283, 367, 367, 367, 367, 288, 275, + /* 1940 */ 290, 367, 367, 367, 367, 367, 367, 283, 367, 367, + /* 1950 */ 367, 367, 288, 315, 290, 367, 318, 319, 320, 321, + /* 1960 */ 322, 323, 247, 325, 367, 315, 367, 367, 318, 319, + /* 1970 */ 320, 321, 322, 323, 367, 325, 367, 367, 367, 315, + /* 1980 */ 367, 367, 318, 319, 320, 321, 322, 323, 247, 325, + /* 1990 */ 275, 367, 367, 367, 367, 367, 367, 367, 283, 367, + /* 2000 */ 367, 367, 367, 288, 367, 290, 367, 367, 367, 367, + /* 2010 */ 367, 367, 367, 367, 367, 367, 275, 367, 367, 367, + /* 2020 */ 367, 367, 367, 367, 283, 367, 367, 367, 367, 288, + /* 2030 */ 315, 290, 367, 318, 319, 320, 321, 322, 323, 367, + /* 2040 */ 325, 247, 367, 367, 367, 367, 367, 367, 367, 367, + /* 2050 */ 367, 367, 367, 247, 367, 367, 315, 367, 367, 318, + /* 2060 */ 319, 320, 321, 322, 323, 367, 325, 367, 367, 275, + /* 2070 */ 367, 367, 367, 367, 367, 367, 367, 283, 367, 367, + /* 2080 */ 367, 275, 288, 367, 290, 367, 367, 367, 367, 283, + /* 2090 */ 367, 367, 367, 367, 288, 367, 290, 367, 367, 367, + /* 2100 */ 367, 367, 367, 367, 247, 367, 367, 367, 367, 315, + /* 2110 */ 367, 367, 318, 319, 320, 321, 322, 323, 247, 325, + /* 2120 */ 367, 315, 367, 367, 318, 319, 320, 321, 322, 323, + /* 2130 */ 367, 325, 275, 367, 367, 367, 367, 367, 367, 367, + /* 2140 */ 283, 367, 367, 367, 367, 288, 275, 290, 367, 367, + /* 2150 */ 367, 367, 367, 367, 283, 367, 367, 367, 367, 288, + /* 2160 */ 367, 290, 367, 367, 367, 367, 367, 367, 367, 247, + /* 2170 */ 367, 367, 315, 367, 367, 318, 319, 320, 321, 322, + /* 2180 */ 323, 247, 325, 367, 367, 367, 315, 367, 367, 318, + /* 2190 */ 319, 320, 321, 322, 323, 367, 325, 275, 367, 367, + /* 2200 */ 367, 367, 367, 367, 367, 283, 367, 367, 367, 275, + /* 2210 */ 288, 367, 290, 367, 367, 367, 367, 283, 367, 367, + /* 2220 */ 367, 367, 288, 367, 290, 367, 367, 367, 367, 367, + /* 2230 */ 367, 367, 367, 367, 367, 367, 367, 315, 367, 367, + /* 2240 */ 318, 319, 320, 321, 322, 323, 367, 325, 367, 315, + /* 2250 */ 367, 367, 318, 319, 320, 321, 322, 323, 367, 325, }; -#define YY_SHIFT_COUNT (635) +#define YY_SHIFT_COUNT (644) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (1866) +#define YY_SHIFT_MAX (1879) static const unsigned short int yy_shift_ofst[] = { - /* 0 */ 728, 0, 0, 48, 208, 208, 208, 208, 256, 256, - /* 10 */ 208, 208, 416, 464, 624, 464, 464, 464, 464, 464, - /* 20 */ 464, 464, 464, 464, 464, 464, 464, 464, 464, 464, - /* 30 */ 464, 464, 464, 464, 464, 464, 464, 464, 464, 464, - /* 40 */ 66, 66, 104, 104, 104, 18, 18, 18, 18, 12, - /* 50 */ 79, 207, 36, 36, 115, 115, 202, 153, 207, 207, - /* 60 */ 36, 36, 36, 36, 36, 36, 36, 36, 73, 36, - /* 70 */ 36, 180, 190, 495, 180, 36, 36, 180, 36, 180, - /* 80 */ 180, 180, 36, 475, 706, 478, 660, 660, 395, 189, - /* 90 */ 830, 830, 830, 830, 830, 830, 830, 830, 830, 830, - /* 100 */ 830, 830, 830, 830, 830, 830, 830, 830, 830, 198, - /* 110 */ 153, 4, 4, 281, 419, 553, 553, 553, 327, 419, - /* 120 */ 601, 495, 180, 180, 571, 571, 578, 727, 246, 246, - /* 130 */ 246, 246, 246, 246, 246, 1215, 351, 638, 350, 187, - /* 140 */ 449, 357, 353, 560, 614, 392, 757, 616, 620, 618, - /* 150 */ 659, 618, 165, 165, 165, 328, 696, 819, 1015, 994, - /* 160 */ 1004, 907, 1015, 1015, 1038, 937, 937, 1015, 1073, 1073, - /* 170 */ 1076, 73, 495, 73, 1084, 1087, 73, 1084, 73, 601, - /* 180 */ 1094, 73, 73, 1015, 73, 1073, 180, 180, 180, 180, - /* 190 */ 180, 180, 180, 180, 180, 180, 180, 1015, 1073, 571, - /* 200 */ 571, 1076, 475, 985, 495, 475, 1015, 1084, 475, 601, - /* 210 */ 1094, 475, 1150, 571, 974, 980, 571, 974, 980, 571, - /* 220 */ 571, 180, 981, 1059, 974, 983, 987, 997, 819, 1005, - /* 230 */ 601, 1207, 1190, 1000, 1018, 1010, 1000, 1018, 1000, 1018, - /* 240 */ 1168, 980, 571, 571, 980, 571, 1110, 601, 1094, 475, - /* 250 */ 578, 475, 601, 1197, 571, 727, 1015, 475, 1073, 2123, - /* 260 */ 2123, 2123, 2123, 2123, 2123, 2123, 481, 1467, 561, 119, - /* 270 */ 645, 887, 63, 13, 303, 203, 582, 374, 443, 607, - /* 280 */ 550, 550, 550, 550, 550, 550, 550, 550, 132, 382, - /* 290 */ 152, 400, 662, 137, 651, 651, 651, 651, 259, 829, - /* 300 */ 765, 786, 787, 788, 930, 953, 961, 878, 862, 864, - /* 310 */ 872, 913, 824, 752, 741, 897, 591, 904, 911, 908, - /* 320 */ 927, 933, 940, 943, 869, 873, 967, 970, 976, 977, - /* 330 */ 978, 979, 277, 838, 947, 1315, 1316, 1278, 1320, 1250, - /* 340 */ 1323, 1277, 1148, 1279, 1280, 1281, 1155, 1333, 1290, 1291, - /* 350 */ 1162, 1340, 1164, 1342, 1296, 1353, 1324, 1356, 1310, 1358, - /* 360 */ 1289, 1196, 1200, 1214, 1218, 1376, 1378, 1224, 1228, 1384, - /* 370 */ 1386, 1344, 1392, 1253, 1395, 1397, 1398, 1400, 1402, 1404, - /* 380 */ 1405, 1406, 1407, 1409, 1411, 1413, 1421, 1423, 1424, 1425, - /* 390 */ 1427, 1428, 1389, 1430, 1431, 1432, 1433, 1442, 1444, 1414, - /* 400 */ 1437, 1438, 1445, 1446, 1447, 1449, 1418, 1416, 1410, 1440, - /* 410 */ 1419, 1450, 1448, 1460, 1429, 1451, 1465, 1468, 1484, 1454, - /* 420 */ 1336, 1490, 1493, 1457, 1495, 1439, 1496, 1499, 1456, 1461, - /* 430 */ 1464, 1507, 1463, 1466, 1471, 1509, 1469, 1472, 1477, 1523, - /* 440 */ 1478, 1479, 1489, 1528, 1530, 1531, 1541, 1452, 1455, 1491, - /* 450 */ 1524, 1545, 1502, 1504, 1505, 1510, 1515, 1517, 1518, 1521, - /* 460 */ 1525, 1564, 1548, 1571, 1551, 1526, 1575, 1554, 1532, 1577, - /* 470 */ 1556, 1581, 1567, 1572, 1593, 1458, 1549, 1595, 1443, 1585, - /* 480 */ 1473, 1470, 1603, 1611, 1613, 1475, 1619, 1539, 1588, 1485, - /* 490 */ 1586, 1589, 1422, 1553, 1547, 1557, 1559, 1598, 1560, 1562, - /* 500 */ 1563, 1565, 1566, 1609, 1592, 1607, 1579, 1626, 1459, 1587, - /* 510 */ 1591, 1627, 1474, 1629, 1640, 1604, 1644, 1476, 1605, 1641, - /* 520 */ 1642, 1643, 1645, 1646, 1647, 1605, 1693, 1513, 1656, 1618, - /* 530 */ 1620, 1621, 1655, 1623, 1624, 1662, 1680, 1527, 1628, 1631, - /* 540 */ 1632, 1630, 1634, 1573, 1636, 1710, 1681, 1583, 1639, 1633, - /* 550 */ 1678, 1683, 1648, 1649, 1651, 1706, 1654, 1638, 1657, 1689, - /* 560 */ 1690, 1660, 1661, 1697, 1664, 1668, 1699, 1676, 1679, 1713, - /* 570 */ 1682, 1684, 1715, 1686, 1659, 1671, 1685, 1687, 1763, 1694, - /* 580 */ 1707, 1708, 1744, 1712, 1754, 1754, 1774, 1738, 1740, 1755, - /* 590 */ 1739, 1725, 1764, 1759, 1760, 1761, 1762, 1765, 1789, 1766, - /* 600 */ 1767, 1751, 1515, 1769, 1517, 1770, 1772, 1773, 1775, 1777, - /* 610 */ 1781, 1821, 1790, 1793, 1802, 1841, 1796, 1799, 1808, 1846, - /* 620 */ 1801, 1804, 1810, 1852, 1812, 1816, 1826, 1864, 1818, 1866, - /* 630 */ 1847, 1855, 1857, 1858, 1860, 1848, + /* 0 */ 707, 0, 0, 119, 238, 238, 238, 238, 259, 259, + /* 10 */ 238, 238, 378, 497, 518, 497, 497, 497, 497, 497, + /* 20 */ 497, 497, 497, 497, 497, 497, 497, 497, 497, 497, + /* 30 */ 497, 497, 497, 497, 497, 497, 497, 497, 497, 497, + /* 40 */ 497, 497, 421, 421, 94, 94, 94, 845, 845, 845, + /* 50 */ 845, 422, 573, 35, 16, 16, 40, 40, 4, 275, + /* 60 */ 35, 35, 16, 16, 16, 16, 16, 16, 16, 16, + /* 70 */ 47, 16, 16, 16, 131, 145, 160, 16, 16, 160, + /* 80 */ 16, 160, 160, 160, 16, 198, 704, 83, 461, 461, + /* 90 */ 306, 98, 114, 114, 114, 114, 114, 114, 114, 114, + /* 100 */ 114, 114, 114, 114, 114, 114, 114, 114, 114, 114, + /* 110 */ 114, 167, 275, 369, 369, 408, 691, 525, 525, 525, + /* 120 */ 580, 691, 183, 145, 160, 160, 171, 171, 266, 211, + /* 130 */ 695, 695, 695, 695, 695, 695, 695, 1553, 565, 749, + /* 140 */ 1468, 42, 352, 365, 10, 221, 282, 458, 802, 647, + /* 150 */ 829, 672, 614, 672, 734, 734, 734, 870, 738, 894, + /* 160 */ 1092, 1067, 1084, 973, 1092, 1092, 1100, 1014, 1014, 1092, + /* 170 */ 1135, 1135, 1143, 47, 145, 47, 1148, 1153, 47, 1148, + /* 180 */ 47, 1161, 47, 47, 1092, 47, 1135, 160, 160, 160, + /* 190 */ 160, 160, 160, 160, 160, 160, 160, 160, 1092, 1135, + /* 200 */ 171, 171, 1143, 198, 1036, 145, 198, 1092, 1092, 1148, + /* 210 */ 198, 1206, 171, 1006, 1015, 171, 1006, 1015, 171, 171, + /* 220 */ 160, 1001, 1077, 1006, 1024, 1028, 1042, 894, 1056, 183, + /* 230 */ 1249, 1228, 1047, 1041, 1051, 1047, 1041, 1047, 1041, 1208, + /* 240 */ 1015, 171, 171, 1015, 171, 1158, 183, 1161, 198, 266, + /* 250 */ 198, 183, 1229, 171, 211, 1092, 198, 1307, 1135, 2260, + /* 260 */ 2260, 2260, 2260, 2260, 2260, 2260, 705, 178, 617, 943, + /* 270 */ 476, 231, 542, 39, 48, 338, 594, 486, 486, 486, + /* 280 */ 486, 486, 486, 486, 486, 801, 808, 348, 3, 3, + /* 290 */ 337, 570, 509, 334, 526, 180, 532, 533, 533, 533, + /* 300 */ 533, 90, 948, 819, 850, 872, 898, 879, 966, 995, + /* 310 */ 875, 786, 975, 976, 981, 918, 722, 857, 978, 901, + /* 320 */ 989, 992, 1002, 1004, 1007, 1009, 1012, 954, 1011, 1019, + /* 330 */ 1037, 1039, 1040, 1054, 1058, 952, 1034, 624, 1362, 1363, + /* 340 */ 1322, 1365, 1294, 1369, 1336, 1193, 1338, 1339, 1340, 1197, + /* 350 */ 1377, 1344, 1345, 1202, 1382, 1205, 1386, 1352, 1388, 1370, + /* 360 */ 1394, 1360, 1402, 1323, 1238, 1242, 1245, 1248, 1412, 1421, + /* 370 */ 1265, 1267, 1424, 1425, 1380, 1428, 1287, 1430, 1431, 1433, + /* 380 */ 1435, 1436, 1300, 1403, 1439, 1303, 1441, 1442, 1443, 1444, + /* 390 */ 1445, 1446, 1447, 1448, 1449, 1450, 1453, 1454, 1455, 1456, + /* 400 */ 1415, 1458, 1470, 1471, 1472, 1473, 1474, 1463, 1475, 1478, + /* 410 */ 1486, 1489, 1490, 1491, 1419, 1429, 1457, 1479, 1451, 1481, + /* 420 */ 1460, 1498, 1459, 1465, 1502, 1507, 1520, 1476, 1358, 1511, + /* 430 */ 1512, 1513, 1482, 1514, 1516, 1483, 1492, 1504, 1517, 1495, + /* 440 */ 1494, 1505, 1546, 1518, 1500, 1509, 1551, 1519, 1508, 1521, + /* 450 */ 1556, 1558, 1559, 1561, 1477, 1480, 1530, 1544, 1567, 1535, + /* 460 */ 1536, 1538, 1541, 1545, 1548, 1543, 1563, 1566, 1579, 1581, + /* 470 */ 1604, 1589, 1564, 1612, 1592, 1584, 1616, 1585, 1622, 1588, + /* 480 */ 1624, 1603, 1607, 1628, 1484, 1594, 1630, 1469, 1613, 1493, + /* 490 */ 1499, 1640, 1643, 1503, 1488, 1653, 1654, 1655, 1614, 1617, + /* 500 */ 1462, 1574, 1576, 1590, 1586, 1631, 1593, 1591, 1595, 1596, + /* 510 */ 1598, 1641, 1625, 1637, 1605, 1642, 1485, 1606, 1608, 1644, + /* 520 */ 1487, 1645, 1646, 1610, 1651, 1501, 1618, 1660, 1661, 1662, + /* 530 */ 1669, 1670, 1671, 1618, 1684, 1524, 1673, 1636, 1647, 1639, + /* 540 */ 1674, 1648, 1649, 1678, 1703, 1539, 1652, 1656, 1658, 1663, + /* 550 */ 1665, 1582, 1666, 1732, 1694, 1599, 1668, 1667, 1691, 1692, + /* 560 */ 1672, 1675, 1679, 1724, 1680, 1659, 1677, 1718, 1720, 1681, + /* 570 */ 1682, 1721, 1686, 1683, 1729, 1696, 1704, 1753, 1707, 1708, + /* 580 */ 1755, 1710, 1688, 1689, 1690, 1693, 1779, 1709, 1722, 1725, + /* 590 */ 1767, 1726, 1763, 1763, 1787, 1751, 1754, 1780, 1749, 1736, + /* 600 */ 1774, 1783, 1784, 1786, 1788, 1789, 1800, 1791, 1792, 1771, + /* 610 */ 1545, 1803, 1548, 1804, 1811, 1814, 1815, 1816, 1817, 1853, + /* 620 */ 1819, 1808, 1818, 1858, 1824, 1813, 1825, 1865, 1831, 1820, + /* 630 */ 1829, 1869, 1836, 1826, 1833, 1874, 1841, 1843, 1879, 1859, + /* 640 */ 1861, 1862, 1864, 1866, 1860, }; #define YY_REDUCE_COUNT (265) -#define YY_REDUCE_MIN (-340) -#define YY_REDUCE_MAX (1800) +#define YY_REDUCE_MIN (-333) +#define YY_REDUCE_MAX (1934) static const short yy_reduce_ofst[] = { - /* 0 */ -233, -228, -20, -176, 606, 647, 725, 751, 803, 812, - /* 10 */ 188, 861, 902, 920, 971, 1032, 1049, 1100, 1154, 1163, - /* 20 */ 1217, 1268, 1282, 1337, 1346, 1360, 1436, 1453, 1462, 1514, - /* 30 */ 1538, 1555, 1606, 1617, 1658, 1669, 1722, 1734, 1748, 1800, - /* 40 */ 437, 1055, -5, 1121, 1225, -274, -269, 49, 417, -227, - /* 50 */ -47, 161, -231, -229, -244, -215, -340, -105, -133, 123, - /* 60 */ -197, -147, 363, 445, 452, 482, 514, 517, -171, 518, - /* 70 */ 543, 200, -24, 234, -201, 637, 677, 251, 524, -101, - /* 80 */ 345, 358, 708, -119, -152, -281, -281, -281, 78, -247, - /* 90 */ -135, -21, 229, 250, 293, 331, 334, 356, 384, 450, - /* 100 */ 468, 503, 504, 520, 603, 610, 611, 612, 613, 253, - /* 110 */ -83, 182, 183, -27, 373, -263, 172, 355, -155, 435, - /* 120 */ 499, -172, 346, 442, 405, 458, 487, 227, -271, 530, - /* 130 */ 552, 576, 581, 609, 670, 644, 704, 690, 594, 632, - /* 140 */ 742, 686, 650, 732, 732, 753, 756, 737, 702, 685, - /* 150 */ 685, 685, 671, 672, 673, 684, 732, 723, 794, 744, - /* 160 */ 804, 770, 822, 825, 793, 796, 797, 837, 847, 848, - /* 170 */ 792, 844, 814, 846, 806, 807, 853, 811, 855, 833, - /* 180 */ 823, 870, 879, 865, 880, 890, 866, 867, 868, 871, - /* 190 */ 877, 882, 883, 884, 885, 888, 889, 893, 895, 874, - /* 200 */ 881, 845, 896, 858, 898, 910, 916, 876, 915, 892, - /* 210 */ 891, 934, 894, 909, 849, 899, 912, 852, 905, 917, - /* 220 */ 918, 732, 854, 863, 886, 900, 914, 901, 919, 685, - /* 230 */ 948, 928, 921, 903, 906, 922, 924, 935, 926, 942, - /* 240 */ 929, 954, 969, 986, 956, 988, 968, 984, 972, 1020, - /* 250 */ 1007, 1022, 992, 1003, 1012, 1040, 1048, 1051, 1056, 1006, - /* 260 */ 999, 1042, 1044, 1045, 1058, 1071, + /* 0 */ 510, -175, 623, -245, -113, 682, 745, 828, 889, 942, + /* 10 */ 806, 956, 1008, 1026, 1078, 96, 1095, 1144, 1204, 1213, + /* 20 */ 1262, 1274, 1327, 1343, 1392, 1452, 1461, 1510, 1522, 1573, + /* 30 */ 1587, 1638, 1650, 1664, 1715, 1741, 1794, 1806, 1857, 1871, + /* 40 */ 1922, 1934, 23, 184, -216, 66, 401, -246, 698, 51, + /* 50 */ 81, -232, 399, 511, 260, 261, -250, -244, -333, -263, + /* 60 */ -330, -251, -131, -128, 252, 357, 395, 504, 593, 598, + /* 70 */ -235, 611, 697, 713, -217, -76, 344, 620, 743, -242, + /* 80 */ 756, 346, 323, 452, 774, -176, -39, -324, -324, -324, + /* 90 */ 127, 80, 193, 236, 371, 386, 433, 500, 604, 605, + /* 100 */ 609, 616, 631, 649, 652, 662, 665, 681, 683, 685, + /* 110 */ 686, -158, -20, 46, 157, 187, 250, 189, 336, 412, + /* 120 */ -257, 299, 305, 460, 445, 471, 447, 524, 549, 117, + /* 130 */ 149, 311, 425, 540, 545, 574, 717, 274, 372, 643, + /* 140 */ 684, 692, 795, 760, 703, 781, 781, 820, 839, 811, + /* 150 */ 783, 762, 762, 762, 746, 747, 748, 763, 781, 797, + /* 160 */ 858, 803, 869, 830, 878, 885, 849, 852, 859, 899, + /* 170 */ 909, 910, 856, 907, 877, 911, 867, 871, 917, 876, + /* 180 */ 921, 886, 924, 925, 930, 927, 938, 913, 915, 916, + /* 190 */ 919, 920, 922, 923, 926, 931, 940, 941, 944, 947, + /* 200 */ 905, 912, 893, 961, 914, 929, 963, 958, 968, 932, + /* 210 */ 964, 933, 936, 887, 937, 950, 890, 949, 953, 955, + /* 220 */ 781, 896, 900, 892, 945, 906, 935, 946, 762, 983, + /* 230 */ 969, 957, 934, 939, 951, 959, 962, 960, 977, 971, + /* 240 */ 994, 1017, 1018, 998, 1020, 1010, 1027, 1016, 1061, 1044, + /* 250 */ 1062, 1030, 1045, 1048, 1073, 1085, 1082, 1102, 1101, 1049, + /* 260 */ 1046, 1088, 1089, 1090, 1096, 1115, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 10 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 20 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 30 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 40 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 50 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 60 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1461, 1392, - /* 70 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 80 */ 1392, 1392, 1392, 1459, 1608, 1392, 1772, 1392, 1392, 1392, - /* 90 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 100 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 110 */ 1392, 1392, 1392, 1461, 1392, 1783, 1783, 1783, 1459, 1392, - /* 120 */ 1392, 1392, 1392, 1392, 1392, 1392, 1555, 1392, 1392, 1392, - /* 130 */ 1392, 1392, 1392, 1392, 1392, 1641, 1392, 1392, 1853, 1392, - /* 140 */ 1392, 1647, 1807, 1392, 1392, 1392, 1392, 1508, 1799, 1775, - /* 150 */ 1789, 1776, 1838, 1838, 1838, 1792, 1392, 1803, 1392, 1392, - /* 160 */ 1392, 1633, 1392, 1392, 1613, 1610, 1610, 1392, 1392, 1392, - /* 170 */ 1392, 1461, 1392, 1461, 1392, 1392, 1461, 1392, 1461, 1392, - /* 180 */ 1392, 1461, 1461, 1392, 1461, 1392, 1392, 1392, 1392, 1392, - /* 190 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 200 */ 1392, 1392, 1459, 1643, 1392, 1459, 1392, 1392, 1459, 1392, - /* 210 */ 1392, 1459, 1392, 1392, 1814, 1812, 1392, 1814, 1812, 1392, - /* 220 */ 1392, 1392, 1826, 1822, 1814, 1830, 1828, 1805, 1803, 1789, - /* 230 */ 1392, 1392, 1392, 1844, 1840, 1856, 1844, 1840, 1844, 1840, - /* 240 */ 1392, 1812, 1392, 1392, 1812, 1392, 1618, 1392, 1392, 1459, - /* 250 */ 1392, 1459, 1392, 1524, 1392, 1392, 1392, 1459, 1392, 1635, - /* 260 */ 1649, 1558, 1558, 1558, 1462, 1397, 1392, 1392, 1392, 1392, - /* 270 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1520, - /* 280 */ 1716, 1825, 1824, 1748, 1747, 1746, 1744, 1715, 1392, 1392, - /* 290 */ 1392, 1392, 1392, 1392, 1709, 1710, 1708, 1707, 1392, 1392, - /* 300 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 310 */ 1392, 1773, 1392, 1841, 1845, 1392, 1392, 1392, 1692, 1392, - /* 320 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 330 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 340 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 350 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 360 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 370 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 380 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 390 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 400 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1426, 1392, - /* 410 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 420 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 430 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 440 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 450 */ 1392, 1392, 1392, 1392, 1392, 1392, 1489, 1488, 1392, 1392, - /* 460 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 470 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 480 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 490 */ 1796, 1806, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 500 */ 1392, 1392, 1392, 1392, 1392, 1692, 1392, 1823, 1392, 1782, - /* 510 */ 1778, 1392, 1392, 1774, 1392, 1392, 1839, 1392, 1392, 1392, - /* 520 */ 1392, 1392, 1392, 1392, 1392, 1392, 1768, 1392, 1741, 1392, - /* 530 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1703, 1392, 1392, - /* 540 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 550 */ 1691, 1392, 1732, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 560 */ 1392, 1552, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 570 */ 1392, 1392, 1392, 1392, 1537, 1535, 1534, 1533, 1392, 1530, - /* 580 */ 1392, 1392, 1392, 1392, 1561, 1560, 1392, 1392, 1392, 1392, - /* 590 */ 1392, 1392, 1481, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 600 */ 1392, 1392, 1472, 1392, 1471, 1392, 1392, 1392, 1392, 1392, - /* 610 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 620 */ 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, 1392, - /* 630 */ 1392, 1392, 1392, 1392, 1392, 1392, + /* 0 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 10 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 20 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 30 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 40 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 50 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 60 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 70 */ 1483, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 80 */ 1411, 1411, 1411, 1411, 1411, 1481, 1632, 1411, 1799, 1411, + /* 90 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 100 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 110 */ 1411, 1411, 1411, 1411, 1411, 1483, 1411, 1811, 1811, 1811, + /* 120 */ 1481, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1577, 1411, + /* 130 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1666, 1411, 1411, + /* 140 */ 1881, 1411, 1411, 1672, 1835, 1411, 1411, 1411, 1411, 1530, + /* 150 */ 1827, 1803, 1817, 1804, 1866, 1866, 1866, 1820, 1411, 1831, + /* 160 */ 1411, 1411, 1411, 1658, 1411, 1411, 1637, 1634, 1634, 1411, + /* 170 */ 1411, 1411, 1411, 1483, 1411, 1483, 1411, 1411, 1483, 1411, + /* 180 */ 1483, 1411, 1483, 1483, 1411, 1483, 1411, 1411, 1411, 1411, + /* 190 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 200 */ 1411, 1411, 1411, 1481, 1668, 1411, 1481, 1411, 1411, 1411, + /* 210 */ 1481, 1411, 1411, 1842, 1840, 1411, 1842, 1840, 1411, 1411, + /* 220 */ 1411, 1854, 1850, 1842, 1858, 1856, 1833, 1831, 1817, 1411, + /* 230 */ 1411, 1801, 1872, 1868, 1884, 1872, 1868, 1872, 1868, 1411, + /* 240 */ 1840, 1411, 1411, 1840, 1411, 1641, 1411, 1411, 1481, 1411, + /* 250 */ 1481, 1411, 1546, 1411, 1411, 1411, 1481, 1443, 1411, 1660, + /* 260 */ 1674, 1580, 1580, 1580, 1484, 1416, 1411, 1411, 1411, 1411, + /* 270 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1743, 1853, 1852, + /* 280 */ 1775, 1774, 1773, 1771, 1742, 1411, 1411, 1542, 1741, 1740, + /* 290 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1734, 1735, 1733, + /* 300 */ 1732, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 310 */ 1411, 1411, 1411, 1411, 1800, 1411, 1869, 1873, 1411, 1411, + /* 320 */ 1411, 1717, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 330 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 340 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 350 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 360 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 370 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 380 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 390 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 400 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 410 */ 1411, 1411, 1411, 1411, 1411, 1411, 1448, 1411, 1411, 1411, + /* 420 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 430 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 440 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 450 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 460 */ 1411, 1411, 1411, 1511, 1510, 1411, 1411, 1411, 1411, 1411, + /* 470 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 480 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 490 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1824, 1834, + /* 500 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 510 */ 1411, 1411, 1411, 1717, 1411, 1851, 1411, 1810, 1806, 1411, + /* 520 */ 1411, 1802, 1411, 1411, 1867, 1411, 1411, 1411, 1411, 1411, + /* 530 */ 1411, 1411, 1411, 1411, 1795, 1411, 1768, 1411, 1411, 1411, + /* 540 */ 1411, 1411, 1411, 1411, 1411, 1728, 1411, 1411, 1411, 1411, + /* 550 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1716, 1411, + /* 560 */ 1759, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1574, + /* 570 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 580 */ 1411, 1411, 1559, 1557, 1556, 1555, 1411, 1552, 1411, 1411, + /* 590 */ 1411, 1411, 1583, 1582, 1411, 1411, 1411, 1411, 1411, 1411, + /* 600 */ 1503, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 610 */ 1494, 1411, 1493, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 620 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 630 */ 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, 1411, + /* 640 */ 1411, 1411, 1411, 1411, 1411, }; /********** End of lemon-generated parsing tables *****************************/ @@ -865,7 +894,9 @@ static const YYCODETYPE yyFallback[] = { 0, /* CONNS => nothing */ 0, /* STATE => nothing */ 0, /* USER => nothing */ - 0, /* PRIVILEGE => nothing */ + 0, /* ENABLE => nothing */ + 0, /* NK_INTEGER => nothing */ + 0, /* SYSINFO => nothing */ 0, /* DROP => nothing */ 0, /* GRANT => nothing */ 0, /* ON => nothing */ @@ -878,7 +909,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* NK_DOT => nothing */ 0, /* DNODE => nothing */ 0, /* PORT => nothing */ - 0, /* NK_INTEGER => nothing */ 0, /* DNODES => nothing */ 0, /* NK_IPTOKEN => nothing */ 0, /* LOCAL => nothing */ @@ -975,11 +1005,12 @@ static const YYCODETYPE yyFallback[] = { 0, /* DISTRIBUTED => nothing */ 0, /* LIKE => nothing */ 0, /* INDEX => nothing */ - 0, /* FULLTEXT => nothing */ 0, /* FUNCTION => nothing */ 0, /* INTERVAL => nothing */ 0, /* TOPIC => nothing */ 0, /* AS => nothing */ + 0, /* WITH => nothing */ + 0, /* META => nothing */ 0, /* CONSUMER => nothing */ 0, /* GROUP => nothing */ 0, /* DESC => nothing */ @@ -1067,12 +1098,12 @@ static const YYCODETYPE yyFallback[] = { 0, /* ASC => nothing */ 0, /* NULLS => nothing */ 0, /* ID => nothing */ - 235, /* NK_BITNOT => ID */ - 235, /* INSERT => ID */ - 235, /* VALUES => ID */ - 235, /* IMPORT => ID */ - 235, /* NK_SEMI => ID */ - 235, /* FILE => ID */ + 237, /* NK_BITNOT => ID */ + 237, /* INSERT => ID */ + 237, /* VALUES => ID */ + 237, /* IMPORT => ID */ + 237, /* NK_SEMI => ID */ + 237, /* FILE => ID */ }; #endif /* YYFALLBACK */ @@ -1194,336 +1225,339 @@ static const char *const yyTokenName[] = { /* 31 */ "CONNS", /* 32 */ "STATE", /* 33 */ "USER", - /* 34 */ "PRIVILEGE", - /* 35 */ "DROP", - /* 36 */ "GRANT", - /* 37 */ "ON", - /* 38 */ "TO", - /* 39 */ "REVOKE", - /* 40 */ "FROM", - /* 41 */ "NK_COMMA", - /* 42 */ "READ", - /* 43 */ "WRITE", - /* 44 */ "NK_DOT", - /* 45 */ "DNODE", - /* 46 */ "PORT", - /* 47 */ "NK_INTEGER", - /* 48 */ "DNODES", - /* 49 */ "NK_IPTOKEN", - /* 50 */ "LOCAL", - /* 51 */ "QNODE", - /* 52 */ "BNODE", - /* 53 */ "SNODE", - /* 54 */ "MNODE", - /* 55 */ "DATABASE", - /* 56 */ "USE", - /* 57 */ "IF", - /* 58 */ "NOT", - /* 59 */ "EXISTS", - /* 60 */ "BUFFER", - /* 61 */ "CACHELAST", - /* 62 */ "COMP", - /* 63 */ "DURATION", - /* 64 */ "NK_VARIABLE", - /* 65 */ "FSYNC", - /* 66 */ "MAXROWS", - /* 67 */ "MINROWS", - /* 68 */ "KEEP", - /* 69 */ "PAGES", - /* 70 */ "PAGESIZE", - /* 71 */ "PRECISION", - /* 72 */ "REPLICA", - /* 73 */ "STRICT", - /* 74 */ "WAL", - /* 75 */ "VGROUPS", - /* 76 */ "SINGLE_STABLE", - /* 77 */ "RETENTIONS", - /* 78 */ "SCHEMALESS", - /* 79 */ "NK_COLON", - /* 80 */ "TABLE", - /* 81 */ "NK_LP", - /* 82 */ "NK_RP", - /* 83 */ "STABLE", - /* 84 */ "ADD", - /* 85 */ "COLUMN", - /* 86 */ "MODIFY", - /* 87 */ "RENAME", - /* 88 */ "TAG", - /* 89 */ "SET", - /* 90 */ "NK_EQ", - /* 91 */ "USING", - /* 92 */ "TAGS", - /* 93 */ "COMMENT", - /* 94 */ "BOOL", - /* 95 */ "TINYINT", - /* 96 */ "SMALLINT", - /* 97 */ "INT", - /* 98 */ "INTEGER", - /* 99 */ "BIGINT", - /* 100 */ "FLOAT", - /* 101 */ "DOUBLE", - /* 102 */ "BINARY", - /* 103 */ "TIMESTAMP", - /* 104 */ "NCHAR", - /* 105 */ "UNSIGNED", - /* 106 */ "JSON", - /* 107 */ "VARCHAR", - /* 108 */ "MEDIUMBLOB", - /* 109 */ "BLOB", - /* 110 */ "VARBINARY", - /* 111 */ "DECIMAL", - /* 112 */ "MAX_DELAY", - /* 113 */ "WATERMARK", - /* 114 */ "ROLLUP", - /* 115 */ "TTL", - /* 116 */ "SMA", - /* 117 */ "FIRST", - /* 118 */ "LAST", - /* 119 */ "SHOW", - /* 120 */ "DATABASES", - /* 121 */ "TABLES", - /* 122 */ "STABLES", - /* 123 */ "MNODES", - /* 124 */ "MODULES", - /* 125 */ "QNODES", - /* 126 */ "FUNCTIONS", - /* 127 */ "INDEXES", - /* 128 */ "ACCOUNTS", - /* 129 */ "APPS", - /* 130 */ "CONNECTIONS", - /* 131 */ "LICENCE", - /* 132 */ "GRANTS", - /* 133 */ "QUERIES", - /* 134 */ "SCORES", - /* 135 */ "TOPICS", - /* 136 */ "VARIABLES", - /* 137 */ "BNODES", - /* 138 */ "SNODES", - /* 139 */ "CLUSTER", - /* 140 */ "TRANSACTIONS", - /* 141 */ "DISTRIBUTED", - /* 142 */ "LIKE", - /* 143 */ "INDEX", - /* 144 */ "FULLTEXT", + /* 34 */ "ENABLE", + /* 35 */ "NK_INTEGER", + /* 36 */ "SYSINFO", + /* 37 */ "DROP", + /* 38 */ "GRANT", + /* 39 */ "ON", + /* 40 */ "TO", + /* 41 */ "REVOKE", + /* 42 */ "FROM", + /* 43 */ "NK_COMMA", + /* 44 */ "READ", + /* 45 */ "WRITE", + /* 46 */ "NK_DOT", + /* 47 */ "DNODE", + /* 48 */ "PORT", + /* 49 */ "DNODES", + /* 50 */ "NK_IPTOKEN", + /* 51 */ "LOCAL", + /* 52 */ "QNODE", + /* 53 */ "BNODE", + /* 54 */ "SNODE", + /* 55 */ "MNODE", + /* 56 */ "DATABASE", + /* 57 */ "USE", + /* 58 */ "IF", + /* 59 */ "NOT", + /* 60 */ "EXISTS", + /* 61 */ "BUFFER", + /* 62 */ "CACHELAST", + /* 63 */ "COMP", + /* 64 */ "DURATION", + /* 65 */ "NK_VARIABLE", + /* 66 */ "FSYNC", + /* 67 */ "MAXROWS", + /* 68 */ "MINROWS", + /* 69 */ "KEEP", + /* 70 */ "PAGES", + /* 71 */ "PAGESIZE", + /* 72 */ "PRECISION", + /* 73 */ "REPLICA", + /* 74 */ "STRICT", + /* 75 */ "WAL", + /* 76 */ "VGROUPS", + /* 77 */ "SINGLE_STABLE", + /* 78 */ "RETENTIONS", + /* 79 */ "SCHEMALESS", + /* 80 */ "NK_COLON", + /* 81 */ "TABLE", + /* 82 */ "NK_LP", + /* 83 */ "NK_RP", + /* 84 */ "STABLE", + /* 85 */ "ADD", + /* 86 */ "COLUMN", + /* 87 */ "MODIFY", + /* 88 */ "RENAME", + /* 89 */ "TAG", + /* 90 */ "SET", + /* 91 */ "NK_EQ", + /* 92 */ "USING", + /* 93 */ "TAGS", + /* 94 */ "COMMENT", + /* 95 */ "BOOL", + /* 96 */ "TINYINT", + /* 97 */ "SMALLINT", + /* 98 */ "INT", + /* 99 */ "INTEGER", + /* 100 */ "BIGINT", + /* 101 */ "FLOAT", + /* 102 */ "DOUBLE", + /* 103 */ "BINARY", + /* 104 */ "TIMESTAMP", + /* 105 */ "NCHAR", + /* 106 */ "UNSIGNED", + /* 107 */ "JSON", + /* 108 */ "VARCHAR", + /* 109 */ "MEDIUMBLOB", + /* 110 */ "BLOB", + /* 111 */ "VARBINARY", + /* 112 */ "DECIMAL", + /* 113 */ "MAX_DELAY", + /* 114 */ "WATERMARK", + /* 115 */ "ROLLUP", + /* 116 */ "TTL", + /* 117 */ "SMA", + /* 118 */ "FIRST", + /* 119 */ "LAST", + /* 120 */ "SHOW", + /* 121 */ "DATABASES", + /* 122 */ "TABLES", + /* 123 */ "STABLES", + /* 124 */ "MNODES", + /* 125 */ "MODULES", + /* 126 */ "QNODES", + /* 127 */ "FUNCTIONS", + /* 128 */ "INDEXES", + /* 129 */ "ACCOUNTS", + /* 130 */ "APPS", + /* 131 */ "CONNECTIONS", + /* 132 */ "LICENCE", + /* 133 */ "GRANTS", + /* 134 */ "QUERIES", + /* 135 */ "SCORES", + /* 136 */ "TOPICS", + /* 137 */ "VARIABLES", + /* 138 */ "BNODES", + /* 139 */ "SNODES", + /* 140 */ "CLUSTER", + /* 141 */ "TRANSACTIONS", + /* 142 */ "DISTRIBUTED", + /* 143 */ "LIKE", + /* 144 */ "INDEX", /* 145 */ "FUNCTION", /* 146 */ "INTERVAL", /* 147 */ "TOPIC", /* 148 */ "AS", - /* 149 */ "CONSUMER", - /* 150 */ "GROUP", - /* 151 */ "DESC", - /* 152 */ "DESCRIBE", - /* 153 */ "RESET", - /* 154 */ "QUERY", - /* 155 */ "CACHE", - /* 156 */ "EXPLAIN", - /* 157 */ "ANALYZE", - /* 158 */ "VERBOSE", - /* 159 */ "NK_BOOL", - /* 160 */ "RATIO", - /* 161 */ "NK_FLOAT", - /* 162 */ "COMPACT", - /* 163 */ "VNODES", - /* 164 */ "IN", - /* 165 */ "OUTPUTTYPE", - /* 166 */ "AGGREGATE", - /* 167 */ "BUFSIZE", - /* 168 */ "STREAM", - /* 169 */ "INTO", - /* 170 */ "TRIGGER", - /* 171 */ "AT_ONCE", - /* 172 */ "WINDOW_CLOSE", - /* 173 */ "KILL", - /* 174 */ "CONNECTION", - /* 175 */ "TRANSACTION", - /* 176 */ "BALANCE", - /* 177 */ "VGROUP", - /* 178 */ "MERGE", - /* 179 */ "REDISTRIBUTE", - /* 180 */ "SPLIT", - /* 181 */ "SYNCDB", - /* 182 */ "DELETE", - /* 183 */ "NULL", - /* 184 */ "NK_QUESTION", - /* 185 */ "NK_ARROW", - /* 186 */ "ROWTS", - /* 187 */ "TBNAME", - /* 188 */ "QSTARTTS", - /* 189 */ "QENDTS", - /* 190 */ "WSTARTTS", - /* 191 */ "WENDTS", - /* 192 */ "WDURATION", - /* 193 */ "CAST", - /* 194 */ "NOW", - /* 195 */ "TODAY", - /* 196 */ "TIMEZONE", - /* 197 */ "COUNT", - /* 198 */ "LAST_ROW", - /* 199 */ "BETWEEN", - /* 200 */ "IS", - /* 201 */ "NK_LT", - /* 202 */ "NK_GT", - /* 203 */ "NK_LE", - /* 204 */ "NK_GE", - /* 205 */ "NK_NE", - /* 206 */ "MATCH", - /* 207 */ "NMATCH", - /* 208 */ "CONTAINS", - /* 209 */ "JOIN", - /* 210 */ "INNER", - /* 211 */ "SELECT", - /* 212 */ "DISTINCT", - /* 213 */ "WHERE", - /* 214 */ "PARTITION", - /* 215 */ "BY", - /* 216 */ "SESSION", - /* 217 */ "STATE_WINDOW", - /* 218 */ "SLIDING", - /* 219 */ "FILL", - /* 220 */ "VALUE", - /* 221 */ "NONE", - /* 222 */ "PREV", - /* 223 */ "LINEAR", - /* 224 */ "NEXT", - /* 225 */ "HAVING", - /* 226 */ "RANGE", - /* 227 */ "EVERY", - /* 228 */ "ORDER", - /* 229 */ "SLIMIT", - /* 230 */ "SOFFSET", - /* 231 */ "LIMIT", - /* 232 */ "OFFSET", - /* 233 */ "ASC", - /* 234 */ "NULLS", - /* 235 */ "ID", - /* 236 */ "NK_BITNOT", - /* 237 */ "INSERT", - /* 238 */ "VALUES", - /* 239 */ "IMPORT", - /* 240 */ "NK_SEMI", - /* 241 */ "FILE", - /* 242 */ "cmd", - /* 243 */ "account_options", - /* 244 */ "alter_account_options", - /* 245 */ "literal", - /* 246 */ "alter_account_option", - /* 247 */ "user_name", - /* 248 */ "privileges", - /* 249 */ "priv_level", - /* 250 */ "priv_type_list", - /* 251 */ "priv_type", - /* 252 */ "db_name", - /* 253 */ "dnode_endpoint", - /* 254 */ "not_exists_opt", - /* 255 */ "db_options", - /* 256 */ "exists_opt", - /* 257 */ "alter_db_options", - /* 258 */ "integer_list", - /* 259 */ "variable_list", - /* 260 */ "retention_list", - /* 261 */ "alter_db_option", - /* 262 */ "retention", - /* 263 */ "full_table_name", - /* 264 */ "column_def_list", - /* 265 */ "tags_def_opt", - /* 266 */ "table_options", - /* 267 */ "multi_create_clause", - /* 268 */ "tags_def", - /* 269 */ "multi_drop_clause", - /* 270 */ "alter_table_clause", - /* 271 */ "alter_table_options", - /* 272 */ "column_name", - /* 273 */ "type_name", - /* 274 */ "signed_literal", - /* 275 */ "create_subtable_clause", - /* 276 */ "specific_tags_opt", - /* 277 */ "literal_list", - /* 278 */ "drop_table_clause", - /* 279 */ "col_name_list", - /* 280 */ "table_name", - /* 281 */ "column_def", - /* 282 */ "duration_list", - /* 283 */ "rollup_func_list", - /* 284 */ "alter_table_option", - /* 285 */ "duration_literal", - /* 286 */ "rollup_func_name", - /* 287 */ "function_name", - /* 288 */ "col_name", - /* 289 */ "db_name_cond_opt", - /* 290 */ "like_pattern_opt", - /* 291 */ "table_name_cond", - /* 292 */ "from_db_opt", - /* 293 */ "index_name", - /* 294 */ "index_options", - /* 295 */ "func_list", - /* 296 */ "sliding_opt", - /* 297 */ "func", - /* 298 */ "expression_list", - /* 299 */ "topic_name", - /* 300 */ "query_expression", - /* 301 */ "cgroup_name", - /* 302 */ "analyze_opt", - /* 303 */ "explain_options", - /* 304 */ "agg_func_opt", - /* 305 */ "bufsize_opt", - /* 306 */ "stream_name", - /* 307 */ "stream_options", - /* 308 */ "into_opt", - /* 309 */ "dnode_list", - /* 310 */ "where_clause_opt", - /* 311 */ "signed", - /* 312 */ "literal_func", - /* 313 */ "table_alias", - /* 314 */ "column_alias", - /* 315 */ "expression", - /* 316 */ "pseudo_column", - /* 317 */ "column_reference", - /* 318 */ "function_expression", - /* 319 */ "subquery", - /* 320 */ "star_func", - /* 321 */ "star_func_para_list", - /* 322 */ "noarg_func", - /* 323 */ "other_para_list", - /* 324 */ "star_func_para", - /* 325 */ "predicate", - /* 326 */ "compare_op", - /* 327 */ "in_op", - /* 328 */ "in_predicate_value", - /* 329 */ "boolean_value_expression", - /* 330 */ "boolean_primary", - /* 331 */ "common_expression", - /* 332 */ "from_clause", - /* 333 */ "table_reference_list", - /* 334 */ "table_reference", - /* 335 */ "table_primary", - /* 336 */ "joined_table", - /* 337 */ "alias_opt", - /* 338 */ "parenthesized_joined_table", - /* 339 */ "join_type", - /* 340 */ "search_condition", - /* 341 */ "query_specification", - /* 342 */ "set_quantifier_opt", - /* 343 */ "select_list", - /* 344 */ "partition_by_clause_opt", - /* 345 */ "range_opt", - /* 346 */ "every_opt", - /* 347 */ "fill_opt", - /* 348 */ "twindow_clause_opt", - /* 349 */ "group_by_clause_opt", - /* 350 */ "having_clause_opt", - /* 351 */ "select_sublist", - /* 352 */ "select_item", - /* 353 */ "fill_mode", - /* 354 */ "group_by_list", - /* 355 */ "query_expression_body", - /* 356 */ "order_by_clause_opt", - /* 357 */ "slimit_clause_opt", - /* 358 */ "limit_clause_opt", - /* 359 */ "query_primary", - /* 360 */ "sort_specification_list", - /* 361 */ "sort_specification", - /* 362 */ "ordering_specification_opt", - /* 363 */ "null_ordering_opt", + /* 149 */ "WITH", + /* 150 */ "META", + /* 151 */ "CONSUMER", + /* 152 */ "GROUP", + /* 153 */ "DESC", + /* 154 */ "DESCRIBE", + /* 155 */ "RESET", + /* 156 */ "QUERY", + /* 157 */ "CACHE", + /* 158 */ "EXPLAIN", + /* 159 */ "ANALYZE", + /* 160 */ "VERBOSE", + /* 161 */ "NK_BOOL", + /* 162 */ "RATIO", + /* 163 */ "NK_FLOAT", + /* 164 */ "COMPACT", + /* 165 */ "VNODES", + /* 166 */ "IN", + /* 167 */ "OUTPUTTYPE", + /* 168 */ "AGGREGATE", + /* 169 */ "BUFSIZE", + /* 170 */ "STREAM", + /* 171 */ "INTO", + /* 172 */ "TRIGGER", + /* 173 */ "AT_ONCE", + /* 174 */ "WINDOW_CLOSE", + /* 175 */ "KILL", + /* 176 */ "CONNECTION", + /* 177 */ "TRANSACTION", + /* 178 */ "BALANCE", + /* 179 */ "VGROUP", + /* 180 */ "MERGE", + /* 181 */ "REDISTRIBUTE", + /* 182 */ "SPLIT", + /* 183 */ "SYNCDB", + /* 184 */ "DELETE", + /* 185 */ "NULL", + /* 186 */ "NK_QUESTION", + /* 187 */ "NK_ARROW", + /* 188 */ "ROWTS", + /* 189 */ "TBNAME", + /* 190 */ "QSTARTTS", + /* 191 */ "QENDTS", + /* 192 */ "WSTARTTS", + /* 193 */ "WENDTS", + /* 194 */ "WDURATION", + /* 195 */ "CAST", + /* 196 */ "NOW", + /* 197 */ "TODAY", + /* 198 */ "TIMEZONE", + /* 199 */ "COUNT", + /* 200 */ "LAST_ROW", + /* 201 */ "BETWEEN", + /* 202 */ "IS", + /* 203 */ "NK_LT", + /* 204 */ "NK_GT", + /* 205 */ "NK_LE", + /* 206 */ "NK_GE", + /* 207 */ "NK_NE", + /* 208 */ "MATCH", + /* 209 */ "NMATCH", + /* 210 */ "CONTAINS", + /* 211 */ "JOIN", + /* 212 */ "INNER", + /* 213 */ "SELECT", + /* 214 */ "DISTINCT", + /* 215 */ "WHERE", + /* 216 */ "PARTITION", + /* 217 */ "BY", + /* 218 */ "SESSION", + /* 219 */ "STATE_WINDOW", + /* 220 */ "SLIDING", + /* 221 */ "FILL", + /* 222 */ "VALUE", + /* 223 */ "NONE", + /* 224 */ "PREV", + /* 225 */ "LINEAR", + /* 226 */ "NEXT", + /* 227 */ "HAVING", + /* 228 */ "RANGE", + /* 229 */ "EVERY", + /* 230 */ "ORDER", + /* 231 */ "SLIMIT", + /* 232 */ "SOFFSET", + /* 233 */ "LIMIT", + /* 234 */ "OFFSET", + /* 235 */ "ASC", + /* 236 */ "NULLS", + /* 237 */ "ID", + /* 238 */ "NK_BITNOT", + /* 239 */ "INSERT", + /* 240 */ "VALUES", + /* 241 */ "IMPORT", + /* 242 */ "NK_SEMI", + /* 243 */ "FILE", + /* 244 */ "cmd", + /* 245 */ "account_options", + /* 246 */ "alter_account_options", + /* 247 */ "literal", + /* 248 */ "alter_account_option", + /* 249 */ "user_name", + /* 250 */ "sysinfo_opt", + /* 251 */ "privileges", + /* 252 */ "priv_level", + /* 253 */ "priv_type_list", + /* 254 */ "priv_type", + /* 255 */ "db_name", + /* 256 */ "dnode_endpoint", + /* 257 */ "not_exists_opt", + /* 258 */ "db_options", + /* 259 */ "exists_opt", + /* 260 */ "alter_db_options", + /* 261 */ "integer_list", + /* 262 */ "variable_list", + /* 263 */ "retention_list", + /* 264 */ "alter_db_option", + /* 265 */ "retention", + /* 266 */ "full_table_name", + /* 267 */ "column_def_list", + /* 268 */ "tags_def_opt", + /* 269 */ "table_options", + /* 270 */ "multi_create_clause", + /* 271 */ "tags_def", + /* 272 */ "multi_drop_clause", + /* 273 */ "alter_table_clause", + /* 274 */ "alter_table_options", + /* 275 */ "column_name", + /* 276 */ "type_name", + /* 277 */ "signed_literal", + /* 278 */ "create_subtable_clause", + /* 279 */ "specific_tags_opt", + /* 280 */ "literal_list", + /* 281 */ "drop_table_clause", + /* 282 */ "col_name_list", + /* 283 */ "table_name", + /* 284 */ "column_def", + /* 285 */ "duration_list", + /* 286 */ "rollup_func_list", + /* 287 */ "alter_table_option", + /* 288 */ "duration_literal", + /* 289 */ "rollup_func_name", + /* 290 */ "function_name", + /* 291 */ "col_name", + /* 292 */ "db_name_cond_opt", + /* 293 */ "like_pattern_opt", + /* 294 */ "table_name_cond", + /* 295 */ "from_db_opt", + /* 296 */ "index_name", + /* 297 */ "index_options", + /* 298 */ "func_list", + /* 299 */ "sliding_opt", + /* 300 */ "func", + /* 301 */ "expression_list", + /* 302 */ "topic_name", + /* 303 */ "query_expression", + /* 304 */ "cgroup_name", + /* 305 */ "analyze_opt", + /* 306 */ "explain_options", + /* 307 */ "agg_func_opt", + /* 308 */ "bufsize_opt", + /* 309 */ "stream_name", + /* 310 */ "stream_options", + /* 311 */ "into_opt", + /* 312 */ "dnode_list", + /* 313 */ "where_clause_opt", + /* 314 */ "signed", + /* 315 */ "literal_func", + /* 316 */ "table_alias", + /* 317 */ "column_alias", + /* 318 */ "expression", + /* 319 */ "pseudo_column", + /* 320 */ "column_reference", + /* 321 */ "function_expression", + /* 322 */ "subquery", + /* 323 */ "star_func", + /* 324 */ "star_func_para_list", + /* 325 */ "noarg_func", + /* 326 */ "other_para_list", + /* 327 */ "star_func_para", + /* 328 */ "predicate", + /* 329 */ "compare_op", + /* 330 */ "in_op", + /* 331 */ "in_predicate_value", + /* 332 */ "boolean_value_expression", + /* 333 */ "boolean_primary", + /* 334 */ "common_expression", + /* 335 */ "from_clause_opt", + /* 336 */ "table_reference_list", + /* 337 */ "table_reference", + /* 338 */ "table_primary", + /* 339 */ "joined_table", + /* 340 */ "alias_opt", + /* 341 */ "parenthesized_joined_table", + /* 342 */ "join_type", + /* 343 */ "search_condition", + /* 344 */ "query_specification", + /* 345 */ "set_quantifier_opt", + /* 346 */ "select_list", + /* 347 */ "partition_by_clause_opt", + /* 348 */ "range_opt", + /* 349 */ "every_opt", + /* 350 */ "fill_opt", + /* 351 */ "twindow_clause_opt", + /* 352 */ "group_by_clause_opt", + /* 353 */ "having_clause_opt", + /* 354 */ "select_sublist", + /* 355 */ "select_item", + /* 356 */ "fill_mode", + /* 357 */ "group_by_list", + /* 358 */ "query_expression_body", + /* 359 */ "order_by_clause_opt", + /* 360 */ "slimit_clause_opt", + /* 361 */ "limit_clause_opt", + /* 362 */ "query_primary", + /* 363 */ "sort_specification_list", + /* 364 */ "sort_specification", + /* 365 */ "ordering_specification_opt", + /* 366 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1555,446 +1589,455 @@ static const char *const yyRuleName[] = { /* 21 */ "alter_account_option ::= USERS literal", /* 22 */ "alter_account_option ::= CONNS literal", /* 23 */ "alter_account_option ::= STATE literal", - /* 24 */ "cmd ::= CREATE USER user_name PASS NK_STRING", + /* 24 */ "cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt", /* 25 */ "cmd ::= ALTER USER user_name PASS NK_STRING", - /* 26 */ "cmd ::= ALTER USER user_name PRIVILEGE NK_STRING", - /* 27 */ "cmd ::= DROP USER user_name", - /* 28 */ "cmd ::= GRANT privileges ON priv_level TO user_name", - /* 29 */ "cmd ::= REVOKE privileges ON priv_level FROM user_name", - /* 30 */ "privileges ::= ALL", - /* 31 */ "privileges ::= priv_type_list", - /* 32 */ "priv_type_list ::= priv_type", - /* 33 */ "priv_type_list ::= priv_type_list NK_COMMA priv_type", - /* 34 */ "priv_type ::= READ", - /* 35 */ "priv_type ::= WRITE", - /* 36 */ "priv_level ::= NK_STAR NK_DOT NK_STAR", - /* 37 */ "priv_level ::= db_name NK_DOT NK_STAR", - /* 38 */ "cmd ::= CREATE DNODE dnode_endpoint", - /* 39 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", - /* 40 */ "cmd ::= DROP DNODE NK_INTEGER", - /* 41 */ "cmd ::= DROP DNODE dnode_endpoint", - /* 42 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", - /* 43 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", - /* 44 */ "cmd ::= ALTER ALL DNODES NK_STRING", - /* 45 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", - /* 46 */ "dnode_endpoint ::= NK_STRING", - /* 47 */ "dnode_endpoint ::= NK_ID", - /* 48 */ "dnode_endpoint ::= NK_IPTOKEN", - /* 49 */ "cmd ::= ALTER LOCAL NK_STRING", - /* 50 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", - /* 51 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", - /* 52 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", - /* 53 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", - /* 54 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", - /* 55 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", - /* 56 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", - /* 57 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", - /* 58 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", - /* 59 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 60 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 61 */ "cmd ::= USE db_name", - /* 62 */ "cmd ::= ALTER DATABASE db_name alter_db_options", - /* 63 */ "not_exists_opt ::= IF NOT EXISTS", - /* 64 */ "not_exists_opt ::=", - /* 65 */ "exists_opt ::= IF EXISTS", - /* 66 */ "exists_opt ::=", - /* 67 */ "db_options ::=", - /* 68 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 69 */ "db_options ::= db_options CACHELAST NK_INTEGER", - /* 70 */ "db_options ::= db_options COMP NK_INTEGER", - /* 71 */ "db_options ::= db_options DURATION NK_INTEGER", - /* 72 */ "db_options ::= db_options DURATION NK_VARIABLE", - /* 73 */ "db_options ::= db_options FSYNC NK_INTEGER", - /* 74 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 75 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 76 */ "db_options ::= db_options KEEP integer_list", - /* 77 */ "db_options ::= db_options KEEP variable_list", - /* 78 */ "db_options ::= db_options PAGES NK_INTEGER", - /* 79 */ "db_options ::= db_options PAGESIZE NK_INTEGER", - /* 80 */ "db_options ::= db_options PRECISION NK_STRING", - /* 81 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 82 */ "db_options ::= db_options STRICT NK_INTEGER", - /* 83 */ "db_options ::= db_options WAL NK_INTEGER", - /* 84 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 85 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 86 */ "db_options ::= db_options RETENTIONS retention_list", - /* 87 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 88 */ "alter_db_options ::= alter_db_option", - /* 89 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 90 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 91 */ "alter_db_option ::= CACHELAST NK_INTEGER", - /* 92 */ "alter_db_option ::= FSYNC NK_INTEGER", - /* 93 */ "alter_db_option ::= KEEP integer_list", - /* 94 */ "alter_db_option ::= KEEP variable_list", - /* 95 */ "alter_db_option ::= PAGES NK_INTEGER", - /* 96 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 97 */ "alter_db_option ::= STRICT NK_INTEGER", - /* 98 */ "alter_db_option ::= WAL NK_INTEGER", - /* 99 */ "integer_list ::= NK_INTEGER", - /* 100 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 101 */ "variable_list ::= NK_VARIABLE", - /* 102 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 103 */ "retention_list ::= retention", - /* 104 */ "retention_list ::= retention_list NK_COMMA retention", - /* 105 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 106 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 107 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 108 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 109 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 110 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 111 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 112 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 113 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 114 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 115 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 116 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 117 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 118 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 119 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 120 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 121 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 122 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 123 */ "multi_create_clause ::= create_subtable_clause", - /* 124 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 125 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options", - /* 126 */ "multi_drop_clause ::= drop_table_clause", - /* 127 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", - /* 128 */ "drop_table_clause ::= exists_opt full_table_name", - /* 129 */ "specific_tags_opt ::=", - /* 130 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", - /* 131 */ "full_table_name ::= table_name", - /* 132 */ "full_table_name ::= db_name NK_DOT table_name", - /* 133 */ "column_def_list ::= column_def", - /* 134 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 135 */ "column_def ::= column_name type_name", - /* 136 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 137 */ "type_name ::= BOOL", - /* 138 */ "type_name ::= TINYINT", - /* 139 */ "type_name ::= SMALLINT", - /* 140 */ "type_name ::= INT", - /* 141 */ "type_name ::= INTEGER", - /* 142 */ "type_name ::= BIGINT", - /* 143 */ "type_name ::= FLOAT", - /* 144 */ "type_name ::= DOUBLE", - /* 145 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 146 */ "type_name ::= TIMESTAMP", - /* 147 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 148 */ "type_name ::= TINYINT UNSIGNED", - /* 149 */ "type_name ::= SMALLINT UNSIGNED", - /* 150 */ "type_name ::= INT UNSIGNED", - /* 151 */ "type_name ::= BIGINT UNSIGNED", - /* 152 */ "type_name ::= JSON", - /* 153 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 154 */ "type_name ::= MEDIUMBLOB", - /* 155 */ "type_name ::= BLOB", - /* 156 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 157 */ "type_name ::= DECIMAL", - /* 158 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 159 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 160 */ "tags_def_opt ::=", - /* 161 */ "tags_def_opt ::= tags_def", - /* 162 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 163 */ "table_options ::=", - /* 164 */ "table_options ::= table_options COMMENT NK_STRING", - /* 165 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 166 */ "table_options ::= table_options WATERMARK duration_list", - /* 167 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 168 */ "table_options ::= table_options TTL NK_INTEGER", - /* 169 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 170 */ "alter_table_options ::= alter_table_option", - /* 171 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 172 */ "alter_table_option ::= COMMENT NK_STRING", - /* 173 */ "alter_table_option ::= TTL NK_INTEGER", - /* 174 */ "duration_list ::= duration_literal", - /* 175 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 176 */ "rollup_func_list ::= rollup_func_name", - /* 177 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 178 */ "rollup_func_name ::= function_name", - /* 179 */ "rollup_func_name ::= FIRST", - /* 180 */ "rollup_func_name ::= LAST", - /* 181 */ "col_name_list ::= col_name", - /* 182 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 183 */ "col_name ::= column_name", - /* 184 */ "cmd ::= SHOW DNODES", - /* 185 */ "cmd ::= SHOW USERS", - /* 186 */ "cmd ::= SHOW DATABASES", - /* 187 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 188 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 189 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 190 */ "cmd ::= SHOW MNODES", - /* 191 */ "cmd ::= SHOW MODULES", - /* 192 */ "cmd ::= SHOW QNODES", - /* 193 */ "cmd ::= SHOW FUNCTIONS", - /* 194 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 195 */ "cmd ::= SHOW STREAMS", - /* 196 */ "cmd ::= SHOW ACCOUNTS", - /* 197 */ "cmd ::= SHOW APPS", - /* 198 */ "cmd ::= SHOW CONNECTIONS", - /* 199 */ "cmd ::= SHOW LICENCE", - /* 200 */ "cmd ::= SHOW GRANTS", - /* 201 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 202 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 203 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 204 */ "cmd ::= SHOW QUERIES", - /* 205 */ "cmd ::= SHOW SCORES", - /* 206 */ "cmd ::= SHOW TOPICS", - /* 207 */ "cmd ::= SHOW VARIABLES", - /* 208 */ "cmd ::= SHOW BNODES", - /* 209 */ "cmd ::= SHOW SNODES", - /* 210 */ "cmd ::= SHOW CLUSTER", - /* 211 */ "cmd ::= SHOW TRANSACTIONS", - /* 212 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 213 */ "db_name_cond_opt ::=", - /* 214 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 215 */ "like_pattern_opt ::=", - /* 216 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 217 */ "table_name_cond ::= table_name", - /* 218 */ "from_db_opt ::=", - /* 219 */ "from_db_opt ::= FROM db_name", - /* 220 */ "cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options", - /* 221 */ "cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP", - /* 222 */ "cmd ::= DROP INDEX exists_opt index_name ON table_name", - /* 223 */ "index_options ::=", - /* 224 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", - /* 225 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", - /* 226 */ "func_list ::= func", - /* 227 */ "func_list ::= func_list NK_COMMA func", - /* 228 */ "func ::= function_name NK_LP expression_list NK_RP", - /* 229 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", - /* 230 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 231 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 232 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 233 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 234 */ "cmd ::= DESC full_table_name", - /* 235 */ "cmd ::= DESCRIBE full_table_name", - /* 236 */ "cmd ::= RESET QUERY CACHE", - /* 237 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", - /* 238 */ "analyze_opt ::=", - /* 239 */ "analyze_opt ::= ANALYZE", - /* 240 */ "explain_options ::=", - /* 241 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 242 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 243 */ "cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP", - /* 244 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", - /* 245 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 246 */ "agg_func_opt ::=", - /* 247 */ "agg_func_opt ::= AGGREGATE", - /* 248 */ "bufsize_opt ::=", - /* 249 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 250 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression", - /* 251 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 252 */ "into_opt ::=", - /* 253 */ "into_opt ::= INTO full_table_name", - /* 254 */ "stream_options ::=", - /* 255 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 256 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 257 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 258 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 259 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 260 */ "cmd ::= KILL QUERY NK_STRING", - /* 261 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 262 */ "cmd ::= BALANCE VGROUP", - /* 263 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 264 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 265 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 266 */ "dnode_list ::= DNODE NK_INTEGER", - /* 267 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 268 */ "cmd ::= SYNCDB db_name REPLICA", - /* 269 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 270 */ "cmd ::= query_expression", - /* 271 */ "literal ::= NK_INTEGER", - /* 272 */ "literal ::= NK_FLOAT", - /* 273 */ "literal ::= NK_STRING", - /* 274 */ "literal ::= NK_BOOL", - /* 275 */ "literal ::= TIMESTAMP NK_STRING", - /* 276 */ "literal ::= duration_literal", - /* 277 */ "literal ::= NULL", - /* 278 */ "literal ::= NK_QUESTION", - /* 279 */ "duration_literal ::= NK_VARIABLE", - /* 280 */ "signed ::= NK_INTEGER", - /* 281 */ "signed ::= NK_PLUS NK_INTEGER", - /* 282 */ "signed ::= NK_MINUS NK_INTEGER", - /* 283 */ "signed ::= NK_FLOAT", - /* 284 */ "signed ::= NK_PLUS NK_FLOAT", - /* 285 */ "signed ::= NK_MINUS NK_FLOAT", - /* 286 */ "signed_literal ::= signed", - /* 287 */ "signed_literal ::= NK_STRING", - /* 288 */ "signed_literal ::= NK_BOOL", - /* 289 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 290 */ "signed_literal ::= duration_literal", - /* 291 */ "signed_literal ::= NULL", - /* 292 */ "signed_literal ::= literal_func", - /* 293 */ "literal_list ::= signed_literal", - /* 294 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 295 */ "db_name ::= NK_ID", - /* 296 */ "table_name ::= NK_ID", - /* 297 */ "column_name ::= NK_ID", - /* 298 */ "function_name ::= NK_ID", - /* 299 */ "table_alias ::= NK_ID", - /* 300 */ "column_alias ::= NK_ID", - /* 301 */ "user_name ::= NK_ID", - /* 302 */ "index_name ::= NK_ID", - /* 303 */ "topic_name ::= NK_ID", - /* 304 */ "stream_name ::= NK_ID", - /* 305 */ "cgroup_name ::= NK_ID", - /* 306 */ "expression ::= literal", - /* 307 */ "expression ::= pseudo_column", - /* 308 */ "expression ::= column_reference", - /* 309 */ "expression ::= function_expression", - /* 310 */ "expression ::= subquery", - /* 311 */ "expression ::= NK_LP expression NK_RP", - /* 312 */ "expression ::= NK_PLUS expression", - /* 313 */ "expression ::= NK_MINUS expression", - /* 314 */ "expression ::= expression NK_PLUS expression", - /* 315 */ "expression ::= expression NK_MINUS expression", - /* 316 */ "expression ::= expression NK_STAR expression", - /* 317 */ "expression ::= expression NK_SLASH expression", - /* 318 */ "expression ::= expression NK_REM expression", - /* 319 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 320 */ "expression_list ::= expression", - /* 321 */ "expression_list ::= expression_list NK_COMMA expression", - /* 322 */ "column_reference ::= column_name", - /* 323 */ "column_reference ::= table_name NK_DOT column_name", - /* 324 */ "pseudo_column ::= ROWTS", - /* 325 */ "pseudo_column ::= TBNAME", - /* 326 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 327 */ "pseudo_column ::= QSTARTTS", - /* 328 */ "pseudo_column ::= QENDTS", - /* 329 */ "pseudo_column ::= WSTARTTS", - /* 330 */ "pseudo_column ::= WENDTS", - /* 331 */ "pseudo_column ::= WDURATION", - /* 332 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 333 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 334 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", - /* 335 */ "function_expression ::= literal_func", - /* 336 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 337 */ "literal_func ::= NOW", - /* 338 */ "noarg_func ::= NOW", - /* 339 */ "noarg_func ::= TODAY", - /* 340 */ "noarg_func ::= TIMEZONE", - /* 341 */ "star_func ::= COUNT", - /* 342 */ "star_func ::= FIRST", - /* 343 */ "star_func ::= LAST", - /* 344 */ "star_func ::= LAST_ROW", - /* 345 */ "star_func_para_list ::= NK_STAR", - /* 346 */ "star_func_para_list ::= other_para_list", - /* 347 */ "other_para_list ::= star_func_para", - /* 348 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 349 */ "star_func_para ::= expression", - /* 350 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 351 */ "predicate ::= expression compare_op expression", - /* 352 */ "predicate ::= expression BETWEEN expression AND expression", - /* 353 */ "predicate ::= expression NOT BETWEEN expression AND expression", - /* 354 */ "predicate ::= expression IS NULL", - /* 355 */ "predicate ::= expression IS NOT NULL", - /* 356 */ "predicate ::= expression in_op in_predicate_value", - /* 357 */ "compare_op ::= NK_LT", - /* 358 */ "compare_op ::= NK_GT", - /* 359 */ "compare_op ::= NK_LE", - /* 360 */ "compare_op ::= NK_GE", - /* 361 */ "compare_op ::= NK_NE", - /* 362 */ "compare_op ::= NK_EQ", - /* 363 */ "compare_op ::= LIKE", - /* 364 */ "compare_op ::= NOT LIKE", - /* 365 */ "compare_op ::= MATCH", - /* 366 */ "compare_op ::= NMATCH", - /* 367 */ "compare_op ::= CONTAINS", - /* 368 */ "in_op ::= IN", - /* 369 */ "in_op ::= NOT IN", - /* 370 */ "in_predicate_value ::= NK_LP expression_list NK_RP", - /* 371 */ "boolean_value_expression ::= boolean_primary", - /* 372 */ "boolean_value_expression ::= NOT boolean_primary", - /* 373 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 374 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 375 */ "boolean_primary ::= predicate", - /* 376 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 377 */ "common_expression ::= expression", - /* 378 */ "common_expression ::= boolean_value_expression", - /* 379 */ "from_clause ::= FROM table_reference_list", - /* 380 */ "table_reference_list ::= table_reference", - /* 381 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 382 */ "table_reference ::= table_primary", - /* 383 */ "table_reference ::= joined_table", - /* 384 */ "table_primary ::= table_name alias_opt", - /* 385 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 386 */ "table_primary ::= subquery alias_opt", - /* 387 */ "table_primary ::= parenthesized_joined_table", - /* 388 */ "alias_opt ::=", - /* 389 */ "alias_opt ::= table_alias", - /* 390 */ "alias_opt ::= AS table_alias", - /* 391 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 392 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 393 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 394 */ "join_type ::=", - /* 395 */ "join_type ::= INNER", - /* 396 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 397 */ "set_quantifier_opt ::=", - /* 398 */ "set_quantifier_opt ::= DISTINCT", - /* 399 */ "set_quantifier_opt ::= ALL", - /* 400 */ "select_list ::= NK_STAR", - /* 401 */ "select_list ::= select_sublist", - /* 402 */ "select_sublist ::= select_item", - /* 403 */ "select_sublist ::= select_sublist NK_COMMA select_item", - /* 404 */ "select_item ::= common_expression", - /* 405 */ "select_item ::= common_expression column_alias", - /* 406 */ "select_item ::= common_expression AS column_alias", - /* 407 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 408 */ "where_clause_opt ::=", - /* 409 */ "where_clause_opt ::= WHERE search_condition", - /* 410 */ "partition_by_clause_opt ::=", - /* 411 */ "partition_by_clause_opt ::= PARTITION BY expression_list", - /* 412 */ "twindow_clause_opt ::=", - /* 413 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 414 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", - /* 415 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 416 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 417 */ "sliding_opt ::=", - /* 418 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 419 */ "fill_opt ::=", - /* 420 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 421 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 422 */ "fill_mode ::= NONE", - /* 423 */ "fill_mode ::= PREV", - /* 424 */ "fill_mode ::= NULL", - /* 425 */ "fill_mode ::= LINEAR", - /* 426 */ "fill_mode ::= NEXT", - /* 427 */ "group_by_clause_opt ::=", - /* 428 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 429 */ "group_by_list ::= expression", - /* 430 */ "group_by_list ::= group_by_list NK_COMMA expression", - /* 431 */ "having_clause_opt ::=", - /* 432 */ "having_clause_opt ::= HAVING search_condition", - /* 433 */ "range_opt ::=", - /* 434 */ "range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP", - /* 435 */ "every_opt ::=", - /* 436 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 437 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 438 */ "query_expression_body ::= query_primary", - /* 439 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", - /* 440 */ "query_expression_body ::= query_expression_body UNION query_expression_body", - /* 441 */ "query_primary ::= query_specification", - /* 442 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", - /* 443 */ "order_by_clause_opt ::=", - /* 444 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 445 */ "slimit_clause_opt ::=", - /* 446 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 447 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 448 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 449 */ "limit_clause_opt ::=", - /* 450 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 451 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 452 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 453 */ "subquery ::= NK_LP query_expression NK_RP", - /* 454 */ "search_condition ::= common_expression", - /* 455 */ "sort_specification_list ::= sort_specification", - /* 456 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 457 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", - /* 458 */ "ordering_specification_opt ::=", - /* 459 */ "ordering_specification_opt ::= ASC", - /* 460 */ "ordering_specification_opt ::= DESC", - /* 461 */ "null_ordering_opt ::=", - /* 462 */ "null_ordering_opt ::= NULLS FIRST", - /* 463 */ "null_ordering_opt ::= NULLS LAST", + /* 26 */ "cmd ::= ALTER USER user_name ENABLE NK_INTEGER", + /* 27 */ "cmd ::= ALTER USER user_name SYSINFO NK_INTEGER", + /* 28 */ "cmd ::= DROP USER user_name", + /* 29 */ "sysinfo_opt ::=", + /* 30 */ "sysinfo_opt ::= SYSINFO NK_INTEGER", + /* 31 */ "cmd ::= GRANT privileges ON priv_level TO user_name", + /* 32 */ "cmd ::= REVOKE privileges ON priv_level FROM user_name", + /* 33 */ "privileges ::= ALL", + /* 34 */ "privileges ::= priv_type_list", + /* 35 */ "priv_type_list ::= priv_type", + /* 36 */ "priv_type_list ::= priv_type_list NK_COMMA priv_type", + /* 37 */ "priv_type ::= READ", + /* 38 */ "priv_type ::= WRITE", + /* 39 */ "priv_level ::= NK_STAR NK_DOT NK_STAR", + /* 40 */ "priv_level ::= db_name NK_DOT NK_STAR", + /* 41 */ "cmd ::= CREATE DNODE dnode_endpoint", + /* 42 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", + /* 43 */ "cmd ::= DROP DNODE NK_INTEGER", + /* 44 */ "cmd ::= DROP DNODE dnode_endpoint", + /* 45 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 46 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 47 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 48 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 49 */ "dnode_endpoint ::= NK_STRING", + /* 50 */ "dnode_endpoint ::= NK_ID", + /* 51 */ "dnode_endpoint ::= NK_IPTOKEN", + /* 52 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 53 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 54 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 55 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 56 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", + /* 57 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", + /* 58 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", + /* 59 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", + /* 60 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", + /* 61 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", + /* 62 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 63 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 64 */ "cmd ::= USE db_name", + /* 65 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 66 */ "not_exists_opt ::= IF NOT EXISTS", + /* 67 */ "not_exists_opt ::=", + /* 68 */ "exists_opt ::= IF EXISTS", + /* 69 */ "exists_opt ::=", + /* 70 */ "db_options ::=", + /* 71 */ "db_options ::= db_options BUFFER NK_INTEGER", + /* 72 */ "db_options ::= db_options CACHELAST NK_INTEGER", + /* 73 */ "db_options ::= db_options COMP NK_INTEGER", + /* 74 */ "db_options ::= db_options DURATION NK_INTEGER", + /* 75 */ "db_options ::= db_options DURATION NK_VARIABLE", + /* 76 */ "db_options ::= db_options FSYNC NK_INTEGER", + /* 77 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 78 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 79 */ "db_options ::= db_options KEEP integer_list", + /* 80 */ "db_options ::= db_options KEEP variable_list", + /* 81 */ "db_options ::= db_options PAGES NK_INTEGER", + /* 82 */ "db_options ::= db_options PAGESIZE NK_INTEGER", + /* 83 */ "db_options ::= db_options PRECISION NK_STRING", + /* 84 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 85 */ "db_options ::= db_options STRICT NK_INTEGER", + /* 86 */ "db_options ::= db_options WAL NK_INTEGER", + /* 87 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 88 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 89 */ "db_options ::= db_options RETENTIONS retention_list", + /* 90 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 91 */ "alter_db_options ::= alter_db_option", + /* 92 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 93 */ "alter_db_option ::= BUFFER NK_INTEGER", + /* 94 */ "alter_db_option ::= CACHELAST NK_INTEGER", + /* 95 */ "alter_db_option ::= FSYNC NK_INTEGER", + /* 96 */ "alter_db_option ::= KEEP integer_list", + /* 97 */ "alter_db_option ::= KEEP variable_list", + /* 98 */ "alter_db_option ::= PAGES NK_INTEGER", + /* 99 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 100 */ "alter_db_option ::= STRICT NK_INTEGER", + /* 101 */ "alter_db_option ::= WAL NK_INTEGER", + /* 102 */ "integer_list ::= NK_INTEGER", + /* 103 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 104 */ "variable_list ::= NK_VARIABLE", + /* 105 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 106 */ "retention_list ::= retention", + /* 107 */ "retention_list ::= retention_list NK_COMMA retention", + /* 108 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 109 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 110 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 111 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 112 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 113 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 114 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 115 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 116 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 117 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 118 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 119 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 120 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 121 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 122 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 123 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 124 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 125 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 126 */ "multi_create_clause ::= create_subtable_clause", + /* 127 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 128 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options", + /* 129 */ "multi_drop_clause ::= drop_table_clause", + /* 130 */ "multi_drop_clause ::= multi_drop_clause drop_table_clause", + /* 131 */ "drop_table_clause ::= exists_opt full_table_name", + /* 132 */ "specific_tags_opt ::=", + /* 133 */ "specific_tags_opt ::= NK_LP col_name_list NK_RP", + /* 134 */ "full_table_name ::= table_name", + /* 135 */ "full_table_name ::= db_name NK_DOT table_name", + /* 136 */ "column_def_list ::= column_def", + /* 137 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 138 */ "column_def ::= column_name type_name", + /* 139 */ "column_def ::= column_name type_name COMMENT NK_STRING", + /* 140 */ "type_name ::= BOOL", + /* 141 */ "type_name ::= TINYINT", + /* 142 */ "type_name ::= SMALLINT", + /* 143 */ "type_name ::= INT", + /* 144 */ "type_name ::= INTEGER", + /* 145 */ "type_name ::= BIGINT", + /* 146 */ "type_name ::= FLOAT", + /* 147 */ "type_name ::= DOUBLE", + /* 148 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 149 */ "type_name ::= TIMESTAMP", + /* 150 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 151 */ "type_name ::= TINYINT UNSIGNED", + /* 152 */ "type_name ::= SMALLINT UNSIGNED", + /* 153 */ "type_name ::= INT UNSIGNED", + /* 154 */ "type_name ::= BIGINT UNSIGNED", + /* 155 */ "type_name ::= JSON", + /* 156 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 157 */ "type_name ::= MEDIUMBLOB", + /* 158 */ "type_name ::= BLOB", + /* 159 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 160 */ "type_name ::= DECIMAL", + /* 161 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 162 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 163 */ "tags_def_opt ::=", + /* 164 */ "tags_def_opt ::= tags_def", + /* 165 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 166 */ "table_options ::=", + /* 167 */ "table_options ::= table_options COMMENT NK_STRING", + /* 168 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 169 */ "table_options ::= table_options WATERMARK duration_list", + /* 170 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 171 */ "table_options ::= table_options TTL NK_INTEGER", + /* 172 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 173 */ "alter_table_options ::= alter_table_option", + /* 174 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 175 */ "alter_table_option ::= COMMENT NK_STRING", + /* 176 */ "alter_table_option ::= TTL NK_INTEGER", + /* 177 */ "duration_list ::= duration_literal", + /* 178 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 179 */ "rollup_func_list ::= rollup_func_name", + /* 180 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 181 */ "rollup_func_name ::= function_name", + /* 182 */ "rollup_func_name ::= FIRST", + /* 183 */ "rollup_func_name ::= LAST", + /* 184 */ "col_name_list ::= col_name", + /* 185 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 186 */ "col_name ::= column_name", + /* 187 */ "cmd ::= SHOW DNODES", + /* 188 */ "cmd ::= SHOW USERS", + /* 189 */ "cmd ::= SHOW DATABASES", + /* 190 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 191 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 192 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 193 */ "cmd ::= SHOW MNODES", + /* 194 */ "cmd ::= SHOW MODULES", + /* 195 */ "cmd ::= SHOW QNODES", + /* 196 */ "cmd ::= SHOW FUNCTIONS", + /* 197 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 198 */ "cmd ::= SHOW STREAMS", + /* 199 */ "cmd ::= SHOW ACCOUNTS", + /* 200 */ "cmd ::= SHOW APPS", + /* 201 */ "cmd ::= SHOW CONNECTIONS", + /* 202 */ "cmd ::= SHOW LICENCE", + /* 203 */ "cmd ::= SHOW GRANTS", + /* 204 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 205 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 206 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 207 */ "cmd ::= SHOW QUERIES", + /* 208 */ "cmd ::= SHOW SCORES", + /* 209 */ "cmd ::= SHOW TOPICS", + /* 210 */ "cmd ::= SHOW VARIABLES", + /* 211 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 212 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES", + /* 213 */ "cmd ::= SHOW BNODES", + /* 214 */ "cmd ::= SHOW SNODES", + /* 215 */ "cmd ::= SHOW CLUSTER", + /* 216 */ "cmd ::= SHOW TRANSACTIONS", + /* 217 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 218 */ "db_name_cond_opt ::=", + /* 219 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 220 */ "like_pattern_opt ::=", + /* 221 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 222 */ "table_name_cond ::= table_name", + /* 223 */ "from_db_opt ::=", + /* 224 */ "from_db_opt ::= FROM db_name", + /* 225 */ "cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options", + /* 226 */ "cmd ::= DROP INDEX exists_opt index_name", + /* 227 */ "index_options ::=", + /* 228 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt", + /* 229 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt", + /* 230 */ "func_list ::= func", + /* 231 */ "func_list ::= func_list NK_COMMA func", + /* 232 */ "func ::= function_name NK_LP expression_list NK_RP", + /* 233 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression", + /* 234 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 235 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 236 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 237 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 238 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 239 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 240 */ "cmd ::= DESC full_table_name", + /* 241 */ "cmd ::= DESCRIBE full_table_name", + /* 242 */ "cmd ::= RESET QUERY CACHE", + /* 243 */ "cmd ::= EXPLAIN analyze_opt explain_options query_expression", + /* 244 */ "analyze_opt ::=", + /* 245 */ "analyze_opt ::= ANALYZE", + /* 246 */ "explain_options ::=", + /* 247 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 248 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 249 */ "cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP", + /* 250 */ "cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt", + /* 251 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 252 */ "agg_func_opt ::=", + /* 253 */ "agg_func_opt ::= AGGREGATE", + /* 254 */ "bufsize_opt ::=", + /* 255 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 256 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression", + /* 257 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 258 */ "into_opt ::=", + /* 259 */ "into_opt ::= INTO full_table_name", + /* 260 */ "stream_options ::=", + /* 261 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 262 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 263 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 264 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 265 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 266 */ "cmd ::= KILL QUERY NK_STRING", + /* 267 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 268 */ "cmd ::= BALANCE VGROUP", + /* 269 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 270 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 271 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 272 */ "dnode_list ::= DNODE NK_INTEGER", + /* 273 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 274 */ "cmd ::= SYNCDB db_name REPLICA", + /* 275 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 276 */ "cmd ::= query_expression", + /* 277 */ "literal ::= NK_INTEGER", + /* 278 */ "literal ::= NK_FLOAT", + /* 279 */ "literal ::= NK_STRING", + /* 280 */ "literal ::= NK_BOOL", + /* 281 */ "literal ::= TIMESTAMP NK_STRING", + /* 282 */ "literal ::= duration_literal", + /* 283 */ "literal ::= NULL", + /* 284 */ "literal ::= NK_QUESTION", + /* 285 */ "duration_literal ::= NK_VARIABLE", + /* 286 */ "signed ::= NK_INTEGER", + /* 287 */ "signed ::= NK_PLUS NK_INTEGER", + /* 288 */ "signed ::= NK_MINUS NK_INTEGER", + /* 289 */ "signed ::= NK_FLOAT", + /* 290 */ "signed ::= NK_PLUS NK_FLOAT", + /* 291 */ "signed ::= NK_MINUS NK_FLOAT", + /* 292 */ "signed_literal ::= signed", + /* 293 */ "signed_literal ::= NK_STRING", + /* 294 */ "signed_literal ::= NK_BOOL", + /* 295 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 296 */ "signed_literal ::= duration_literal", + /* 297 */ "signed_literal ::= NULL", + /* 298 */ "signed_literal ::= literal_func", + /* 299 */ "literal_list ::= signed_literal", + /* 300 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 301 */ "db_name ::= NK_ID", + /* 302 */ "table_name ::= NK_ID", + /* 303 */ "column_name ::= NK_ID", + /* 304 */ "function_name ::= NK_ID", + /* 305 */ "table_alias ::= NK_ID", + /* 306 */ "column_alias ::= NK_ID", + /* 307 */ "user_name ::= NK_ID", + /* 308 */ "index_name ::= NK_ID", + /* 309 */ "topic_name ::= NK_ID", + /* 310 */ "stream_name ::= NK_ID", + /* 311 */ "cgroup_name ::= NK_ID", + /* 312 */ "expression ::= literal", + /* 313 */ "expression ::= pseudo_column", + /* 314 */ "expression ::= column_reference", + /* 315 */ "expression ::= function_expression", + /* 316 */ "expression ::= subquery", + /* 317 */ "expression ::= NK_LP expression NK_RP", + /* 318 */ "expression ::= NK_PLUS expression", + /* 319 */ "expression ::= NK_MINUS expression", + /* 320 */ "expression ::= expression NK_PLUS expression", + /* 321 */ "expression ::= expression NK_MINUS expression", + /* 322 */ "expression ::= expression NK_STAR expression", + /* 323 */ "expression ::= expression NK_SLASH expression", + /* 324 */ "expression ::= expression NK_REM expression", + /* 325 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 326 */ "expression ::= expression NK_BITAND expression", + /* 327 */ "expression ::= expression NK_BITOR expression", + /* 328 */ "expression_list ::= expression", + /* 329 */ "expression_list ::= expression_list NK_COMMA expression", + /* 330 */ "column_reference ::= column_name", + /* 331 */ "column_reference ::= table_name NK_DOT column_name", + /* 332 */ "pseudo_column ::= ROWTS", + /* 333 */ "pseudo_column ::= TBNAME", + /* 334 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 335 */ "pseudo_column ::= QSTARTTS", + /* 336 */ "pseudo_column ::= QENDTS", + /* 337 */ "pseudo_column ::= WSTARTTS", + /* 338 */ "pseudo_column ::= WENDTS", + /* 339 */ "pseudo_column ::= WDURATION", + /* 340 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 341 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 342 */ "function_expression ::= CAST NK_LP expression AS type_name NK_RP", + /* 343 */ "function_expression ::= literal_func", + /* 344 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 345 */ "literal_func ::= NOW", + /* 346 */ "noarg_func ::= NOW", + /* 347 */ "noarg_func ::= TODAY", + /* 348 */ "noarg_func ::= TIMEZONE", + /* 349 */ "star_func ::= COUNT", + /* 350 */ "star_func ::= FIRST", + /* 351 */ "star_func ::= LAST", + /* 352 */ "star_func ::= LAST_ROW", + /* 353 */ "star_func_para_list ::= NK_STAR", + /* 354 */ "star_func_para_list ::= other_para_list", + /* 355 */ "other_para_list ::= star_func_para", + /* 356 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 357 */ "star_func_para ::= expression", + /* 358 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 359 */ "predicate ::= expression compare_op expression", + /* 360 */ "predicate ::= expression BETWEEN expression AND expression", + /* 361 */ "predicate ::= expression NOT BETWEEN expression AND expression", + /* 362 */ "predicate ::= expression IS NULL", + /* 363 */ "predicate ::= expression IS NOT NULL", + /* 364 */ "predicate ::= expression in_op in_predicate_value", + /* 365 */ "compare_op ::= NK_LT", + /* 366 */ "compare_op ::= NK_GT", + /* 367 */ "compare_op ::= NK_LE", + /* 368 */ "compare_op ::= NK_GE", + /* 369 */ "compare_op ::= NK_NE", + /* 370 */ "compare_op ::= NK_EQ", + /* 371 */ "compare_op ::= LIKE", + /* 372 */ "compare_op ::= NOT LIKE", + /* 373 */ "compare_op ::= MATCH", + /* 374 */ "compare_op ::= NMATCH", + /* 375 */ "compare_op ::= CONTAINS", + /* 376 */ "in_op ::= IN", + /* 377 */ "in_op ::= NOT IN", + /* 378 */ "in_predicate_value ::= NK_LP expression_list NK_RP", + /* 379 */ "boolean_value_expression ::= boolean_primary", + /* 380 */ "boolean_value_expression ::= NOT boolean_primary", + /* 381 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 382 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 383 */ "boolean_primary ::= predicate", + /* 384 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 385 */ "common_expression ::= expression", + /* 386 */ "common_expression ::= boolean_value_expression", + /* 387 */ "from_clause_opt ::=", + /* 388 */ "from_clause_opt ::= FROM table_reference_list", + /* 389 */ "table_reference_list ::= table_reference", + /* 390 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 391 */ "table_reference ::= table_primary", + /* 392 */ "table_reference ::= joined_table", + /* 393 */ "table_primary ::= table_name alias_opt", + /* 394 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 395 */ "table_primary ::= subquery alias_opt", + /* 396 */ "table_primary ::= parenthesized_joined_table", + /* 397 */ "alias_opt ::=", + /* 398 */ "alias_opt ::= table_alias", + /* 399 */ "alias_opt ::= AS table_alias", + /* 400 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 401 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 402 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 403 */ "join_type ::=", + /* 404 */ "join_type ::= INNER", + /* 405 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 406 */ "set_quantifier_opt ::=", + /* 407 */ "set_quantifier_opt ::= DISTINCT", + /* 408 */ "set_quantifier_opt ::= ALL", + /* 409 */ "select_list ::= NK_STAR", + /* 410 */ "select_list ::= select_sublist", + /* 411 */ "select_sublist ::= select_item", + /* 412 */ "select_sublist ::= select_sublist NK_COMMA select_item", + /* 413 */ "select_item ::= common_expression", + /* 414 */ "select_item ::= common_expression column_alias", + /* 415 */ "select_item ::= common_expression AS column_alias", + /* 416 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 417 */ "where_clause_opt ::=", + /* 418 */ "where_clause_opt ::= WHERE search_condition", + /* 419 */ "partition_by_clause_opt ::=", + /* 420 */ "partition_by_clause_opt ::= PARTITION BY expression_list", + /* 421 */ "twindow_clause_opt ::=", + /* 422 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 423 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP", + /* 424 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 425 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 426 */ "sliding_opt ::=", + /* 427 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 428 */ "fill_opt ::=", + /* 429 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 430 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", + /* 431 */ "fill_mode ::= NONE", + /* 432 */ "fill_mode ::= PREV", + /* 433 */ "fill_mode ::= NULL", + /* 434 */ "fill_mode ::= LINEAR", + /* 435 */ "fill_mode ::= NEXT", + /* 436 */ "group_by_clause_opt ::=", + /* 437 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 438 */ "group_by_list ::= expression", + /* 439 */ "group_by_list ::= group_by_list NK_COMMA expression", + /* 440 */ "having_clause_opt ::=", + /* 441 */ "having_clause_opt ::= HAVING search_condition", + /* 442 */ "range_opt ::=", + /* 443 */ "range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP", + /* 444 */ "every_opt ::=", + /* 445 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 446 */ "query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 447 */ "query_expression_body ::= query_primary", + /* 448 */ "query_expression_body ::= query_expression_body UNION ALL query_expression_body", + /* 449 */ "query_expression_body ::= query_expression_body UNION query_expression_body", + /* 450 */ "query_primary ::= query_specification", + /* 451 */ "query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP", + /* 452 */ "order_by_clause_opt ::=", + /* 453 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 454 */ "slimit_clause_opt ::=", + /* 455 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 456 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 457 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 458 */ "limit_clause_opt ::=", + /* 459 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 460 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 461 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 462 */ "subquery ::= NK_LP query_expression NK_RP", + /* 463 */ "search_condition ::= common_expression", + /* 464 */ "sort_specification_list ::= sort_specification", + /* 465 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 466 */ "sort_specification ::= expression ordering_specification_opt null_ordering_opt", + /* 467 */ "ordering_specification_opt ::=", + /* 468 */ "ordering_specification_opt ::= ASC", + /* 469 */ "ordering_specification_opt ::= DESC", + /* 470 */ "null_ordering_opt ::=", + /* 471 */ "null_ordering_opt ::= NULLS FIRST", + /* 472 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2121,176 +2164,181 @@ static void yy_destructor( */ /********* Begin destructor definitions ***************************************/ /* Default NON-TERMINAL Destructor */ - case 242: /* cmd */ - case 245: /* literal */ - case 255: /* db_options */ - case 257: /* alter_db_options */ - case 262: /* retention */ - case 263: /* full_table_name */ - case 266: /* table_options */ - case 270: /* alter_table_clause */ - case 271: /* alter_table_options */ - case 274: /* signed_literal */ - case 275: /* create_subtable_clause */ - case 278: /* drop_table_clause */ - case 281: /* column_def */ - case 285: /* duration_literal */ - case 286: /* rollup_func_name */ - case 288: /* col_name */ - case 289: /* db_name_cond_opt */ - case 290: /* like_pattern_opt */ - case 291: /* table_name_cond */ - case 292: /* from_db_opt */ - case 294: /* index_options */ - case 296: /* sliding_opt */ - case 297: /* func */ - case 300: /* query_expression */ - case 303: /* explain_options */ - case 307: /* stream_options */ - case 308: /* into_opt */ - case 310: /* where_clause_opt */ - case 311: /* signed */ - case 312: /* literal_func */ - case 315: /* expression */ - case 316: /* pseudo_column */ - case 317: /* column_reference */ - case 318: /* function_expression */ - case 319: /* subquery */ - case 324: /* star_func_para */ - case 325: /* predicate */ - case 328: /* in_predicate_value */ - case 329: /* boolean_value_expression */ - case 330: /* boolean_primary */ - case 331: /* common_expression */ - case 332: /* from_clause */ - case 333: /* table_reference_list */ - case 334: /* table_reference */ - case 335: /* table_primary */ - case 336: /* joined_table */ - case 338: /* parenthesized_joined_table */ - case 340: /* search_condition */ - case 341: /* query_specification */ - case 345: /* range_opt */ - case 346: /* every_opt */ - case 347: /* fill_opt */ - case 348: /* twindow_clause_opt */ - case 350: /* having_clause_opt */ - case 352: /* select_item */ - case 355: /* query_expression_body */ - case 357: /* slimit_clause_opt */ - case 358: /* limit_clause_opt */ - case 359: /* query_primary */ - case 361: /* sort_specification */ + case 244: /* cmd */ + case 247: /* literal */ + case 258: /* db_options */ + case 260: /* alter_db_options */ + case 265: /* retention */ + case 266: /* full_table_name */ + case 269: /* table_options */ + case 273: /* alter_table_clause */ + case 274: /* alter_table_options */ + case 277: /* signed_literal */ + case 278: /* create_subtable_clause */ + case 281: /* drop_table_clause */ + case 284: /* column_def */ + case 288: /* duration_literal */ + case 289: /* rollup_func_name */ + case 291: /* col_name */ + case 292: /* db_name_cond_opt */ + case 293: /* like_pattern_opt */ + case 294: /* table_name_cond */ + case 295: /* from_db_opt */ + case 297: /* index_options */ + case 299: /* sliding_opt */ + case 300: /* func */ + case 303: /* query_expression */ + case 306: /* explain_options */ + case 310: /* stream_options */ + case 311: /* into_opt */ + case 313: /* where_clause_opt */ + case 314: /* signed */ + case 315: /* literal_func */ + case 318: /* expression */ + case 319: /* pseudo_column */ + case 320: /* column_reference */ + case 321: /* function_expression */ + case 322: /* subquery */ + case 327: /* star_func_para */ + case 328: /* predicate */ + case 331: /* in_predicate_value */ + case 332: /* boolean_value_expression */ + case 333: /* boolean_primary */ + case 334: /* common_expression */ + case 335: /* from_clause_opt */ + case 336: /* table_reference_list */ + case 337: /* table_reference */ + case 338: /* table_primary */ + case 339: /* joined_table */ + case 341: /* parenthesized_joined_table */ + case 343: /* search_condition */ + case 344: /* query_specification */ + case 348: /* range_opt */ + case 349: /* every_opt */ + case 350: /* fill_opt */ + case 351: /* twindow_clause_opt */ + case 353: /* having_clause_opt */ + case 355: /* select_item */ + case 358: /* query_expression_body */ + case 360: /* slimit_clause_opt */ + case 361: /* limit_clause_opt */ + case 362: /* query_primary */ + case 364: /* sort_specification */ +{ + nodesDestroyNode((yypminor->yy442)); +} + break; + case 245: /* account_options */ + case 246: /* alter_account_options */ + case 248: /* alter_account_option */ + case 308: /* bufsize_opt */ { - nodesDestroyNode((yypminor->yy392)); + } break; - case 243: /* account_options */ - case 244: /* alter_account_options */ - case 246: /* alter_account_option */ - case 305: /* bufsize_opt */ + case 249: /* user_name */ + case 252: /* priv_level */ + case 255: /* db_name */ + case 256: /* dnode_endpoint */ + case 275: /* column_name */ + case 283: /* table_name */ + case 290: /* function_name */ + case 296: /* index_name */ + case 302: /* topic_name */ + case 304: /* cgroup_name */ + case 309: /* stream_name */ + case 316: /* table_alias */ + case 317: /* column_alias */ + case 323: /* star_func */ + case 325: /* noarg_func */ + case 340: /* alias_opt */ { } break; - case 247: /* user_name */ - case 249: /* priv_level */ - case 252: /* db_name */ - case 253: /* dnode_endpoint */ - case 272: /* column_name */ - case 280: /* table_name */ - case 287: /* function_name */ - case 293: /* index_name */ - case 299: /* topic_name */ - case 301: /* cgroup_name */ - case 306: /* stream_name */ - case 313: /* table_alias */ - case 314: /* column_alias */ - case 320: /* star_func */ - case 322: /* noarg_func */ - case 337: /* alias_opt */ + case 250: /* sysinfo_opt */ { } break; - case 248: /* privileges */ - case 250: /* priv_type_list */ - case 251: /* priv_type */ + case 251: /* privileges */ + case 253: /* priv_type_list */ + case 254: /* priv_type */ { } break; - case 254: /* not_exists_opt */ - case 256: /* exists_opt */ - case 302: /* analyze_opt */ - case 304: /* agg_func_opt */ - case 342: /* set_quantifier_opt */ + case 257: /* not_exists_opt */ + case 259: /* exists_opt */ + case 305: /* analyze_opt */ + case 307: /* agg_func_opt */ + case 345: /* set_quantifier_opt */ { } break; - case 258: /* integer_list */ - case 259: /* variable_list */ - case 260: /* retention_list */ - case 264: /* column_def_list */ - case 265: /* tags_def_opt */ - case 267: /* multi_create_clause */ - case 268: /* tags_def */ - case 269: /* multi_drop_clause */ - case 276: /* specific_tags_opt */ - case 277: /* literal_list */ - case 279: /* col_name_list */ - case 282: /* duration_list */ - case 283: /* rollup_func_list */ - case 295: /* func_list */ - case 298: /* expression_list */ - case 309: /* dnode_list */ - case 321: /* star_func_para_list */ - case 323: /* other_para_list */ - case 343: /* select_list */ - case 344: /* partition_by_clause_opt */ - case 349: /* group_by_clause_opt */ - case 351: /* select_sublist */ - case 354: /* group_by_list */ - case 356: /* order_by_clause_opt */ - case 360: /* sort_specification_list */ + case 261: /* integer_list */ + case 262: /* variable_list */ + case 263: /* retention_list */ + case 267: /* column_def_list */ + case 268: /* tags_def_opt */ + case 270: /* multi_create_clause */ + case 271: /* tags_def */ + case 272: /* multi_drop_clause */ + case 279: /* specific_tags_opt */ + case 280: /* literal_list */ + case 282: /* col_name_list */ + case 285: /* duration_list */ + case 286: /* rollup_func_list */ + case 298: /* func_list */ + case 301: /* expression_list */ + case 312: /* dnode_list */ + case 324: /* star_func_para_list */ + case 326: /* other_para_list */ + case 346: /* select_list */ + case 347: /* partition_by_clause_opt */ + case 352: /* group_by_clause_opt */ + case 354: /* select_sublist */ + case 357: /* group_by_list */ + case 359: /* order_by_clause_opt */ + case 363: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy600)); + nodesDestroyList((yypminor->yy368)); } break; - case 261: /* alter_db_option */ - case 284: /* alter_table_option */ + case 264: /* alter_db_option */ + case 287: /* alter_table_option */ { } break; - case 273: /* type_name */ + case 276: /* type_name */ { } break; - case 326: /* compare_op */ - case 327: /* in_op */ + case 329: /* compare_op */ + case 330: /* in_op */ { } break; - case 339: /* join_type */ + case 342: /* join_type */ { } break; - case 353: /* fill_mode */ + case 356: /* fill_mode */ { } break; - case 362: /* ordering_specification_opt */ + case 365: /* ordering_specification_opt */ { } break; - case 363: /* null_ordering_opt */ + case 366: /* null_ordering_opt */ { } @@ -2589,470 +2637,479 @@ static const struct { YYCODETYPE lhs; /* Symbol on the left-hand side of the rule */ signed char nrhs; /* Negative of the number of RHS symbols in the rule */ } yyRuleInfo[] = { - { 242, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ - { 242, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ - { 243, 0 }, /* (2) account_options ::= */ - { 243, -3 }, /* (3) account_options ::= account_options PPS literal */ - { 243, -3 }, /* (4) account_options ::= account_options TSERIES literal */ - { 243, -3 }, /* (5) account_options ::= account_options STORAGE literal */ - { 243, -3 }, /* (6) account_options ::= account_options STREAMS literal */ - { 243, -3 }, /* (7) account_options ::= account_options QTIME literal */ - { 243, -3 }, /* (8) account_options ::= account_options DBS literal */ - { 243, -3 }, /* (9) account_options ::= account_options USERS literal */ - { 243, -3 }, /* (10) account_options ::= account_options CONNS literal */ - { 243, -3 }, /* (11) account_options ::= account_options STATE literal */ - { 244, -1 }, /* (12) alter_account_options ::= alter_account_option */ - { 244, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ - { 246, -2 }, /* (14) alter_account_option ::= PASS literal */ - { 246, -2 }, /* (15) alter_account_option ::= PPS literal */ - { 246, -2 }, /* (16) alter_account_option ::= TSERIES literal */ - { 246, -2 }, /* (17) alter_account_option ::= STORAGE literal */ - { 246, -2 }, /* (18) alter_account_option ::= STREAMS literal */ - { 246, -2 }, /* (19) alter_account_option ::= QTIME literal */ - { 246, -2 }, /* (20) alter_account_option ::= DBS literal */ - { 246, -2 }, /* (21) alter_account_option ::= USERS literal */ - { 246, -2 }, /* (22) alter_account_option ::= CONNS literal */ - { 246, -2 }, /* (23) alter_account_option ::= STATE literal */ - { 242, -5 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING */ - { 242, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ - { 242, -5 }, /* (26) cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ - { 242, -3 }, /* (27) cmd ::= DROP USER user_name */ - { 242, -6 }, /* (28) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 242, -6 }, /* (29) cmd ::= REVOKE privileges ON priv_level FROM user_name */ - { 248, -1 }, /* (30) privileges ::= ALL */ - { 248, -1 }, /* (31) privileges ::= priv_type_list */ - { 250, -1 }, /* (32) priv_type_list ::= priv_type */ - { 250, -3 }, /* (33) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 251, -1 }, /* (34) priv_type ::= READ */ - { 251, -1 }, /* (35) priv_type ::= WRITE */ - { 249, -3 }, /* (36) priv_level ::= NK_STAR NK_DOT NK_STAR */ - { 249, -3 }, /* (37) priv_level ::= db_name NK_DOT NK_STAR */ - { 242, -3 }, /* (38) cmd ::= CREATE DNODE dnode_endpoint */ - { 242, -5 }, /* (39) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 242, -3 }, /* (40) cmd ::= DROP DNODE NK_INTEGER */ - { 242, -3 }, /* (41) cmd ::= DROP DNODE dnode_endpoint */ - { 242, -4 }, /* (42) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 242, -5 }, /* (43) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 242, -4 }, /* (44) cmd ::= ALTER ALL DNODES NK_STRING */ - { 242, -5 }, /* (45) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 253, -1 }, /* (46) dnode_endpoint ::= NK_STRING */ - { 253, -1 }, /* (47) dnode_endpoint ::= NK_ID */ - { 253, -1 }, /* (48) dnode_endpoint ::= NK_IPTOKEN */ - { 242, -3 }, /* (49) cmd ::= ALTER LOCAL NK_STRING */ - { 242, -4 }, /* (50) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 242, -5 }, /* (51) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (52) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (53) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (54) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (55) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (56) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (57) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (58) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 242, -5 }, /* (59) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 242, -4 }, /* (60) cmd ::= DROP DATABASE exists_opt db_name */ - { 242, -2 }, /* (61) cmd ::= USE db_name */ - { 242, -4 }, /* (62) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 254, -3 }, /* (63) not_exists_opt ::= IF NOT EXISTS */ - { 254, 0 }, /* (64) not_exists_opt ::= */ - { 256, -2 }, /* (65) exists_opt ::= IF EXISTS */ - { 256, 0 }, /* (66) exists_opt ::= */ - { 255, 0 }, /* (67) db_options ::= */ - { 255, -3 }, /* (68) db_options ::= db_options BUFFER NK_INTEGER */ - { 255, -3 }, /* (69) db_options ::= db_options CACHELAST NK_INTEGER */ - { 255, -3 }, /* (70) db_options ::= db_options COMP NK_INTEGER */ - { 255, -3 }, /* (71) db_options ::= db_options DURATION NK_INTEGER */ - { 255, -3 }, /* (72) db_options ::= db_options DURATION NK_VARIABLE */ - { 255, -3 }, /* (73) db_options ::= db_options FSYNC NK_INTEGER */ - { 255, -3 }, /* (74) db_options ::= db_options MAXROWS NK_INTEGER */ - { 255, -3 }, /* (75) db_options ::= db_options MINROWS NK_INTEGER */ - { 255, -3 }, /* (76) db_options ::= db_options KEEP integer_list */ - { 255, -3 }, /* (77) db_options ::= db_options KEEP variable_list */ - { 255, -3 }, /* (78) db_options ::= db_options PAGES NK_INTEGER */ - { 255, -3 }, /* (79) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 255, -3 }, /* (80) db_options ::= db_options PRECISION NK_STRING */ - { 255, -3 }, /* (81) db_options ::= db_options REPLICA NK_INTEGER */ - { 255, -3 }, /* (82) db_options ::= db_options STRICT NK_INTEGER */ - { 255, -3 }, /* (83) db_options ::= db_options WAL NK_INTEGER */ - { 255, -3 }, /* (84) db_options ::= db_options VGROUPS NK_INTEGER */ - { 255, -3 }, /* (85) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 255, -3 }, /* (86) db_options ::= db_options RETENTIONS retention_list */ - { 255, -3 }, /* (87) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 257, -1 }, /* (88) alter_db_options ::= alter_db_option */ - { 257, -2 }, /* (89) alter_db_options ::= alter_db_options alter_db_option */ - { 261, -2 }, /* (90) alter_db_option ::= BUFFER NK_INTEGER */ - { 261, -2 }, /* (91) alter_db_option ::= CACHELAST NK_INTEGER */ - { 261, -2 }, /* (92) alter_db_option ::= FSYNC NK_INTEGER */ - { 261, -2 }, /* (93) alter_db_option ::= KEEP integer_list */ - { 261, -2 }, /* (94) alter_db_option ::= KEEP variable_list */ - { 261, -2 }, /* (95) alter_db_option ::= PAGES NK_INTEGER */ - { 261, -2 }, /* (96) alter_db_option ::= REPLICA NK_INTEGER */ - { 261, -2 }, /* (97) alter_db_option ::= STRICT NK_INTEGER */ - { 261, -2 }, /* (98) alter_db_option ::= WAL NK_INTEGER */ - { 258, -1 }, /* (99) integer_list ::= NK_INTEGER */ - { 258, -3 }, /* (100) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 259, -1 }, /* (101) variable_list ::= NK_VARIABLE */ - { 259, -3 }, /* (102) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 260, -1 }, /* (103) retention_list ::= retention */ - { 260, -3 }, /* (104) retention_list ::= retention_list NK_COMMA retention */ - { 262, -3 }, /* (105) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 242, -9 }, /* (106) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 242, -3 }, /* (107) cmd ::= CREATE TABLE multi_create_clause */ - { 242, -9 }, /* (108) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 242, -3 }, /* (109) cmd ::= DROP TABLE multi_drop_clause */ - { 242, -4 }, /* (110) cmd ::= DROP STABLE exists_opt full_table_name */ - { 242, -3 }, /* (111) cmd ::= ALTER TABLE alter_table_clause */ - { 242, -3 }, /* (112) cmd ::= ALTER STABLE alter_table_clause */ - { 270, -2 }, /* (113) alter_table_clause ::= full_table_name alter_table_options */ - { 270, -5 }, /* (114) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 270, -4 }, /* (115) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 270, -5 }, /* (116) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 270, -5 }, /* (117) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 270, -5 }, /* (118) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 270, -4 }, /* (119) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 270, -5 }, /* (120) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 270, -5 }, /* (121) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 270, -6 }, /* (122) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 267, -1 }, /* (123) multi_create_clause ::= create_subtable_clause */ - { 267, -2 }, /* (124) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 275, -10 }, /* (125) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ - { 269, -1 }, /* (126) multi_drop_clause ::= drop_table_clause */ - { 269, -2 }, /* (127) multi_drop_clause ::= multi_drop_clause drop_table_clause */ - { 278, -2 }, /* (128) drop_table_clause ::= exists_opt full_table_name */ - { 276, 0 }, /* (129) specific_tags_opt ::= */ - { 276, -3 }, /* (130) specific_tags_opt ::= NK_LP col_name_list NK_RP */ - { 263, -1 }, /* (131) full_table_name ::= table_name */ - { 263, -3 }, /* (132) full_table_name ::= db_name NK_DOT table_name */ - { 264, -1 }, /* (133) column_def_list ::= column_def */ - { 264, -3 }, /* (134) column_def_list ::= column_def_list NK_COMMA column_def */ - { 281, -2 }, /* (135) column_def ::= column_name type_name */ - { 281, -4 }, /* (136) column_def ::= column_name type_name COMMENT NK_STRING */ - { 273, -1 }, /* (137) type_name ::= BOOL */ - { 273, -1 }, /* (138) type_name ::= TINYINT */ - { 273, -1 }, /* (139) type_name ::= SMALLINT */ - { 273, -1 }, /* (140) type_name ::= INT */ - { 273, -1 }, /* (141) type_name ::= INTEGER */ - { 273, -1 }, /* (142) type_name ::= BIGINT */ - { 273, -1 }, /* (143) type_name ::= FLOAT */ - { 273, -1 }, /* (144) type_name ::= DOUBLE */ - { 273, -4 }, /* (145) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 273, -1 }, /* (146) type_name ::= TIMESTAMP */ - { 273, -4 }, /* (147) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 273, -2 }, /* (148) type_name ::= TINYINT UNSIGNED */ - { 273, -2 }, /* (149) type_name ::= SMALLINT UNSIGNED */ - { 273, -2 }, /* (150) type_name ::= INT UNSIGNED */ - { 273, -2 }, /* (151) type_name ::= BIGINT UNSIGNED */ - { 273, -1 }, /* (152) type_name ::= JSON */ - { 273, -4 }, /* (153) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 273, -1 }, /* (154) type_name ::= MEDIUMBLOB */ - { 273, -1 }, /* (155) type_name ::= BLOB */ - { 273, -4 }, /* (156) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 273, -1 }, /* (157) type_name ::= DECIMAL */ - { 273, -4 }, /* (158) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 273, -6 }, /* (159) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 265, 0 }, /* (160) tags_def_opt ::= */ - { 265, -1 }, /* (161) tags_def_opt ::= tags_def */ - { 268, -4 }, /* (162) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 266, 0 }, /* (163) table_options ::= */ - { 266, -3 }, /* (164) table_options ::= table_options COMMENT NK_STRING */ - { 266, -3 }, /* (165) table_options ::= table_options MAX_DELAY duration_list */ - { 266, -3 }, /* (166) table_options ::= table_options WATERMARK duration_list */ - { 266, -5 }, /* (167) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 266, -3 }, /* (168) table_options ::= table_options TTL NK_INTEGER */ - { 266, -5 }, /* (169) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 271, -1 }, /* (170) alter_table_options ::= alter_table_option */ - { 271, -2 }, /* (171) alter_table_options ::= alter_table_options alter_table_option */ - { 284, -2 }, /* (172) alter_table_option ::= COMMENT NK_STRING */ - { 284, -2 }, /* (173) alter_table_option ::= TTL NK_INTEGER */ - { 282, -1 }, /* (174) duration_list ::= duration_literal */ - { 282, -3 }, /* (175) duration_list ::= duration_list NK_COMMA duration_literal */ - { 283, -1 }, /* (176) rollup_func_list ::= rollup_func_name */ - { 283, -3 }, /* (177) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 286, -1 }, /* (178) rollup_func_name ::= function_name */ - { 286, -1 }, /* (179) rollup_func_name ::= FIRST */ - { 286, -1 }, /* (180) rollup_func_name ::= LAST */ - { 279, -1 }, /* (181) col_name_list ::= col_name */ - { 279, -3 }, /* (182) col_name_list ::= col_name_list NK_COMMA col_name */ - { 288, -1 }, /* (183) col_name ::= column_name */ - { 242, -2 }, /* (184) cmd ::= SHOW DNODES */ - { 242, -2 }, /* (185) cmd ::= SHOW USERS */ - { 242, -2 }, /* (186) cmd ::= SHOW DATABASES */ - { 242, -4 }, /* (187) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 242, -4 }, /* (188) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 242, -3 }, /* (189) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 242, -2 }, /* (190) cmd ::= SHOW MNODES */ - { 242, -2 }, /* (191) cmd ::= SHOW MODULES */ - { 242, -2 }, /* (192) cmd ::= SHOW QNODES */ - { 242, -2 }, /* (193) cmd ::= SHOW FUNCTIONS */ - { 242, -5 }, /* (194) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 242, -2 }, /* (195) cmd ::= SHOW STREAMS */ - { 242, -2 }, /* (196) cmd ::= SHOW ACCOUNTS */ - { 242, -2 }, /* (197) cmd ::= SHOW APPS */ - { 242, -2 }, /* (198) cmd ::= SHOW CONNECTIONS */ - { 242, -2 }, /* (199) cmd ::= SHOW LICENCE */ - { 242, -2 }, /* (200) cmd ::= SHOW GRANTS */ - { 242, -4 }, /* (201) cmd ::= SHOW CREATE DATABASE db_name */ - { 242, -4 }, /* (202) cmd ::= SHOW CREATE TABLE full_table_name */ - { 242, -4 }, /* (203) cmd ::= SHOW CREATE STABLE full_table_name */ - { 242, -2 }, /* (204) cmd ::= SHOW QUERIES */ - { 242, -2 }, /* (205) cmd ::= SHOW SCORES */ - { 242, -2 }, /* (206) cmd ::= SHOW TOPICS */ - { 242, -2 }, /* (207) cmd ::= SHOW VARIABLES */ - { 242, -2 }, /* (208) cmd ::= SHOW BNODES */ - { 242, -2 }, /* (209) cmd ::= SHOW SNODES */ - { 242, -2 }, /* (210) cmd ::= SHOW CLUSTER */ - { 242, -2 }, /* (211) cmd ::= SHOW TRANSACTIONS */ - { 242, -4 }, /* (212) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 289, 0 }, /* (213) db_name_cond_opt ::= */ - { 289, -2 }, /* (214) db_name_cond_opt ::= db_name NK_DOT */ - { 290, 0 }, /* (215) like_pattern_opt ::= */ - { 290, -2 }, /* (216) like_pattern_opt ::= LIKE NK_STRING */ - { 291, -1 }, /* (217) table_name_cond ::= table_name */ - { 292, 0 }, /* (218) from_db_opt ::= */ - { 292, -2 }, /* (219) from_db_opt ::= FROM db_name */ - { 242, -8 }, /* (220) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ - { 242, -10 }, /* (221) cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ - { 242, -6 }, /* (222) cmd ::= DROP INDEX exists_opt index_name ON table_name */ - { 294, 0 }, /* (223) index_options ::= */ - { 294, -9 }, /* (224) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ - { 294, -11 }, /* (225) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ - { 295, -1 }, /* (226) func_list ::= func */ - { 295, -3 }, /* (227) func_list ::= func_list NK_COMMA func */ - { 297, -4 }, /* (228) func ::= function_name NK_LP expression_list NK_RP */ - { 242, -6 }, /* (229) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ - { 242, -7 }, /* (230) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 242, -7 }, /* (231) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 242, -4 }, /* (232) cmd ::= DROP TOPIC exists_opt topic_name */ - { 242, -7 }, /* (233) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 242, -2 }, /* (234) cmd ::= DESC full_table_name */ - { 242, -2 }, /* (235) cmd ::= DESCRIBE full_table_name */ - { 242, -3 }, /* (236) cmd ::= RESET QUERY CACHE */ - { 242, -4 }, /* (237) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ - { 302, 0 }, /* (238) analyze_opt ::= */ - { 302, -1 }, /* (239) analyze_opt ::= ANALYZE */ - { 303, 0 }, /* (240) explain_options ::= */ - { 303, -3 }, /* (241) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 303, -3 }, /* (242) explain_options ::= explain_options RATIO NK_FLOAT */ - { 242, -6 }, /* (243) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ - { 242, -10 }, /* (244) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ - { 242, -4 }, /* (245) cmd ::= DROP FUNCTION exists_opt function_name */ - { 304, 0 }, /* (246) agg_func_opt ::= */ - { 304, -1 }, /* (247) agg_func_opt ::= AGGREGATE */ - { 305, 0 }, /* (248) bufsize_opt ::= */ - { 305, -2 }, /* (249) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 242, -8 }, /* (250) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ - { 242, -4 }, /* (251) cmd ::= DROP STREAM exists_opt stream_name */ - { 308, 0 }, /* (252) into_opt ::= */ - { 308, -2 }, /* (253) into_opt ::= INTO full_table_name */ - { 307, 0 }, /* (254) stream_options ::= */ - { 307, -3 }, /* (255) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 307, -3 }, /* (256) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 307, -4 }, /* (257) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 307, -3 }, /* (258) stream_options ::= stream_options WATERMARK duration_literal */ - { 242, -3 }, /* (259) cmd ::= KILL CONNECTION NK_INTEGER */ - { 242, -3 }, /* (260) cmd ::= KILL QUERY NK_STRING */ - { 242, -3 }, /* (261) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 242, -2 }, /* (262) cmd ::= BALANCE VGROUP */ - { 242, -4 }, /* (263) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 242, -4 }, /* (264) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 242, -3 }, /* (265) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 309, -2 }, /* (266) dnode_list ::= DNODE NK_INTEGER */ - { 309, -3 }, /* (267) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 242, -3 }, /* (268) cmd ::= SYNCDB db_name REPLICA */ - { 242, -4 }, /* (269) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 242, -1 }, /* (270) cmd ::= query_expression */ - { 245, -1 }, /* (271) literal ::= NK_INTEGER */ - { 245, -1 }, /* (272) literal ::= NK_FLOAT */ - { 245, -1 }, /* (273) literal ::= NK_STRING */ - { 245, -1 }, /* (274) literal ::= NK_BOOL */ - { 245, -2 }, /* (275) literal ::= TIMESTAMP NK_STRING */ - { 245, -1 }, /* (276) literal ::= duration_literal */ - { 245, -1 }, /* (277) literal ::= NULL */ - { 245, -1 }, /* (278) literal ::= NK_QUESTION */ - { 285, -1 }, /* (279) duration_literal ::= NK_VARIABLE */ - { 311, -1 }, /* (280) signed ::= NK_INTEGER */ - { 311, -2 }, /* (281) signed ::= NK_PLUS NK_INTEGER */ - { 311, -2 }, /* (282) signed ::= NK_MINUS NK_INTEGER */ - { 311, -1 }, /* (283) signed ::= NK_FLOAT */ - { 311, -2 }, /* (284) signed ::= NK_PLUS NK_FLOAT */ - { 311, -2 }, /* (285) signed ::= NK_MINUS NK_FLOAT */ - { 274, -1 }, /* (286) signed_literal ::= signed */ - { 274, -1 }, /* (287) signed_literal ::= NK_STRING */ - { 274, -1 }, /* (288) signed_literal ::= NK_BOOL */ - { 274, -2 }, /* (289) signed_literal ::= TIMESTAMP NK_STRING */ - { 274, -1 }, /* (290) signed_literal ::= duration_literal */ - { 274, -1 }, /* (291) signed_literal ::= NULL */ - { 274, -1 }, /* (292) signed_literal ::= literal_func */ - { 277, -1 }, /* (293) literal_list ::= signed_literal */ - { 277, -3 }, /* (294) literal_list ::= literal_list NK_COMMA signed_literal */ - { 252, -1 }, /* (295) db_name ::= NK_ID */ - { 280, -1 }, /* (296) table_name ::= NK_ID */ - { 272, -1 }, /* (297) column_name ::= NK_ID */ - { 287, -1 }, /* (298) function_name ::= NK_ID */ - { 313, -1 }, /* (299) table_alias ::= NK_ID */ - { 314, -1 }, /* (300) column_alias ::= NK_ID */ - { 247, -1 }, /* (301) user_name ::= NK_ID */ - { 293, -1 }, /* (302) index_name ::= NK_ID */ - { 299, -1 }, /* (303) topic_name ::= NK_ID */ - { 306, -1 }, /* (304) stream_name ::= NK_ID */ - { 301, -1 }, /* (305) cgroup_name ::= NK_ID */ - { 315, -1 }, /* (306) expression ::= literal */ - { 315, -1 }, /* (307) expression ::= pseudo_column */ - { 315, -1 }, /* (308) expression ::= column_reference */ - { 315, -1 }, /* (309) expression ::= function_expression */ - { 315, -1 }, /* (310) expression ::= subquery */ - { 315, -3 }, /* (311) expression ::= NK_LP expression NK_RP */ - { 315, -2 }, /* (312) expression ::= NK_PLUS expression */ - { 315, -2 }, /* (313) expression ::= NK_MINUS expression */ - { 315, -3 }, /* (314) expression ::= expression NK_PLUS expression */ - { 315, -3 }, /* (315) expression ::= expression NK_MINUS expression */ - { 315, -3 }, /* (316) expression ::= expression NK_STAR expression */ - { 315, -3 }, /* (317) expression ::= expression NK_SLASH expression */ - { 315, -3 }, /* (318) expression ::= expression NK_REM expression */ - { 315, -3 }, /* (319) expression ::= column_reference NK_ARROW NK_STRING */ - { 298, -1 }, /* (320) expression_list ::= expression */ - { 298, -3 }, /* (321) expression_list ::= expression_list NK_COMMA expression */ - { 317, -1 }, /* (322) column_reference ::= column_name */ - { 317, -3 }, /* (323) column_reference ::= table_name NK_DOT column_name */ - { 316, -1 }, /* (324) pseudo_column ::= ROWTS */ - { 316, -1 }, /* (325) pseudo_column ::= TBNAME */ - { 316, -3 }, /* (326) pseudo_column ::= table_name NK_DOT TBNAME */ - { 316, -1 }, /* (327) pseudo_column ::= QSTARTTS */ - { 316, -1 }, /* (328) pseudo_column ::= QENDTS */ - { 316, -1 }, /* (329) pseudo_column ::= WSTARTTS */ - { 316, -1 }, /* (330) pseudo_column ::= WENDTS */ - { 316, -1 }, /* (331) pseudo_column ::= WDURATION */ - { 318, -4 }, /* (332) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 318, -4 }, /* (333) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 318, -6 }, /* (334) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ - { 318, -1 }, /* (335) function_expression ::= literal_func */ - { 312, -3 }, /* (336) literal_func ::= noarg_func NK_LP NK_RP */ - { 312, -1 }, /* (337) literal_func ::= NOW */ - { 322, -1 }, /* (338) noarg_func ::= NOW */ - { 322, -1 }, /* (339) noarg_func ::= TODAY */ - { 322, -1 }, /* (340) noarg_func ::= TIMEZONE */ - { 320, -1 }, /* (341) star_func ::= COUNT */ - { 320, -1 }, /* (342) star_func ::= FIRST */ - { 320, -1 }, /* (343) star_func ::= LAST */ - { 320, -1 }, /* (344) star_func ::= LAST_ROW */ - { 321, -1 }, /* (345) star_func_para_list ::= NK_STAR */ - { 321, -1 }, /* (346) star_func_para_list ::= other_para_list */ - { 323, -1 }, /* (347) other_para_list ::= star_func_para */ - { 323, -3 }, /* (348) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 324, -1 }, /* (349) star_func_para ::= expression */ - { 324, -3 }, /* (350) star_func_para ::= table_name NK_DOT NK_STAR */ - { 325, -3 }, /* (351) predicate ::= expression compare_op expression */ - { 325, -5 }, /* (352) predicate ::= expression BETWEEN expression AND expression */ - { 325, -6 }, /* (353) predicate ::= expression NOT BETWEEN expression AND expression */ - { 325, -3 }, /* (354) predicate ::= expression IS NULL */ - { 325, -4 }, /* (355) predicate ::= expression IS NOT NULL */ - { 325, -3 }, /* (356) predicate ::= expression in_op in_predicate_value */ - { 326, -1 }, /* (357) compare_op ::= NK_LT */ - { 326, -1 }, /* (358) compare_op ::= NK_GT */ - { 326, -1 }, /* (359) compare_op ::= NK_LE */ - { 326, -1 }, /* (360) compare_op ::= NK_GE */ - { 326, -1 }, /* (361) compare_op ::= NK_NE */ - { 326, -1 }, /* (362) compare_op ::= NK_EQ */ - { 326, -1 }, /* (363) compare_op ::= LIKE */ - { 326, -2 }, /* (364) compare_op ::= NOT LIKE */ - { 326, -1 }, /* (365) compare_op ::= MATCH */ - { 326, -1 }, /* (366) compare_op ::= NMATCH */ - { 326, -1 }, /* (367) compare_op ::= CONTAINS */ - { 327, -1 }, /* (368) in_op ::= IN */ - { 327, -2 }, /* (369) in_op ::= NOT IN */ - { 328, -3 }, /* (370) in_predicate_value ::= NK_LP expression_list NK_RP */ - { 329, -1 }, /* (371) boolean_value_expression ::= boolean_primary */ - { 329, -2 }, /* (372) boolean_value_expression ::= NOT boolean_primary */ - { 329, -3 }, /* (373) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 329, -3 }, /* (374) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 330, -1 }, /* (375) boolean_primary ::= predicate */ - { 330, -3 }, /* (376) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 331, -1 }, /* (377) common_expression ::= expression */ - { 331, -1 }, /* (378) common_expression ::= boolean_value_expression */ - { 332, -2 }, /* (379) from_clause ::= FROM table_reference_list */ - { 333, -1 }, /* (380) table_reference_list ::= table_reference */ - { 333, -3 }, /* (381) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 334, -1 }, /* (382) table_reference ::= table_primary */ - { 334, -1 }, /* (383) table_reference ::= joined_table */ - { 335, -2 }, /* (384) table_primary ::= table_name alias_opt */ - { 335, -4 }, /* (385) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 335, -2 }, /* (386) table_primary ::= subquery alias_opt */ - { 335, -1 }, /* (387) table_primary ::= parenthesized_joined_table */ - { 337, 0 }, /* (388) alias_opt ::= */ - { 337, -1 }, /* (389) alias_opt ::= table_alias */ - { 337, -2 }, /* (390) alias_opt ::= AS table_alias */ - { 338, -3 }, /* (391) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 338, -3 }, /* (392) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 336, -6 }, /* (393) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 339, 0 }, /* (394) join_type ::= */ - { 339, -1 }, /* (395) join_type ::= INNER */ - { 341, -12 }, /* (396) query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 342, 0 }, /* (397) set_quantifier_opt ::= */ - { 342, -1 }, /* (398) set_quantifier_opt ::= DISTINCT */ - { 342, -1 }, /* (399) set_quantifier_opt ::= ALL */ - { 343, -1 }, /* (400) select_list ::= NK_STAR */ - { 343, -1 }, /* (401) select_list ::= select_sublist */ - { 351, -1 }, /* (402) select_sublist ::= select_item */ - { 351, -3 }, /* (403) select_sublist ::= select_sublist NK_COMMA select_item */ - { 352, -1 }, /* (404) select_item ::= common_expression */ - { 352, -2 }, /* (405) select_item ::= common_expression column_alias */ - { 352, -3 }, /* (406) select_item ::= common_expression AS column_alias */ - { 352, -3 }, /* (407) select_item ::= table_name NK_DOT NK_STAR */ - { 310, 0 }, /* (408) where_clause_opt ::= */ - { 310, -2 }, /* (409) where_clause_opt ::= WHERE search_condition */ - { 344, 0 }, /* (410) partition_by_clause_opt ::= */ - { 344, -3 }, /* (411) partition_by_clause_opt ::= PARTITION BY expression_list */ - { 348, 0 }, /* (412) twindow_clause_opt ::= */ - { 348, -6 }, /* (413) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 348, -4 }, /* (414) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ - { 348, -6 }, /* (415) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 348, -8 }, /* (416) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 296, 0 }, /* (417) sliding_opt ::= */ - { 296, -4 }, /* (418) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 347, 0 }, /* (419) fill_opt ::= */ - { 347, -4 }, /* (420) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 347, -6 }, /* (421) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 353, -1 }, /* (422) fill_mode ::= NONE */ - { 353, -1 }, /* (423) fill_mode ::= PREV */ - { 353, -1 }, /* (424) fill_mode ::= NULL */ - { 353, -1 }, /* (425) fill_mode ::= LINEAR */ - { 353, -1 }, /* (426) fill_mode ::= NEXT */ - { 349, 0 }, /* (427) group_by_clause_opt ::= */ - { 349, -3 }, /* (428) group_by_clause_opt ::= GROUP BY group_by_list */ - { 354, -1 }, /* (429) group_by_list ::= expression */ - { 354, -3 }, /* (430) group_by_list ::= group_by_list NK_COMMA expression */ - { 350, 0 }, /* (431) having_clause_opt ::= */ - { 350, -2 }, /* (432) having_clause_opt ::= HAVING search_condition */ - { 345, 0 }, /* (433) range_opt ::= */ - { 345, -6 }, /* (434) range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ - { 346, 0 }, /* (435) every_opt ::= */ - { 346, -4 }, /* (436) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 300, -4 }, /* (437) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 355, -1 }, /* (438) query_expression_body ::= query_primary */ - { 355, -4 }, /* (439) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ - { 355, -3 }, /* (440) query_expression_body ::= query_expression_body UNION query_expression_body */ - { 359, -1 }, /* (441) query_primary ::= query_specification */ - { 359, -6 }, /* (442) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ - { 356, 0 }, /* (443) order_by_clause_opt ::= */ - { 356, -3 }, /* (444) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 357, 0 }, /* (445) slimit_clause_opt ::= */ - { 357, -2 }, /* (446) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 357, -4 }, /* (447) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 357, -4 }, /* (448) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 358, 0 }, /* (449) limit_clause_opt ::= */ - { 358, -2 }, /* (450) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 358, -4 }, /* (451) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 358, -4 }, /* (452) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 319, -3 }, /* (453) subquery ::= NK_LP query_expression NK_RP */ - { 340, -1 }, /* (454) search_condition ::= common_expression */ - { 360, -1 }, /* (455) sort_specification_list ::= sort_specification */ - { 360, -3 }, /* (456) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 361, -3 }, /* (457) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ - { 362, 0 }, /* (458) ordering_specification_opt ::= */ - { 362, -1 }, /* (459) ordering_specification_opt ::= ASC */ - { 362, -1 }, /* (460) ordering_specification_opt ::= DESC */ - { 363, 0 }, /* (461) null_ordering_opt ::= */ - { 363, -2 }, /* (462) null_ordering_opt ::= NULLS FIRST */ - { 363, -2 }, /* (463) null_ordering_opt ::= NULLS LAST */ + { 244, -6 }, /* (0) cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ + { 244, -4 }, /* (1) cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ + { 245, 0 }, /* (2) account_options ::= */ + { 245, -3 }, /* (3) account_options ::= account_options PPS literal */ + { 245, -3 }, /* (4) account_options ::= account_options TSERIES literal */ + { 245, -3 }, /* (5) account_options ::= account_options STORAGE literal */ + { 245, -3 }, /* (6) account_options ::= account_options STREAMS literal */ + { 245, -3 }, /* (7) account_options ::= account_options QTIME literal */ + { 245, -3 }, /* (8) account_options ::= account_options DBS literal */ + { 245, -3 }, /* (9) account_options ::= account_options USERS literal */ + { 245, -3 }, /* (10) account_options ::= account_options CONNS literal */ + { 245, -3 }, /* (11) account_options ::= account_options STATE literal */ + { 246, -1 }, /* (12) alter_account_options ::= alter_account_option */ + { 246, -2 }, /* (13) alter_account_options ::= alter_account_options alter_account_option */ + { 248, -2 }, /* (14) alter_account_option ::= PASS literal */ + { 248, -2 }, /* (15) alter_account_option ::= PPS literal */ + { 248, -2 }, /* (16) alter_account_option ::= TSERIES literal */ + { 248, -2 }, /* (17) alter_account_option ::= STORAGE literal */ + { 248, -2 }, /* (18) alter_account_option ::= STREAMS literal */ + { 248, -2 }, /* (19) alter_account_option ::= QTIME literal */ + { 248, -2 }, /* (20) alter_account_option ::= DBS literal */ + { 248, -2 }, /* (21) alter_account_option ::= USERS literal */ + { 248, -2 }, /* (22) alter_account_option ::= CONNS literal */ + { 248, -2 }, /* (23) alter_account_option ::= STATE literal */ + { 244, -6 }, /* (24) cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ + { 244, -5 }, /* (25) cmd ::= ALTER USER user_name PASS NK_STRING */ + { 244, -5 }, /* (26) cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ + { 244, -5 }, /* (27) cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ + { 244, -3 }, /* (28) cmd ::= DROP USER user_name */ + { 250, 0 }, /* (29) sysinfo_opt ::= */ + { 250, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ + { 244, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ + { 244, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ + { 251, -1 }, /* (33) privileges ::= ALL */ + { 251, -1 }, /* (34) privileges ::= priv_type_list */ + { 253, -1 }, /* (35) priv_type_list ::= priv_type */ + { 253, -3 }, /* (36) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 254, -1 }, /* (37) priv_type ::= READ */ + { 254, -1 }, /* (38) priv_type ::= WRITE */ + { 252, -3 }, /* (39) priv_level ::= NK_STAR NK_DOT NK_STAR */ + { 252, -3 }, /* (40) priv_level ::= db_name NK_DOT NK_STAR */ + { 244, -3 }, /* (41) cmd ::= CREATE DNODE dnode_endpoint */ + { 244, -5 }, /* (42) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 244, -3 }, /* (43) cmd ::= DROP DNODE NK_INTEGER */ + { 244, -3 }, /* (44) cmd ::= DROP DNODE dnode_endpoint */ + { 244, -4 }, /* (45) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 244, -5 }, /* (46) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 244, -4 }, /* (47) cmd ::= ALTER ALL DNODES NK_STRING */ + { 244, -5 }, /* (48) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 256, -1 }, /* (49) dnode_endpoint ::= NK_STRING */ + { 256, -1 }, /* (50) dnode_endpoint ::= NK_ID */ + { 256, -1 }, /* (51) dnode_endpoint ::= NK_IPTOKEN */ + { 244, -3 }, /* (52) cmd ::= ALTER LOCAL NK_STRING */ + { 244, -4 }, /* (53) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 244, -5 }, /* (54) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (55) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (56) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (57) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (58) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (59) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (60) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (61) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 244, -5 }, /* (62) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 244, -4 }, /* (63) cmd ::= DROP DATABASE exists_opt db_name */ + { 244, -2 }, /* (64) cmd ::= USE db_name */ + { 244, -4 }, /* (65) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 257, -3 }, /* (66) not_exists_opt ::= IF NOT EXISTS */ + { 257, 0 }, /* (67) not_exists_opt ::= */ + { 259, -2 }, /* (68) exists_opt ::= IF EXISTS */ + { 259, 0 }, /* (69) exists_opt ::= */ + { 258, 0 }, /* (70) db_options ::= */ + { 258, -3 }, /* (71) db_options ::= db_options BUFFER NK_INTEGER */ + { 258, -3 }, /* (72) db_options ::= db_options CACHELAST NK_INTEGER */ + { 258, -3 }, /* (73) db_options ::= db_options COMP NK_INTEGER */ + { 258, -3 }, /* (74) db_options ::= db_options DURATION NK_INTEGER */ + { 258, -3 }, /* (75) db_options ::= db_options DURATION NK_VARIABLE */ + { 258, -3 }, /* (76) db_options ::= db_options FSYNC NK_INTEGER */ + { 258, -3 }, /* (77) db_options ::= db_options MAXROWS NK_INTEGER */ + { 258, -3 }, /* (78) db_options ::= db_options MINROWS NK_INTEGER */ + { 258, -3 }, /* (79) db_options ::= db_options KEEP integer_list */ + { 258, -3 }, /* (80) db_options ::= db_options KEEP variable_list */ + { 258, -3 }, /* (81) db_options ::= db_options PAGES NK_INTEGER */ + { 258, -3 }, /* (82) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 258, -3 }, /* (83) db_options ::= db_options PRECISION NK_STRING */ + { 258, -3 }, /* (84) db_options ::= db_options REPLICA NK_INTEGER */ + { 258, -3 }, /* (85) db_options ::= db_options STRICT NK_INTEGER */ + { 258, -3 }, /* (86) db_options ::= db_options WAL NK_INTEGER */ + { 258, -3 }, /* (87) db_options ::= db_options VGROUPS NK_INTEGER */ + { 258, -3 }, /* (88) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 258, -3 }, /* (89) db_options ::= db_options RETENTIONS retention_list */ + { 258, -3 }, /* (90) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 260, -1 }, /* (91) alter_db_options ::= alter_db_option */ + { 260, -2 }, /* (92) alter_db_options ::= alter_db_options alter_db_option */ + { 264, -2 }, /* (93) alter_db_option ::= BUFFER NK_INTEGER */ + { 264, -2 }, /* (94) alter_db_option ::= CACHELAST NK_INTEGER */ + { 264, -2 }, /* (95) alter_db_option ::= FSYNC NK_INTEGER */ + { 264, -2 }, /* (96) alter_db_option ::= KEEP integer_list */ + { 264, -2 }, /* (97) alter_db_option ::= KEEP variable_list */ + { 264, -2 }, /* (98) alter_db_option ::= PAGES NK_INTEGER */ + { 264, -2 }, /* (99) alter_db_option ::= REPLICA NK_INTEGER */ + { 264, -2 }, /* (100) alter_db_option ::= STRICT NK_INTEGER */ + { 264, -2 }, /* (101) alter_db_option ::= WAL NK_INTEGER */ + { 261, -1 }, /* (102) integer_list ::= NK_INTEGER */ + { 261, -3 }, /* (103) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 262, -1 }, /* (104) variable_list ::= NK_VARIABLE */ + { 262, -3 }, /* (105) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 263, -1 }, /* (106) retention_list ::= retention */ + { 263, -3 }, /* (107) retention_list ::= retention_list NK_COMMA retention */ + { 265, -3 }, /* (108) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 244, -9 }, /* (109) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 244, -3 }, /* (110) cmd ::= CREATE TABLE multi_create_clause */ + { 244, -9 }, /* (111) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 244, -3 }, /* (112) cmd ::= DROP TABLE multi_drop_clause */ + { 244, -4 }, /* (113) cmd ::= DROP STABLE exists_opt full_table_name */ + { 244, -3 }, /* (114) cmd ::= ALTER TABLE alter_table_clause */ + { 244, -3 }, /* (115) cmd ::= ALTER STABLE alter_table_clause */ + { 273, -2 }, /* (116) alter_table_clause ::= full_table_name alter_table_options */ + { 273, -5 }, /* (117) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 273, -4 }, /* (118) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 273, -5 }, /* (119) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 273, -5 }, /* (120) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 273, -5 }, /* (121) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 273, -4 }, /* (122) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 273, -5 }, /* (123) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 273, -5 }, /* (124) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 273, -6 }, /* (125) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 270, -1 }, /* (126) multi_create_clause ::= create_subtable_clause */ + { 270, -2 }, /* (127) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 278, -10 }, /* (128) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ + { 272, -1 }, /* (129) multi_drop_clause ::= drop_table_clause */ + { 272, -2 }, /* (130) multi_drop_clause ::= multi_drop_clause drop_table_clause */ + { 281, -2 }, /* (131) drop_table_clause ::= exists_opt full_table_name */ + { 279, 0 }, /* (132) specific_tags_opt ::= */ + { 279, -3 }, /* (133) specific_tags_opt ::= NK_LP col_name_list NK_RP */ + { 266, -1 }, /* (134) full_table_name ::= table_name */ + { 266, -3 }, /* (135) full_table_name ::= db_name NK_DOT table_name */ + { 267, -1 }, /* (136) column_def_list ::= column_def */ + { 267, -3 }, /* (137) column_def_list ::= column_def_list NK_COMMA column_def */ + { 284, -2 }, /* (138) column_def ::= column_name type_name */ + { 284, -4 }, /* (139) column_def ::= column_name type_name COMMENT NK_STRING */ + { 276, -1 }, /* (140) type_name ::= BOOL */ + { 276, -1 }, /* (141) type_name ::= TINYINT */ + { 276, -1 }, /* (142) type_name ::= SMALLINT */ + { 276, -1 }, /* (143) type_name ::= INT */ + { 276, -1 }, /* (144) type_name ::= INTEGER */ + { 276, -1 }, /* (145) type_name ::= BIGINT */ + { 276, -1 }, /* (146) type_name ::= FLOAT */ + { 276, -1 }, /* (147) type_name ::= DOUBLE */ + { 276, -4 }, /* (148) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 276, -1 }, /* (149) type_name ::= TIMESTAMP */ + { 276, -4 }, /* (150) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 276, -2 }, /* (151) type_name ::= TINYINT UNSIGNED */ + { 276, -2 }, /* (152) type_name ::= SMALLINT UNSIGNED */ + { 276, -2 }, /* (153) type_name ::= INT UNSIGNED */ + { 276, -2 }, /* (154) type_name ::= BIGINT UNSIGNED */ + { 276, -1 }, /* (155) type_name ::= JSON */ + { 276, -4 }, /* (156) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 276, -1 }, /* (157) type_name ::= MEDIUMBLOB */ + { 276, -1 }, /* (158) type_name ::= BLOB */ + { 276, -4 }, /* (159) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 276, -1 }, /* (160) type_name ::= DECIMAL */ + { 276, -4 }, /* (161) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 276, -6 }, /* (162) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 268, 0 }, /* (163) tags_def_opt ::= */ + { 268, -1 }, /* (164) tags_def_opt ::= tags_def */ + { 271, -4 }, /* (165) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 269, 0 }, /* (166) table_options ::= */ + { 269, -3 }, /* (167) table_options ::= table_options COMMENT NK_STRING */ + { 269, -3 }, /* (168) table_options ::= table_options MAX_DELAY duration_list */ + { 269, -3 }, /* (169) table_options ::= table_options WATERMARK duration_list */ + { 269, -5 }, /* (170) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 269, -3 }, /* (171) table_options ::= table_options TTL NK_INTEGER */ + { 269, -5 }, /* (172) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 274, -1 }, /* (173) alter_table_options ::= alter_table_option */ + { 274, -2 }, /* (174) alter_table_options ::= alter_table_options alter_table_option */ + { 287, -2 }, /* (175) alter_table_option ::= COMMENT NK_STRING */ + { 287, -2 }, /* (176) alter_table_option ::= TTL NK_INTEGER */ + { 285, -1 }, /* (177) duration_list ::= duration_literal */ + { 285, -3 }, /* (178) duration_list ::= duration_list NK_COMMA duration_literal */ + { 286, -1 }, /* (179) rollup_func_list ::= rollup_func_name */ + { 286, -3 }, /* (180) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 289, -1 }, /* (181) rollup_func_name ::= function_name */ + { 289, -1 }, /* (182) rollup_func_name ::= FIRST */ + { 289, -1 }, /* (183) rollup_func_name ::= LAST */ + { 282, -1 }, /* (184) col_name_list ::= col_name */ + { 282, -3 }, /* (185) col_name_list ::= col_name_list NK_COMMA col_name */ + { 291, -1 }, /* (186) col_name ::= column_name */ + { 244, -2 }, /* (187) cmd ::= SHOW DNODES */ + { 244, -2 }, /* (188) cmd ::= SHOW USERS */ + { 244, -2 }, /* (189) cmd ::= SHOW DATABASES */ + { 244, -4 }, /* (190) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 244, -4 }, /* (191) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 244, -3 }, /* (192) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 244, -2 }, /* (193) cmd ::= SHOW MNODES */ + { 244, -2 }, /* (194) cmd ::= SHOW MODULES */ + { 244, -2 }, /* (195) cmd ::= SHOW QNODES */ + { 244, -2 }, /* (196) cmd ::= SHOW FUNCTIONS */ + { 244, -5 }, /* (197) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 244, -2 }, /* (198) cmd ::= SHOW STREAMS */ + { 244, -2 }, /* (199) cmd ::= SHOW ACCOUNTS */ + { 244, -2 }, /* (200) cmd ::= SHOW APPS */ + { 244, -2 }, /* (201) cmd ::= SHOW CONNECTIONS */ + { 244, -2 }, /* (202) cmd ::= SHOW LICENCE */ + { 244, -2 }, /* (203) cmd ::= SHOW GRANTS */ + { 244, -4 }, /* (204) cmd ::= SHOW CREATE DATABASE db_name */ + { 244, -4 }, /* (205) cmd ::= SHOW CREATE TABLE full_table_name */ + { 244, -4 }, /* (206) cmd ::= SHOW CREATE STABLE full_table_name */ + { 244, -2 }, /* (207) cmd ::= SHOW QUERIES */ + { 244, -2 }, /* (208) cmd ::= SHOW SCORES */ + { 244, -2 }, /* (209) cmd ::= SHOW TOPICS */ + { 244, -2 }, /* (210) cmd ::= SHOW VARIABLES */ + { 244, -3 }, /* (211) cmd ::= SHOW LOCAL VARIABLES */ + { 244, -4 }, /* (212) cmd ::= SHOW DNODE NK_INTEGER VARIABLES */ + { 244, -2 }, /* (213) cmd ::= SHOW BNODES */ + { 244, -2 }, /* (214) cmd ::= SHOW SNODES */ + { 244, -2 }, /* (215) cmd ::= SHOW CLUSTER */ + { 244, -2 }, /* (216) cmd ::= SHOW TRANSACTIONS */ + { 244, -4 }, /* (217) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 292, 0 }, /* (218) db_name_cond_opt ::= */ + { 292, -2 }, /* (219) db_name_cond_opt ::= db_name NK_DOT */ + { 293, 0 }, /* (220) like_pattern_opt ::= */ + { 293, -2 }, /* (221) like_pattern_opt ::= LIKE NK_STRING */ + { 294, -1 }, /* (222) table_name_cond ::= table_name */ + { 295, 0 }, /* (223) from_db_opt ::= */ + { 295, -2 }, /* (224) from_db_opt ::= FROM db_name */ + { 244, -8 }, /* (225) cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ + { 244, -4 }, /* (226) cmd ::= DROP INDEX exists_opt index_name */ + { 297, 0 }, /* (227) index_options ::= */ + { 297, -9 }, /* (228) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ + { 297, -11 }, /* (229) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ + { 298, -1 }, /* (230) func_list ::= func */ + { 298, -3 }, /* (231) func_list ::= func_list NK_COMMA func */ + { 300, -4 }, /* (232) func ::= function_name NK_LP expression_list NK_RP */ + { 244, -6 }, /* (233) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ + { 244, -7 }, /* (234) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 244, -9 }, /* (235) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 244, -7 }, /* (236) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 244, -9 }, /* (237) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 244, -4 }, /* (238) cmd ::= DROP TOPIC exists_opt topic_name */ + { 244, -7 }, /* (239) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 244, -2 }, /* (240) cmd ::= DESC full_table_name */ + { 244, -2 }, /* (241) cmd ::= DESCRIBE full_table_name */ + { 244, -3 }, /* (242) cmd ::= RESET QUERY CACHE */ + { 244, -4 }, /* (243) cmd ::= EXPLAIN analyze_opt explain_options query_expression */ + { 305, 0 }, /* (244) analyze_opt ::= */ + { 305, -1 }, /* (245) analyze_opt ::= ANALYZE */ + { 306, 0 }, /* (246) explain_options ::= */ + { 306, -3 }, /* (247) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 306, -3 }, /* (248) explain_options ::= explain_options RATIO NK_FLOAT */ + { 244, -6 }, /* (249) cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ + { 244, -10 }, /* (250) cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ + { 244, -4 }, /* (251) cmd ::= DROP FUNCTION exists_opt function_name */ + { 307, 0 }, /* (252) agg_func_opt ::= */ + { 307, -1 }, /* (253) agg_func_opt ::= AGGREGATE */ + { 308, 0 }, /* (254) bufsize_opt ::= */ + { 308, -2 }, /* (255) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 244, -8 }, /* (256) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ + { 244, -4 }, /* (257) cmd ::= DROP STREAM exists_opt stream_name */ + { 311, 0 }, /* (258) into_opt ::= */ + { 311, -2 }, /* (259) into_opt ::= INTO full_table_name */ + { 310, 0 }, /* (260) stream_options ::= */ + { 310, -3 }, /* (261) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 310, -3 }, /* (262) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 310, -4 }, /* (263) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 310, -3 }, /* (264) stream_options ::= stream_options WATERMARK duration_literal */ + { 244, -3 }, /* (265) cmd ::= KILL CONNECTION NK_INTEGER */ + { 244, -3 }, /* (266) cmd ::= KILL QUERY NK_STRING */ + { 244, -3 }, /* (267) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 244, -2 }, /* (268) cmd ::= BALANCE VGROUP */ + { 244, -4 }, /* (269) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 244, -4 }, /* (270) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 244, -3 }, /* (271) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 312, -2 }, /* (272) dnode_list ::= DNODE NK_INTEGER */ + { 312, -3 }, /* (273) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 244, -3 }, /* (274) cmd ::= SYNCDB db_name REPLICA */ + { 244, -4 }, /* (275) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 244, -1 }, /* (276) cmd ::= query_expression */ + { 247, -1 }, /* (277) literal ::= NK_INTEGER */ + { 247, -1 }, /* (278) literal ::= NK_FLOAT */ + { 247, -1 }, /* (279) literal ::= NK_STRING */ + { 247, -1 }, /* (280) literal ::= NK_BOOL */ + { 247, -2 }, /* (281) literal ::= TIMESTAMP NK_STRING */ + { 247, -1 }, /* (282) literal ::= duration_literal */ + { 247, -1 }, /* (283) literal ::= NULL */ + { 247, -1 }, /* (284) literal ::= NK_QUESTION */ + { 288, -1 }, /* (285) duration_literal ::= NK_VARIABLE */ + { 314, -1 }, /* (286) signed ::= NK_INTEGER */ + { 314, -2 }, /* (287) signed ::= NK_PLUS NK_INTEGER */ + { 314, -2 }, /* (288) signed ::= NK_MINUS NK_INTEGER */ + { 314, -1 }, /* (289) signed ::= NK_FLOAT */ + { 314, -2 }, /* (290) signed ::= NK_PLUS NK_FLOAT */ + { 314, -2 }, /* (291) signed ::= NK_MINUS NK_FLOAT */ + { 277, -1 }, /* (292) signed_literal ::= signed */ + { 277, -1 }, /* (293) signed_literal ::= NK_STRING */ + { 277, -1 }, /* (294) signed_literal ::= NK_BOOL */ + { 277, -2 }, /* (295) signed_literal ::= TIMESTAMP NK_STRING */ + { 277, -1 }, /* (296) signed_literal ::= duration_literal */ + { 277, -1 }, /* (297) signed_literal ::= NULL */ + { 277, -1 }, /* (298) signed_literal ::= literal_func */ + { 280, -1 }, /* (299) literal_list ::= signed_literal */ + { 280, -3 }, /* (300) literal_list ::= literal_list NK_COMMA signed_literal */ + { 255, -1 }, /* (301) db_name ::= NK_ID */ + { 283, -1 }, /* (302) table_name ::= NK_ID */ + { 275, -1 }, /* (303) column_name ::= NK_ID */ + { 290, -1 }, /* (304) function_name ::= NK_ID */ + { 316, -1 }, /* (305) table_alias ::= NK_ID */ + { 317, -1 }, /* (306) column_alias ::= NK_ID */ + { 249, -1 }, /* (307) user_name ::= NK_ID */ + { 296, -1 }, /* (308) index_name ::= NK_ID */ + { 302, -1 }, /* (309) topic_name ::= NK_ID */ + { 309, -1 }, /* (310) stream_name ::= NK_ID */ + { 304, -1 }, /* (311) cgroup_name ::= NK_ID */ + { 318, -1 }, /* (312) expression ::= literal */ + { 318, -1 }, /* (313) expression ::= pseudo_column */ + { 318, -1 }, /* (314) expression ::= column_reference */ + { 318, -1 }, /* (315) expression ::= function_expression */ + { 318, -1 }, /* (316) expression ::= subquery */ + { 318, -3 }, /* (317) expression ::= NK_LP expression NK_RP */ + { 318, -2 }, /* (318) expression ::= NK_PLUS expression */ + { 318, -2 }, /* (319) expression ::= NK_MINUS expression */ + { 318, -3 }, /* (320) expression ::= expression NK_PLUS expression */ + { 318, -3 }, /* (321) expression ::= expression NK_MINUS expression */ + { 318, -3 }, /* (322) expression ::= expression NK_STAR expression */ + { 318, -3 }, /* (323) expression ::= expression NK_SLASH expression */ + { 318, -3 }, /* (324) expression ::= expression NK_REM expression */ + { 318, -3 }, /* (325) expression ::= column_reference NK_ARROW NK_STRING */ + { 318, -3 }, /* (326) expression ::= expression NK_BITAND expression */ + { 318, -3 }, /* (327) expression ::= expression NK_BITOR expression */ + { 301, -1 }, /* (328) expression_list ::= expression */ + { 301, -3 }, /* (329) expression_list ::= expression_list NK_COMMA expression */ + { 320, -1 }, /* (330) column_reference ::= column_name */ + { 320, -3 }, /* (331) column_reference ::= table_name NK_DOT column_name */ + { 319, -1 }, /* (332) pseudo_column ::= ROWTS */ + { 319, -1 }, /* (333) pseudo_column ::= TBNAME */ + { 319, -3 }, /* (334) pseudo_column ::= table_name NK_DOT TBNAME */ + { 319, -1 }, /* (335) pseudo_column ::= QSTARTTS */ + { 319, -1 }, /* (336) pseudo_column ::= QENDTS */ + { 319, -1 }, /* (337) pseudo_column ::= WSTARTTS */ + { 319, -1 }, /* (338) pseudo_column ::= WENDTS */ + { 319, -1 }, /* (339) pseudo_column ::= WDURATION */ + { 321, -4 }, /* (340) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 321, -4 }, /* (341) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 321, -6 }, /* (342) function_expression ::= CAST NK_LP expression AS type_name NK_RP */ + { 321, -1 }, /* (343) function_expression ::= literal_func */ + { 315, -3 }, /* (344) literal_func ::= noarg_func NK_LP NK_RP */ + { 315, -1 }, /* (345) literal_func ::= NOW */ + { 325, -1 }, /* (346) noarg_func ::= NOW */ + { 325, -1 }, /* (347) noarg_func ::= TODAY */ + { 325, -1 }, /* (348) noarg_func ::= TIMEZONE */ + { 323, -1 }, /* (349) star_func ::= COUNT */ + { 323, -1 }, /* (350) star_func ::= FIRST */ + { 323, -1 }, /* (351) star_func ::= LAST */ + { 323, -1 }, /* (352) star_func ::= LAST_ROW */ + { 324, -1 }, /* (353) star_func_para_list ::= NK_STAR */ + { 324, -1 }, /* (354) star_func_para_list ::= other_para_list */ + { 326, -1 }, /* (355) other_para_list ::= star_func_para */ + { 326, -3 }, /* (356) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 327, -1 }, /* (357) star_func_para ::= expression */ + { 327, -3 }, /* (358) star_func_para ::= table_name NK_DOT NK_STAR */ + { 328, -3 }, /* (359) predicate ::= expression compare_op expression */ + { 328, -5 }, /* (360) predicate ::= expression BETWEEN expression AND expression */ + { 328, -6 }, /* (361) predicate ::= expression NOT BETWEEN expression AND expression */ + { 328, -3 }, /* (362) predicate ::= expression IS NULL */ + { 328, -4 }, /* (363) predicate ::= expression IS NOT NULL */ + { 328, -3 }, /* (364) predicate ::= expression in_op in_predicate_value */ + { 329, -1 }, /* (365) compare_op ::= NK_LT */ + { 329, -1 }, /* (366) compare_op ::= NK_GT */ + { 329, -1 }, /* (367) compare_op ::= NK_LE */ + { 329, -1 }, /* (368) compare_op ::= NK_GE */ + { 329, -1 }, /* (369) compare_op ::= NK_NE */ + { 329, -1 }, /* (370) compare_op ::= NK_EQ */ + { 329, -1 }, /* (371) compare_op ::= LIKE */ + { 329, -2 }, /* (372) compare_op ::= NOT LIKE */ + { 329, -1 }, /* (373) compare_op ::= MATCH */ + { 329, -1 }, /* (374) compare_op ::= NMATCH */ + { 329, -1 }, /* (375) compare_op ::= CONTAINS */ + { 330, -1 }, /* (376) in_op ::= IN */ + { 330, -2 }, /* (377) in_op ::= NOT IN */ + { 331, -3 }, /* (378) in_predicate_value ::= NK_LP expression_list NK_RP */ + { 332, -1 }, /* (379) boolean_value_expression ::= boolean_primary */ + { 332, -2 }, /* (380) boolean_value_expression ::= NOT boolean_primary */ + { 332, -3 }, /* (381) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 332, -3 }, /* (382) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 333, -1 }, /* (383) boolean_primary ::= predicate */ + { 333, -3 }, /* (384) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 334, -1 }, /* (385) common_expression ::= expression */ + { 334, -1 }, /* (386) common_expression ::= boolean_value_expression */ + { 335, 0 }, /* (387) from_clause_opt ::= */ + { 335, -2 }, /* (388) from_clause_opt ::= FROM table_reference_list */ + { 336, -1 }, /* (389) table_reference_list ::= table_reference */ + { 336, -3 }, /* (390) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 337, -1 }, /* (391) table_reference ::= table_primary */ + { 337, -1 }, /* (392) table_reference ::= joined_table */ + { 338, -2 }, /* (393) table_primary ::= table_name alias_opt */ + { 338, -4 }, /* (394) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 338, -2 }, /* (395) table_primary ::= subquery alias_opt */ + { 338, -1 }, /* (396) table_primary ::= parenthesized_joined_table */ + { 340, 0 }, /* (397) alias_opt ::= */ + { 340, -1 }, /* (398) alias_opt ::= table_alias */ + { 340, -2 }, /* (399) alias_opt ::= AS table_alias */ + { 341, -3 }, /* (400) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 341, -3 }, /* (401) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 339, -6 }, /* (402) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 342, 0 }, /* (403) join_type ::= */ + { 342, -1 }, /* (404) join_type ::= INNER */ + { 344, -12 }, /* (405) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 345, 0 }, /* (406) set_quantifier_opt ::= */ + { 345, -1 }, /* (407) set_quantifier_opt ::= DISTINCT */ + { 345, -1 }, /* (408) set_quantifier_opt ::= ALL */ + { 346, -1 }, /* (409) select_list ::= NK_STAR */ + { 346, -1 }, /* (410) select_list ::= select_sublist */ + { 354, -1 }, /* (411) select_sublist ::= select_item */ + { 354, -3 }, /* (412) select_sublist ::= select_sublist NK_COMMA select_item */ + { 355, -1 }, /* (413) select_item ::= common_expression */ + { 355, -2 }, /* (414) select_item ::= common_expression column_alias */ + { 355, -3 }, /* (415) select_item ::= common_expression AS column_alias */ + { 355, -3 }, /* (416) select_item ::= table_name NK_DOT NK_STAR */ + { 313, 0 }, /* (417) where_clause_opt ::= */ + { 313, -2 }, /* (418) where_clause_opt ::= WHERE search_condition */ + { 347, 0 }, /* (419) partition_by_clause_opt ::= */ + { 347, -3 }, /* (420) partition_by_clause_opt ::= PARTITION BY expression_list */ + { 351, 0 }, /* (421) twindow_clause_opt ::= */ + { 351, -6 }, /* (422) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 351, -4 }, /* (423) twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ + { 351, -6 }, /* (424) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 351, -8 }, /* (425) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 299, 0 }, /* (426) sliding_opt ::= */ + { 299, -4 }, /* (427) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 350, 0 }, /* (428) fill_opt ::= */ + { 350, -4 }, /* (429) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 350, -6 }, /* (430) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ + { 356, -1 }, /* (431) fill_mode ::= NONE */ + { 356, -1 }, /* (432) fill_mode ::= PREV */ + { 356, -1 }, /* (433) fill_mode ::= NULL */ + { 356, -1 }, /* (434) fill_mode ::= LINEAR */ + { 356, -1 }, /* (435) fill_mode ::= NEXT */ + { 352, 0 }, /* (436) group_by_clause_opt ::= */ + { 352, -3 }, /* (437) group_by_clause_opt ::= GROUP BY group_by_list */ + { 357, -1 }, /* (438) group_by_list ::= expression */ + { 357, -3 }, /* (439) group_by_list ::= group_by_list NK_COMMA expression */ + { 353, 0 }, /* (440) having_clause_opt ::= */ + { 353, -2 }, /* (441) having_clause_opt ::= HAVING search_condition */ + { 348, 0 }, /* (442) range_opt ::= */ + { 348, -6 }, /* (443) range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ + { 349, 0 }, /* (444) every_opt ::= */ + { 349, -4 }, /* (445) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 303, -4 }, /* (446) query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 358, -1 }, /* (447) query_expression_body ::= query_primary */ + { 358, -4 }, /* (448) query_expression_body ::= query_expression_body UNION ALL query_expression_body */ + { 358, -3 }, /* (449) query_expression_body ::= query_expression_body UNION query_expression_body */ + { 362, -1 }, /* (450) query_primary ::= query_specification */ + { 362, -6 }, /* (451) query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ + { 359, 0 }, /* (452) order_by_clause_opt ::= */ + { 359, -3 }, /* (453) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 360, 0 }, /* (454) slimit_clause_opt ::= */ + { 360, -2 }, /* (455) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 360, -4 }, /* (456) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 360, -4 }, /* (457) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 361, 0 }, /* (458) limit_clause_opt ::= */ + { 361, -2 }, /* (459) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 361, -4 }, /* (460) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 361, -4 }, /* (461) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 322, -3 }, /* (462) subquery ::= NK_LP query_expression NK_RP */ + { 343, -1 }, /* (463) search_condition ::= common_expression */ + { 363, -1 }, /* (464) sort_specification_list ::= sort_specification */ + { 363, -3 }, /* (465) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 364, -3 }, /* (466) sort_specification ::= expression ordering_specification_opt null_ordering_opt */ + { 365, 0 }, /* (467) ordering_specification_opt ::= */ + { 365, -1 }, /* (468) ordering_specification_opt ::= ASC */ + { 365, -1 }, /* (469) ordering_specification_opt ::= DESC */ + { 366, 0 }, /* (470) null_ordering_opt ::= */ + { 366, -2 }, /* (471) null_ordering_opt ::= NULLS FIRST */ + { 366, -2 }, /* (472) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3141,11 +3198,11 @@ static YYACTIONTYPE yy_reduce( YYMINORTYPE yylhsminor; case 0: /* cmd ::= CREATE ACCOUNT NK_ID PASS NK_STRING account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,243,&yymsp[0].minor); + yy_destructor(yypParser,245,&yymsp[0].minor); break; case 1: /* cmd ::= ALTER ACCOUNT NK_ID alter_account_options */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } - yy_destructor(yypParser,244,&yymsp[0].minor); + yy_destructor(yypParser,246,&yymsp[0].minor); break; case 2: /* account_options ::= */ { } @@ -3159,20 +3216,20 @@ static YYACTIONTYPE yy_reduce( case 9: /* account_options ::= account_options USERS literal */ yytestcase(yyruleno==9); case 10: /* account_options ::= account_options CONNS literal */ yytestcase(yyruleno==10); case 11: /* account_options ::= account_options STATE literal */ yytestcase(yyruleno==11); -{ yy_destructor(yypParser,243,&yymsp[-2].minor); +{ yy_destructor(yypParser,245,&yymsp[-2].minor); { } - yy_destructor(yypParser,245,&yymsp[0].minor); + yy_destructor(yypParser,247,&yymsp[0].minor); } break; case 12: /* alter_account_options ::= alter_account_option */ -{ yy_destructor(yypParser,246,&yymsp[0].minor); +{ yy_destructor(yypParser,248,&yymsp[0].minor); { } } break; case 13: /* alter_account_options ::= alter_account_options alter_account_option */ -{ yy_destructor(yypParser,244,&yymsp[-1].minor); +{ yy_destructor(yypParser,246,&yymsp[-1].minor); { } - yy_destructor(yypParser,246,&yymsp[0].minor); + yy_destructor(yypParser,248,&yymsp[0].minor); } break; case 14: /* alter_account_option ::= PASS literal */ @@ -3186,1303 +3243,1338 @@ static YYACTIONTYPE yy_reduce( case 22: /* alter_account_option ::= CONNS literal */ yytestcase(yyruleno==22); case 23: /* alter_account_option ::= STATE literal */ yytestcase(yyruleno==23); { } - yy_destructor(yypParser,245,&yymsp[0].minor); + yy_destructor(yypParser,247,&yymsp[0].minor); break; - case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy0); } + case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy0, yymsp[0].minor.yy629); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy57, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy241, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } + break; + case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy241, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; - case 26: /* cmd ::= ALTER USER user_name PRIVILEGE NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy57, TSDB_ALTER_USER_PRIVILEGES, &yymsp[0].minor.yy0); } + case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy241, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; - case 27: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy57); } + case 28: /* cmd ::= DROP USER user_name */ +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy241); } break; - case 28: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy389, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57); } + case 29: /* sysinfo_opt ::= */ +{ yymsp[1].minor.yy629 = 1; } break; - case 29: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy389, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57); } + case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ +{ yymsp[-1].minor.yy629 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 30: /* privileges ::= ALL */ -{ yymsp[0].minor.yy389 = PRIVILEGE_TYPE_ALL; } + case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy683, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241); } break; - case 31: /* privileges ::= priv_type_list */ - case 32: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==32); -{ yylhsminor.yy389 = yymsp[0].minor.yy389; } - yymsp[0].minor.yy389 = yylhsminor.yy389; + case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy683, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241); } break; - case 33: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy389 = yymsp[-2].minor.yy389 | yymsp[0].minor.yy389; } - yymsp[-2].minor.yy389 = yylhsminor.yy389; + case 33: /* privileges ::= ALL */ +{ yymsp[0].minor.yy683 = PRIVILEGE_TYPE_ALL; } break; - case 34: /* priv_type ::= READ */ -{ yymsp[0].minor.yy389 = PRIVILEGE_TYPE_READ; } + case 34: /* privileges ::= priv_type_list */ + case 35: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==35); +{ yylhsminor.yy683 = yymsp[0].minor.yy683; } + yymsp[0].minor.yy683 = yylhsminor.yy683; break; - case 35: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy389 = PRIVILEGE_TYPE_WRITE; } + case 36: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ +{ yylhsminor.yy683 = yymsp[-2].minor.yy683 | yymsp[0].minor.yy683; } + yymsp[-2].minor.yy683 = yylhsminor.yy683; break; - case 36: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy57 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy57 = yylhsminor.yy57; + case 37: /* priv_type ::= READ */ +{ yymsp[0].minor.yy683 = PRIVILEGE_TYPE_READ; } break; - case 37: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy57 = yymsp[-2].minor.yy57; } - yymsp[-2].minor.yy57 = yylhsminor.yy57; + case 38: /* priv_type ::= WRITE */ +{ yymsp[0].minor.yy683 = PRIVILEGE_TYPE_WRITE; } break; - case 38: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy57, NULL); } + case 39: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ +{ yylhsminor.yy241 = yymsp[-2].minor.yy0; } + yymsp[-2].minor.yy241 = yylhsminor.yy241; break; - case 39: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy0); } + case 40: /* priv_level ::= db_name NK_DOT NK_STAR */ +{ yylhsminor.yy241 = yymsp[-2].minor.yy241; } + yymsp[-2].minor.yy241 = yylhsminor.yy241; break; - case 40: /* cmd ::= DROP DNODE NK_INTEGER */ + case 41: /* cmd ::= CREATE DNODE dnode_endpoint */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy241, NULL); } + break; + case 42: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0); } + break; + case 43: /* cmd ::= DROP DNODE NK_INTEGER */ { pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 41: /* cmd ::= DROP DNODE dnode_endpoint */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy57); } + case 44: /* cmd ::= DROP DNODE dnode_endpoint */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[0].minor.yy241); } break; - case 42: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + case 45: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 43: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + case 46: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 44: /* cmd ::= ALTER ALL DNODES NK_STRING */ + case 47: /* cmd ::= ALTER ALL DNODES NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } break; - case 45: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + case 48: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 46: /* dnode_endpoint ::= NK_STRING */ - case 47: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==47); - case 48: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==48); - case 295: /* db_name ::= NK_ID */ yytestcase(yyruleno==295); - case 296: /* table_name ::= NK_ID */ yytestcase(yyruleno==296); - case 297: /* column_name ::= NK_ID */ yytestcase(yyruleno==297); - case 298: /* function_name ::= NK_ID */ yytestcase(yyruleno==298); - case 299: /* table_alias ::= NK_ID */ yytestcase(yyruleno==299); - case 300: /* column_alias ::= NK_ID */ yytestcase(yyruleno==300); - case 301: /* user_name ::= NK_ID */ yytestcase(yyruleno==301); - case 302: /* index_name ::= NK_ID */ yytestcase(yyruleno==302); - case 303: /* topic_name ::= NK_ID */ yytestcase(yyruleno==303); - case 304: /* stream_name ::= NK_ID */ yytestcase(yyruleno==304); - case 305: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==305); - case 338: /* noarg_func ::= NOW */ yytestcase(yyruleno==338); - case 339: /* noarg_func ::= TODAY */ yytestcase(yyruleno==339); - case 340: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==340); - case 341: /* star_func ::= COUNT */ yytestcase(yyruleno==341); - case 342: /* star_func ::= FIRST */ yytestcase(yyruleno==342); - case 343: /* star_func ::= LAST */ yytestcase(yyruleno==343); - case 344: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==344); -{ yylhsminor.yy57 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy57 = yylhsminor.yy57; - break; - case 49: /* cmd ::= ALTER LOCAL NK_STRING */ + case 49: /* dnode_endpoint ::= NK_STRING */ + case 50: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==50); + case 51: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==51); + case 301: /* db_name ::= NK_ID */ yytestcase(yyruleno==301); + case 302: /* table_name ::= NK_ID */ yytestcase(yyruleno==302); + case 303: /* column_name ::= NK_ID */ yytestcase(yyruleno==303); + case 304: /* function_name ::= NK_ID */ yytestcase(yyruleno==304); + case 305: /* table_alias ::= NK_ID */ yytestcase(yyruleno==305); + case 306: /* column_alias ::= NK_ID */ yytestcase(yyruleno==306); + case 307: /* user_name ::= NK_ID */ yytestcase(yyruleno==307); + case 308: /* index_name ::= NK_ID */ yytestcase(yyruleno==308); + case 309: /* topic_name ::= NK_ID */ yytestcase(yyruleno==309); + case 310: /* stream_name ::= NK_ID */ yytestcase(yyruleno==310); + case 311: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==311); + case 346: /* noarg_func ::= NOW */ yytestcase(yyruleno==346); + case 347: /* noarg_func ::= TODAY */ yytestcase(yyruleno==347); + case 348: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==348); + case 349: /* star_func ::= COUNT */ yytestcase(yyruleno==349); + case 350: /* star_func ::= FIRST */ yytestcase(yyruleno==350); + case 351: /* star_func ::= LAST */ yytestcase(yyruleno==351); + case 352: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==352); +{ yylhsminor.yy241 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy241 = yylhsminor.yy241; + break; + case 52: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 50: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + case 53: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 51: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + case 54: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); } break; - case 52: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + case 55: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); } break; - case 53: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + case 56: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); } break; - case 54: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + case 57: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); } break; - case 55: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + case 58: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); } break; - case 56: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + case 59: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); } break; - case 57: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + case 60: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); } break; - case 58: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + case 61: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; - case 59: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy481, &yymsp[-1].minor.yy57, yymsp[0].minor.yy392); } + case 62: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy197, &yymsp[-1].minor.yy241, yymsp[0].minor.yy442); } break; - case 60: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy481, &yymsp[0].minor.yy57); } + case 63: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy197, &yymsp[0].minor.yy241); } break; - case 61: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy57); } + case 64: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy241); } break; - case 62: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy57, yymsp[0].minor.yy392); } + case 65: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy241, yymsp[0].minor.yy442); } break; - case 63: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy481 = true; } + case 66: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy197 = true; } break; - case 64: /* not_exists_opt ::= */ - case 66: /* exists_opt ::= */ yytestcase(yyruleno==66); - case 238: /* analyze_opt ::= */ yytestcase(yyruleno==238); - case 246: /* agg_func_opt ::= */ yytestcase(yyruleno==246); - case 397: /* set_quantifier_opt ::= */ yytestcase(yyruleno==397); -{ yymsp[1].minor.yy481 = false; } + case 67: /* not_exists_opt ::= */ + case 69: /* exists_opt ::= */ yytestcase(yyruleno==69); + case 244: /* analyze_opt ::= */ yytestcase(yyruleno==244); + case 252: /* agg_func_opt ::= */ yytestcase(yyruleno==252); + case 406: /* set_quantifier_opt ::= */ yytestcase(yyruleno==406); +{ yymsp[1].minor.yy197 = false; } break; - case 65: /* exists_opt ::= IF EXISTS */ -{ yymsp[-1].minor.yy481 = true; } + case 68: /* exists_opt ::= IF EXISTS */ +{ yymsp[-1].minor.yy197 = true; } break; - case 67: /* db_options ::= */ -{ yymsp[1].minor.yy392 = createDefaultDatabaseOptions(pCxt); } + case 70: /* db_options ::= */ +{ yymsp[1].minor.yy442 = createDefaultDatabaseOptions(pCxt); } break; - case 68: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 71: /* db_options ::= db_options BUFFER NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 69: /* db_options ::= db_options CACHELAST NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 72: /* db_options ::= db_options CACHELAST NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_CACHELAST, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 70: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 73: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 71: /* db_options ::= db_options DURATION NK_INTEGER */ - case 72: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==72); -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 74: /* db_options ::= db_options DURATION NK_INTEGER */ + case 75: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==75); +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 73: /* db_options ::= db_options FSYNC NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 76: /* db_options ::= db_options FSYNC NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 74: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 77: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 75: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 78: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 76: /* db_options ::= db_options KEEP integer_list */ - case 77: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==77); -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_KEEP, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 79: /* db_options ::= db_options KEEP integer_list */ + case 80: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==80); +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_KEEP, yymsp[0].minor.yy368); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 78: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 81: /* db_options ::= db_options PAGES NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 79: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 82: /* db_options ::= db_options PAGESIZE NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 80: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 81: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 82: /* db_options ::= db_options STRICT NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 83: /* db_options ::= db_options WAL NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 84: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 85: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 86: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_RETENTIONS, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 87: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy392 = setDatabaseOption(pCxt, yymsp[-2].minor.yy392, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 88: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy392 = createAlterDatabaseOptions(pCxt); yylhsminor.yy392 = setAlterDatabaseOption(pCxt, yylhsminor.yy392, &yymsp[0].minor.yy221); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 89: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy392 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy392, &yymsp[0].minor.yy221); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; - break; - case 90: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 91: /* alter_db_option ::= CACHELAST NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 92: /* alter_db_option ::= FSYNC NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 93: /* alter_db_option ::= KEEP integer_list */ - case 94: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==94); -{ yymsp[-1].minor.yy221.type = DB_OPTION_KEEP; yymsp[-1].minor.yy221.pList = yymsp[0].minor.yy600; } - break; - case 95: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_PAGES; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 96: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 97: /* alter_db_option ::= STRICT NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_STRICT; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 98: /* alter_db_option ::= WAL NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = DB_OPTION_WAL; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } - break; - case 99: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy600 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 100: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 267: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==267); -{ yylhsminor.yy600 = addNodeToList(pCxt, yymsp[-2].minor.yy600, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 101: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy600 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 102: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy600 = addNodeToList(pCxt, yymsp[-2].minor.yy600, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 103: /* retention_list ::= retention */ - case 123: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==123); - case 126: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==126); - case 133: /* column_def_list ::= column_def */ yytestcase(yyruleno==133); - case 176: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==176); - case 181: /* col_name_list ::= col_name */ yytestcase(yyruleno==181); - case 226: /* func_list ::= func */ yytestcase(yyruleno==226); - case 293: /* literal_list ::= signed_literal */ yytestcase(yyruleno==293); - case 347: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==347); - case 402: /* select_sublist ::= select_item */ yytestcase(yyruleno==402); - case 455: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==455); -{ yylhsminor.yy600 = createNodeList(pCxt, yymsp[0].minor.yy392); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 104: /* retention_list ::= retention_list NK_COMMA retention */ - case 134: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==134); - case 177: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==177); - case 182: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==182); - case 227: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==227); - case 294: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==294); - case 348: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==348); - case 403: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==403); - case 456: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==456); -{ yylhsminor.yy600 = addNodeToList(pCxt, yymsp[-2].minor.yy600, yymsp[0].minor.yy392); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; - break; - case 105: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy392 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 106: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 108: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==108); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy481, yymsp[-5].minor.yy392, yymsp[-3].minor.yy600, yymsp[-1].minor.yy600, yymsp[0].minor.yy392); } - break; - case 107: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy600); } - break; - case 109: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy600); } - break; - case 110: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy481, yymsp[0].minor.yy392); } - break; - case 111: /* cmd ::= ALTER TABLE alter_table_clause */ - case 112: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==112); - case 270: /* cmd ::= query_expression */ yytestcase(yyruleno==270); -{ pCxt->pRootNode = yymsp[0].minor.yy392; } - break; - case 113: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy392 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; - break; - case 114: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy57, yymsp[0].minor.yy448); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; - break; - case 115: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy392 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy392, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy57); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; - break; - case 116: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy57, yymsp[0].minor.yy448); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; + case 83: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 84: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 85: /* db_options ::= db_options STRICT NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_STRICT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 86: /* db_options ::= db_options WAL NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 87: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 88: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 89: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_RETENTIONS, yymsp[0].minor.yy368); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 90: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy442 = setDatabaseOption(pCxt, yymsp[-2].minor.yy442, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 91: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy442 = createAlterDatabaseOptions(pCxt); yylhsminor.yy442 = setAlterDatabaseOption(pCxt, yylhsminor.yy442, &yymsp[0].minor.yy301); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 92: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy442 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy442, &yymsp[0].minor.yy301); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; + break; + case 93: /* alter_db_option ::= BUFFER NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 94: /* alter_db_option ::= CACHELAST NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_CACHELAST; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 95: /* alter_db_option ::= FSYNC NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 96: /* alter_db_option ::= KEEP integer_list */ + case 97: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==97); +{ yymsp[-1].minor.yy301.type = DB_OPTION_KEEP; yymsp[-1].minor.yy301.pList = yymsp[0].minor.yy368; } + break; + case 98: /* alter_db_option ::= PAGES NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_PAGES; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 99: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 100: /* alter_db_option ::= STRICT NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_STRICT; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 101: /* alter_db_option ::= WAL NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = DB_OPTION_WAL; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } + break; + case 102: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy368 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy368 = yylhsminor.yy368; + break; + case 103: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 273: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==273); +{ yylhsminor.yy368 = addNodeToList(pCxt, yymsp[-2].minor.yy368, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy368 = yylhsminor.yy368; + break; + case 104: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy368 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy368 = yylhsminor.yy368; + break; + case 105: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy368 = addNodeToList(pCxt, yymsp[-2].minor.yy368, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy368 = yylhsminor.yy368; + break; + case 106: /* retention_list ::= retention */ + case 126: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==126); + case 129: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==129); + case 136: /* column_def_list ::= column_def */ yytestcase(yyruleno==136); + case 179: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==179); + case 184: /* col_name_list ::= col_name */ yytestcase(yyruleno==184); + case 230: /* func_list ::= func */ yytestcase(yyruleno==230); + case 299: /* literal_list ::= signed_literal */ yytestcase(yyruleno==299); + case 355: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==355); + case 411: /* select_sublist ::= select_item */ yytestcase(yyruleno==411); + case 464: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==464); +{ yylhsminor.yy368 = createNodeList(pCxt, yymsp[0].minor.yy442); } + yymsp[0].minor.yy368 = yylhsminor.yy368; + break; + case 107: /* retention_list ::= retention_list NK_COMMA retention */ + case 137: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==137); + case 180: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==180); + case 185: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==185); + case 231: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==231); + case 300: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==300); + case 356: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==356); + case 412: /* select_sublist ::= select_sublist NK_COMMA select_item */ yytestcase(yyruleno==412); + case 465: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==465); +{ yylhsminor.yy368 = addNodeToList(pCxt, yymsp[-2].minor.yy368, yymsp[0].minor.yy442); } + yymsp[-2].minor.yy368 = yylhsminor.yy368; + break; + case 108: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy442 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 109: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 111: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==111); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy197, yymsp[-5].minor.yy442, yymsp[-3].minor.yy368, yymsp[-1].minor.yy368, yymsp[0].minor.yy442); } + break; + case 110: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy368); } + break; + case 112: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy368); } + break; + case 113: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy197, yymsp[0].minor.yy442); } + break; + case 114: /* cmd ::= ALTER TABLE alter_table_clause */ + case 115: /* cmd ::= ALTER STABLE alter_table_clause */ yytestcase(yyruleno==115); + case 276: /* cmd ::= query_expression */ yytestcase(yyruleno==276); +{ pCxt->pRootNode = yymsp[0].minor.yy442; } + break; + case 116: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy442 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy442, yymsp[0].minor.yy442); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; + break; + case 117: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy442 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy442, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy241, yymsp[0].minor.yy392); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; + break; + case 118: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy442 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy442, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy241); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; + break; + case 119: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy442 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy442, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy241, yymsp[0].minor.yy392); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; break; - case 117: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy392 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy57, &yymsp[0].minor.yy57); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; - break; - case 118: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy57, yymsp[0].minor.yy448); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; - break; - case 119: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy392 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy392, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy57); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 120: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy442 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy442, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; + break; + case 121: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy442 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy442, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy241, yymsp[0].minor.yy392); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; + break; + case 122: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy442 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy442, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy241); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 120: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy392 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy57, yymsp[0].minor.yy448); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; + case 123: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy442 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy442, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy241, yymsp[0].minor.yy392); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; break; - case 121: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy392 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy392, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy57, &yymsp[0].minor.yy57); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; - break; - case 122: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy392 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy392, &yymsp[-2].minor.yy57, yymsp[0].minor.yy392); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; + case 124: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy442 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy442, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; + break; + case 125: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy442 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy442, &yymsp[-2].minor.yy241, yymsp[0].minor.yy442); } + yymsp[-5].minor.yy442 = yylhsminor.yy442; break; - case 124: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 127: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==127); -{ yylhsminor.yy600 = addNodeToList(pCxt, yymsp[-1].minor.yy600, yymsp[0].minor.yy392); } - yymsp[-1].minor.yy600 = yylhsminor.yy600; + case 127: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 130: /* multi_drop_clause ::= multi_drop_clause drop_table_clause */ yytestcase(yyruleno==130); +{ yylhsminor.yy368 = addNodeToList(pCxt, yymsp[-1].minor.yy368, yymsp[0].minor.yy442); } + yymsp[-1].minor.yy368 = yylhsminor.yy368; break; - case 125: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ -{ yylhsminor.yy392 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy481, yymsp[-8].minor.yy392, yymsp[-6].minor.yy392, yymsp[-5].minor.yy600, yymsp[-2].minor.yy600, yymsp[0].minor.yy392); } - yymsp[-9].minor.yy392 = yylhsminor.yy392; + case 128: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_tags_opt TAGS NK_LP literal_list NK_RP table_options */ +{ yylhsminor.yy442 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy197, yymsp[-8].minor.yy442, yymsp[-6].minor.yy442, yymsp[-5].minor.yy368, yymsp[-2].minor.yy368, yymsp[0].minor.yy442); } + yymsp[-9].minor.yy442 = yylhsminor.yy442; break; - case 128: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy392 = createDropTableClause(pCxt, yymsp[-1].minor.yy481, yymsp[0].minor.yy392); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 131: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy442 = createDropTableClause(pCxt, yymsp[-1].minor.yy197, yymsp[0].minor.yy442); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 129: /* specific_tags_opt ::= */ - case 160: /* tags_def_opt ::= */ yytestcase(yyruleno==160); - case 410: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==410); - case 427: /* group_by_clause_opt ::= */ yytestcase(yyruleno==427); - case 443: /* order_by_clause_opt ::= */ yytestcase(yyruleno==443); -{ yymsp[1].minor.yy600 = NULL; } + case 132: /* specific_tags_opt ::= */ + case 163: /* tags_def_opt ::= */ yytestcase(yyruleno==163); + case 419: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==419); + case 436: /* group_by_clause_opt ::= */ yytestcase(yyruleno==436); + case 452: /* order_by_clause_opt ::= */ yytestcase(yyruleno==452); +{ yymsp[1].minor.yy368 = NULL; } break; - case 130: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ -{ yymsp[-2].minor.yy600 = yymsp[-1].minor.yy600; } + case 133: /* specific_tags_opt ::= NK_LP col_name_list NK_RP */ +{ yymsp[-2].minor.yy368 = yymsp[-1].minor.yy368; } break; - case 131: /* full_table_name ::= table_name */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy57, NULL); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 134: /* full_table_name ::= table_name */ +{ yylhsminor.yy442 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy241, NULL); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 132: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57, NULL); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 135: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy442 = createRealTableNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241, NULL); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 135: /* column_def ::= column_name type_name */ -{ yylhsminor.yy392 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy57, yymsp[0].minor.yy448, NULL); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 138: /* column_def ::= column_name type_name */ +{ yylhsminor.yy442 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy241, yymsp[0].minor.yy392, NULL); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 136: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy392 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy57, yymsp[-2].minor.yy448, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 139: /* column_def ::= column_name type_name COMMENT NK_STRING */ +{ yylhsminor.yy442 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 137: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 140: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 138: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 141: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 139: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 142: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 140: /* type_name ::= INT */ - case 141: /* type_name ::= INTEGER */ yytestcase(yyruleno==141); -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_INT); } + case 143: /* type_name ::= INT */ + case 144: /* type_name ::= INTEGER */ yytestcase(yyruleno==144); +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 142: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 145: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 143: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 146: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 144: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 147: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 145: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 148: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy392 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 146: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 149: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 147: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 150: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy392 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 148: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 151: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy392 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 149: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 152: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy392 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 150: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_UINT); } + case 153: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy392 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 151: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy448 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 154: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy392 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 152: /* type_name ::= JSON */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_JSON); } + case 155: /* type_name ::= JSON */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 153: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 156: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy392 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 154: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 157: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 155: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 158: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 156: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy448 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 159: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy392 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 157: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy448 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 160: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy392 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 158: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy448 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 161: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy392 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 159: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy448 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 162: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy392 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 161: /* tags_def_opt ::= tags_def */ - case 346: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==346); - case 401: /* select_list ::= select_sublist */ yytestcase(yyruleno==401); -{ yylhsminor.yy600 = yymsp[0].minor.yy600; } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 164: /* tags_def_opt ::= tags_def */ + case 354: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==354); + case 410: /* select_list ::= select_sublist */ yytestcase(yyruleno==410); +{ yylhsminor.yy368 = yymsp[0].minor.yy368; } + yymsp[0].minor.yy368 = yylhsminor.yy368; break; - case 162: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ -{ yymsp[-3].minor.yy600 = yymsp[-1].minor.yy600; } + case 165: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ +{ yymsp[-3].minor.yy368 = yymsp[-1].minor.yy368; } break; - case 163: /* table_options ::= */ -{ yymsp[1].minor.yy392 = createDefaultTableOptions(pCxt); } + case 166: /* table_options ::= */ +{ yymsp[1].minor.yy442 = createDefaultTableOptions(pCxt); } break; - case 164: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-2].minor.yy392, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 167: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-2].minor.yy442, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 165: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-2].minor.yy392, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 168: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-2].minor.yy442, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy368); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 166: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-2].minor.yy392, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy600); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 169: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-2].minor.yy442, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy368); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 167: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-4].minor.yy392, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy600); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; + case 170: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-4].minor.yy442, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy368); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; break; - case 168: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-2].minor.yy392, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 171: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-2].minor.yy442, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 169: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-4].minor.yy392, TABLE_OPTION_SMA, yymsp[-1].minor.yy600); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; + case 172: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-4].minor.yy442, TABLE_OPTION_SMA, yymsp[-1].minor.yy368); } + yymsp[-4].minor.yy442 = yylhsminor.yy442; break; - case 170: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy392 = createAlterTableOptions(pCxt); yylhsminor.yy392 = setTableOption(pCxt, yylhsminor.yy392, yymsp[0].minor.yy221.type, &yymsp[0].minor.yy221.val); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 173: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy442 = createAlterTableOptions(pCxt); yylhsminor.yy442 = setTableOption(pCxt, yylhsminor.yy442, yymsp[0].minor.yy301.type, &yymsp[0].minor.yy301.val); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 171: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy392 = setTableOption(pCxt, yymsp[-1].minor.yy392, yymsp[0].minor.yy221.type, &yymsp[0].minor.yy221.val); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 174: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy442 = setTableOption(pCxt, yymsp[-1].minor.yy442, yymsp[0].minor.yy301.type, &yymsp[0].minor.yy301.val); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 172: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy221.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } + case 175: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy301.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } break; - case 173: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy221.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy221.val = yymsp[0].minor.yy0; } + case 176: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy301.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy301.val = yymsp[0].minor.yy0; } break; - case 174: /* duration_list ::= duration_literal */ - case 320: /* expression_list ::= expression */ yytestcase(yyruleno==320); -{ yylhsminor.yy600 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 177: /* duration_list ::= duration_literal */ + case 328: /* expression_list ::= expression */ yytestcase(yyruleno==328); +{ yylhsminor.yy368 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy442)); } + yymsp[0].minor.yy368 = yylhsminor.yy368; break; - case 175: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 321: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==321); -{ yylhsminor.yy600 = addNodeToList(pCxt, yymsp[-2].minor.yy600, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 178: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 329: /* expression_list ::= expression_list NK_COMMA expression */ yytestcase(yyruleno==329); +{ yylhsminor.yy368 = addNodeToList(pCxt, yymsp[-2].minor.yy368, releaseRawExprNode(pCxt, yymsp[0].minor.yy442)); } + yymsp[-2].minor.yy368 = yylhsminor.yy368; break; - case 178: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[0].minor.yy57, NULL); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 181: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy442 = createFunctionNode(pCxt, &yymsp[0].minor.yy241, NULL); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 179: /* rollup_func_name ::= FIRST */ - case 180: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==180); -{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 182: /* rollup_func_name ::= FIRST */ + case 183: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==183); +{ yylhsminor.yy442 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 183: /* col_name ::= column_name */ -{ yylhsminor.yy392 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy57); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 186: /* col_name ::= column_name */ +{ yylhsminor.yy442 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy241); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 184: /* cmd ::= SHOW DNODES */ + case 187: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 185: /* cmd ::= SHOW USERS */ + case 188: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 186: /* cmd ::= SHOW DATABASES */ + case 189: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 187: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy392, yymsp[0].minor.yy392, OP_TYPE_LIKE); } + case 190: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy442, yymsp[0].minor.yy442, OP_TYPE_LIKE); } break; - case 188: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy392, yymsp[0].minor.yy392, OP_TYPE_LIKE); } + case 191: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy442, yymsp[0].minor.yy442, OP_TYPE_LIKE); } break; - case 189: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy392, NULL, OP_TYPE_LIKE); } + case 192: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy442, NULL, OP_TYPE_LIKE); } break; - case 190: /* cmd ::= SHOW MNODES */ + case 193: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 191: /* cmd ::= SHOW MODULES */ + case 194: /* cmd ::= SHOW MODULES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MODULES_STMT); } break; - case 192: /* cmd ::= SHOW QNODES */ + case 195: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 193: /* cmd ::= SHOW FUNCTIONS */ + case 196: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 194: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy392, yymsp[-1].minor.yy392, OP_TYPE_EQUAL); } + case 197: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy442, yymsp[-1].minor.yy442, OP_TYPE_EQUAL); } break; - case 195: /* cmd ::= SHOW STREAMS */ + case 198: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 196: /* cmd ::= SHOW ACCOUNTS */ + case 199: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 197: /* cmd ::= SHOW APPS */ + case 200: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 198: /* cmd ::= SHOW CONNECTIONS */ + case 201: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 199: /* cmd ::= SHOW LICENCE */ - case 200: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==200); + case 202: /* cmd ::= SHOW LICENCE */ + case 203: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==203); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCE_STMT); } break; - case 201: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy57); } + case 204: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy241); } break; - case 202: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy392); } + case 205: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy442); } break; - case 203: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy392); } + case 206: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy442); } break; - case 204: /* cmd ::= SHOW QUERIES */ + case 207: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 205: /* cmd ::= SHOW SCORES */ + case 208: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 206: /* cmd ::= SHOW TOPICS */ + case 209: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 207: /* cmd ::= SHOW VARIABLES */ -{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLE_STMT); } + case 210: /* cmd ::= SHOW VARIABLES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } + break; + case 211: /* cmd ::= SHOW LOCAL VARIABLES */ +{ pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 208: /* cmd ::= SHOW BNODES */ + case 212: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-1].minor.yy0)); } + break; + case 213: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 209: /* cmd ::= SHOW SNODES */ + case 214: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 210: /* cmd ::= SHOW CLUSTER */ + case 215: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 211: /* cmd ::= SHOW TRANSACTIONS */ + case 216: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 212: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy392); } + case 217: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy442); } + break; + case 218: /* db_name_cond_opt ::= */ + case 223: /* from_db_opt ::= */ yytestcase(yyruleno==223); +{ yymsp[1].minor.yy442 = createDefaultDatabaseCondValue(pCxt); } break; - case 213: /* db_name_cond_opt ::= */ - case 218: /* from_db_opt ::= */ yytestcase(yyruleno==218); -{ yymsp[1].minor.yy392 = createDefaultDatabaseCondValue(pCxt); } + case 219: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy241); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 214: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy57); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 220: /* like_pattern_opt ::= */ + case 227: /* index_options ::= */ yytestcase(yyruleno==227); + case 258: /* into_opt ::= */ yytestcase(yyruleno==258); + case 387: /* from_clause_opt ::= */ yytestcase(yyruleno==387); + case 417: /* where_clause_opt ::= */ yytestcase(yyruleno==417); + case 421: /* twindow_clause_opt ::= */ yytestcase(yyruleno==421); + case 426: /* sliding_opt ::= */ yytestcase(yyruleno==426); + case 428: /* fill_opt ::= */ yytestcase(yyruleno==428); + case 440: /* having_clause_opt ::= */ yytestcase(yyruleno==440); + case 442: /* range_opt ::= */ yytestcase(yyruleno==442); + case 444: /* every_opt ::= */ yytestcase(yyruleno==444); + case 454: /* slimit_clause_opt ::= */ yytestcase(yyruleno==454); + case 458: /* limit_clause_opt ::= */ yytestcase(yyruleno==458); +{ yymsp[1].minor.yy442 = NULL; } break; - case 215: /* like_pattern_opt ::= */ - case 223: /* index_options ::= */ yytestcase(yyruleno==223); - case 252: /* into_opt ::= */ yytestcase(yyruleno==252); - case 408: /* where_clause_opt ::= */ yytestcase(yyruleno==408); - case 412: /* twindow_clause_opt ::= */ yytestcase(yyruleno==412); - case 417: /* sliding_opt ::= */ yytestcase(yyruleno==417); - case 419: /* fill_opt ::= */ yytestcase(yyruleno==419); - case 431: /* having_clause_opt ::= */ yytestcase(yyruleno==431); - case 433: /* range_opt ::= */ yytestcase(yyruleno==433); - case 435: /* every_opt ::= */ yytestcase(yyruleno==435); - case 445: /* slimit_clause_opt ::= */ yytestcase(yyruleno==445); - case 449: /* limit_clause_opt ::= */ yytestcase(yyruleno==449); -{ yymsp[1].minor.yy392 = NULL; } + case 221: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 216: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 222: /* table_name_cond ::= table_name */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy241); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 217: /* table_name_cond ::= table_name */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy57); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 224: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy241); } break; - case 219: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy57); } + case 225: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy197, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy241, NULL, yymsp[0].minor.yy442); } break; - case 220: /* cmd ::= CREATE SMA INDEX not_exists_opt index_name ON table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy481, &yymsp[-3].minor.yy57, &yymsp[-1].minor.yy57, NULL, yymsp[0].minor.yy392); } + case 226: /* cmd ::= DROP INDEX exists_opt index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy197, &yymsp[0].minor.yy241); } break; - case 221: /* cmd ::= CREATE FULLTEXT INDEX not_exists_opt index_name ON table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_FULLTEXT, yymsp[-6].minor.yy481, &yymsp[-5].minor.yy57, &yymsp[-3].minor.yy57, yymsp[-1].minor.yy600, NULL); } + case 228: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ +{ yymsp[-8].minor.yy442 = createIndexOption(pCxt, yymsp[-6].minor.yy368, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), NULL, yymsp[0].minor.yy442); } break; - case 222: /* cmd ::= DROP INDEX exists_opt index_name ON table_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-3].minor.yy481, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57); } + case 229: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ +{ yymsp[-10].minor.yy442 = createIndexOption(pCxt, yymsp[-8].minor.yy368, releaseRawExprNode(pCxt, yymsp[-4].minor.yy442), releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), yymsp[0].minor.yy442); } break; - case 224: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt */ -{ yymsp[-8].minor.yy392 = createIndexOption(pCxt, yymsp[-6].minor.yy600, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), NULL, yymsp[0].minor.yy392); } + case 232: /* func ::= function_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy442 = createFunctionNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-1].minor.yy368); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 225: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt */ -{ yymsp[-10].minor.yy392 = createIndexOption(pCxt, yymsp[-8].minor.yy600, releaseRawExprNode(pCxt, yymsp[-4].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[0].minor.yy392); } + case 233: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy197, &yymsp[-2].minor.yy241, yymsp[0].minor.yy442); } break; - case 228: /* func ::= function_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy392 = createFunctionNode(pCxt, &yymsp[-3].minor.yy57, yymsp[-1].minor.yy600); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 234: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy197, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy241, false); } break; - case 229: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_expression */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-3].minor.yy481, &yymsp[-2].minor.yy57, yymsp[0].minor.yy392, NULL, NULL); } + case 235: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy197, &yymsp[-5].minor.yy241, &yymsp[0].minor.yy241, true); } break; - case 230: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy481, &yymsp[-3].minor.yy57, NULL, &yymsp[0].minor.yy57, NULL); } + case 236: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy197, &yymsp[-3].minor.yy241, yymsp[0].minor.yy442, false); } break; - case 231: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmt(pCxt, yymsp[-4].minor.yy481, &yymsp[-3].minor.yy57, NULL, NULL, yymsp[0].minor.yy392); } + case 237: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy197, &yymsp[-5].minor.yy241, yymsp[0].minor.yy442, true); } break; - case 232: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy481, &yymsp[0].minor.yy57); } + case 238: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy197, &yymsp[0].minor.yy241); } break; - case 233: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy481, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57); } + case 239: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy197, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241); } break; - case 234: /* cmd ::= DESC full_table_name */ - case 235: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==235); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy392); } + case 240: /* cmd ::= DESC full_table_name */ + case 241: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==241); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy442); } break; - case 236: /* cmd ::= RESET QUERY CACHE */ + case 242: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 237: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy481, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + case 243: /* cmd ::= EXPLAIN analyze_opt explain_options query_expression */ +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy197, yymsp[-1].minor.yy442, yymsp[0].minor.yy442); } break; - case 239: /* analyze_opt ::= ANALYZE */ - case 247: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==247); - case 398: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==398); -{ yymsp[0].minor.yy481 = true; } + case 245: /* analyze_opt ::= ANALYZE */ + case 253: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==253); + case 407: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==407); +{ yymsp[0].minor.yy197 = true; } break; - case 240: /* explain_options ::= */ -{ yymsp[1].minor.yy392 = createDefaultExplainOptions(pCxt); } + case 246: /* explain_options ::= */ +{ yymsp[1].minor.yy442 = createDefaultExplainOptions(pCxt); } break; - case 241: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy392 = setExplainVerbose(pCxt, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 247: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy442 = setExplainVerbose(pCxt, yymsp[-2].minor.yy442, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 242: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy392 = setExplainRatio(pCxt, yymsp[-2].minor.yy392, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 248: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy442 = setExplainRatio(pCxt, yymsp[-2].minor.yy442, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 243: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ -{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy600); } + case 249: /* cmd ::= COMPACT VNODES IN NK_LP integer_list NK_RP */ +{ pCxt->pRootNode = createCompactStmt(pCxt, yymsp[-1].minor.yy368); } break; - case 244: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy481, yymsp[-8].minor.yy481, &yymsp[-5].minor.yy57, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy448, yymsp[0].minor.yy228); } + case 250: /* cmd ::= CREATE agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-6].minor.yy197, yymsp[-8].minor.yy197, &yymsp[-5].minor.yy241, &yymsp[-3].minor.yy0, yymsp[-1].minor.yy392, yymsp[0].minor.yy578); } break; - case 245: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy481, &yymsp[0].minor.yy57); } + case 251: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy197, &yymsp[0].minor.yy241); } break; - case 248: /* bufsize_opt ::= */ -{ yymsp[1].minor.yy228 = 0; } + case 254: /* bufsize_opt ::= */ +{ yymsp[1].minor.yy578 = 0; } break; - case 249: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ -{ yymsp[-1].minor.yy228 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + case 255: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ +{ yymsp[-1].minor.yy578 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 250: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy481, &yymsp[-4].minor.yy57, yymsp[-2].minor.yy392, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } + case 256: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options into_opt AS query_expression */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-5].minor.yy197, &yymsp[-4].minor.yy241, yymsp[-2].minor.yy442, yymsp[-3].minor.yy442, yymsp[0].minor.yy442); } break; - case 251: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy481, &yymsp[0].minor.yy57); } + case 257: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy197, &yymsp[0].minor.yy241); } break; - case 253: /* into_opt ::= INTO full_table_name */ - case 379: /* from_clause ::= FROM table_reference_list */ yytestcase(yyruleno==379); - case 409: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==409); - case 432: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==432); -{ yymsp[-1].minor.yy392 = yymsp[0].minor.yy392; } + case 259: /* into_opt ::= INTO full_table_name */ + case 388: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==388); + case 418: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==418); + case 441: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==441); +{ yymsp[-1].minor.yy442 = yymsp[0].minor.yy442; } break; - case 254: /* stream_options ::= */ -{ yymsp[1].minor.yy392 = createStreamOptions(pCxt); } + case 260: /* stream_options ::= */ +{ yymsp[1].minor.yy442 = createStreamOptions(pCxt); } break; - case 255: /* stream_options ::= stream_options TRIGGER AT_ONCE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy392)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 261: /* stream_options ::= stream_options TRIGGER AT_ONCE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy442)->triggerType = STREAM_TRIGGER_AT_ONCE; yylhsminor.yy442 = yymsp[-2].minor.yy442; } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 256: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ -{ ((SStreamOptions*)yymsp[-2].minor.yy392)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 262: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ +{ ((SStreamOptions*)yymsp[-2].minor.yy442)->triggerType = STREAM_TRIGGER_WINDOW_CLOSE; yylhsminor.yy442 = yymsp[-2].minor.yy442; } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 257: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-3].minor.yy392)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy392)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); yylhsminor.yy392 = yymsp[-3].minor.yy392; } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 263: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-3].minor.yy442)->triggerType = STREAM_TRIGGER_MAX_DELAY; ((SStreamOptions*)yymsp[-3].minor.yy442)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy442); yylhsminor.yy442 = yymsp[-3].minor.yy442; } + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 258: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy392)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); yylhsminor.yy392 = yymsp[-2].minor.yy392; } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 264: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy442)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy442); yylhsminor.yy442 = yymsp[-2].minor.yy442; } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 259: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 265: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 260: /* cmd ::= KILL QUERY NK_STRING */ + case 266: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 261: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 267: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 262: /* cmd ::= BALANCE VGROUP */ + case 268: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 263: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 269: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 264: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy600); } + case 270: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy368); } break; - case 265: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 271: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 266: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy600 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 268: /* cmd ::= SYNCDB db_name REPLICA */ -{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy57); } - break; - case 269: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } - break; - case 271: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 272: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 273: /* literal ::= NK_STRING */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 274: /* literal ::= NK_BOOL */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 275: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; - break; - case 276: /* literal ::= duration_literal */ - case 286: /* signed_literal ::= signed */ yytestcase(yyruleno==286); - case 306: /* expression ::= literal */ yytestcase(yyruleno==306); - case 307: /* expression ::= pseudo_column */ yytestcase(yyruleno==307); - case 308: /* expression ::= column_reference */ yytestcase(yyruleno==308); - case 309: /* expression ::= function_expression */ yytestcase(yyruleno==309); - case 310: /* expression ::= subquery */ yytestcase(yyruleno==310); - case 335: /* function_expression ::= literal_func */ yytestcase(yyruleno==335); - case 371: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==371); - case 375: /* boolean_primary ::= predicate */ yytestcase(yyruleno==375); - case 377: /* common_expression ::= expression */ yytestcase(yyruleno==377); - case 378: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==378); - case 380: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==380); - case 382: /* table_reference ::= table_primary */ yytestcase(yyruleno==382); - case 383: /* table_reference ::= joined_table */ yytestcase(yyruleno==383); - case 387: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==387); - case 438: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==438); - case 441: /* query_primary ::= query_specification */ yytestcase(yyruleno==441); -{ yylhsminor.yy392 = yymsp[0].minor.yy392; } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 277: /* literal ::= NULL */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 278: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 279: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 280: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 281: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 282: /* signed ::= NK_MINUS NK_INTEGER */ + case 272: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy368 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 274: /* cmd ::= SYNCDB db_name REPLICA */ +{ pCxt->pRootNode = createSyncdbStmt(pCxt, &yymsp[-1].minor.yy241); } + break; + case 275: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy442, yymsp[0].minor.yy442); } + break; + case 277: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 278: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 279: /* literal ::= NK_STRING */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 280: /* literal ::= NK_BOOL */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 281: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; + break; + case 282: /* literal ::= duration_literal */ + case 292: /* signed_literal ::= signed */ yytestcase(yyruleno==292); + case 312: /* expression ::= literal */ yytestcase(yyruleno==312); + case 313: /* expression ::= pseudo_column */ yytestcase(yyruleno==313); + case 314: /* expression ::= column_reference */ yytestcase(yyruleno==314); + case 315: /* expression ::= function_expression */ yytestcase(yyruleno==315); + case 316: /* expression ::= subquery */ yytestcase(yyruleno==316); + case 343: /* function_expression ::= literal_func */ yytestcase(yyruleno==343); + case 379: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==379); + case 383: /* boolean_primary ::= predicate */ yytestcase(yyruleno==383); + case 385: /* common_expression ::= expression */ yytestcase(yyruleno==385); + case 386: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==386); + case 389: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==389); + case 391: /* table_reference ::= table_primary */ yytestcase(yyruleno==391); + case 392: /* table_reference ::= joined_table */ yytestcase(yyruleno==392); + case 396: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==396); + case 447: /* query_expression_body ::= query_primary */ yytestcase(yyruleno==447); + case 450: /* query_primary ::= query_specification */ yytestcase(yyruleno==450); +{ yylhsminor.yy442 = yymsp[0].minor.yy442; } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 283: /* literal ::= NULL */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 284: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 285: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 286: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 287: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 288: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 283: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 289: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 284: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 290: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 285: /* signed ::= NK_MINUS NK_FLOAT */ + case 291: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy442 = yylhsminor.yy442; + break; + case 293: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 287: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 294: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 288: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 295: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } break; - case 289: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + case 296: /* signed_literal ::= duration_literal */ + case 298: /* signed_literal ::= literal_func */ yytestcase(yyruleno==298); + case 357: /* star_func_para ::= expression */ yytestcase(yyruleno==357); + case 413: /* select_item ::= common_expression */ yytestcase(yyruleno==413); + case 463: /* search_condition ::= common_expression */ yytestcase(yyruleno==463); +{ yylhsminor.yy442 = releaseRawExprNode(pCxt, yymsp[0].minor.yy442); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 290: /* signed_literal ::= duration_literal */ - case 292: /* signed_literal ::= literal_func */ yytestcase(yyruleno==292); - case 349: /* star_func_para ::= expression */ yytestcase(yyruleno==349); - case 404: /* select_item ::= common_expression */ yytestcase(yyruleno==404); - case 454: /* search_condition ::= common_expression */ yytestcase(yyruleno==454); -{ yylhsminor.yy392 = releaseRawExprNode(pCxt, yymsp[0].minor.yy392); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 297: /* signed_literal ::= NULL */ +{ yylhsminor.yy442 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy442 = yylhsminor.yy442; break; - case 291: /* signed_literal ::= NULL */ -{ yylhsminor.yy392 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy392 = yylhsminor.yy392; + case 317: /* expression ::= NK_LP expression NK_RP */ + case 384: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==384); +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy442)); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 311: /* expression ::= NK_LP expression NK_RP */ - case 376: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==376); -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 318: /* expression ::= NK_PLUS expression */ +{ + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy442)); + } + yymsp[-1].minor.yy442 = yylhsminor.yy442; + break; + case 319: /* expression ::= NK_MINUS expression */ +{ + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy442), NULL)); + } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 312: /* expression ::= NK_PLUS expression */ + case 320: /* expression ::= expression NK_PLUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy392)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 313: /* expression ::= NK_MINUS expression */ + case 321: /* expression ::= expression NK_MINUS expression */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy392), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 314: /* expression ::= expression NK_PLUS expression */ + case 322: /* expression ::= expression NK_STAR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 315: /* expression ::= expression NK_MINUS expression */ + case 323: /* expression ::= expression NK_SLASH expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 316: /* expression ::= expression NK_STAR expression */ + case 324: /* expression ::= expression NK_REM expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 317: /* expression ::= expression NK_SLASH expression */ + case 325: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 318: /* expression ::= expression NK_REM expression */ + case 326: /* expression ::= expression NK_BITAND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 319: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 327: /* expression ::= expression NK_BITOR expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 322: /* column_reference ::= column_name */ -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy57, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy57)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 323: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57, createColumnNode(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy57)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 324: /* pseudo_column ::= ROWTS */ - case 325: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==325); - case 327: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==327); - case 328: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==328); - case 329: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==329); - case 330: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==330); - case 331: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==331); - case 337: /* literal_func ::= NOW */ yytestcase(yyruleno==337); -{ yylhsminor.yy392 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy392 = yylhsminor.yy392; - break; - case 326: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy57)))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 332: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 333: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==333); -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy57, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy57, yymsp[-1].minor.yy600)); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; - break; - case 334: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), yymsp[-1].minor.yy448)); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; - break; - case 336: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy57, NULL)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 345: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy600 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy600 = yylhsminor.yy600; - break; - case 350: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 407: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==407); -{ yylhsminor.yy392 = createColumnNode(pCxt, &yymsp[-2].minor.yy57, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; - break; - case 351: /* predicate ::= expression compare_op expression */ - case 356: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==356); + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 330: /* column_reference ::= column_name */ +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy241, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy241)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 331: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241, createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy241)); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 332: /* pseudo_column ::= ROWTS */ + case 333: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==333); + case 335: /* pseudo_column ::= QSTARTTS */ yytestcase(yyruleno==335); + case 336: /* pseudo_column ::= QENDTS */ yytestcase(yyruleno==336); + case 337: /* pseudo_column ::= WSTARTTS */ yytestcase(yyruleno==337); + case 338: /* pseudo_column ::= WENDTS */ yytestcase(yyruleno==338); + case 339: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==339); + case 345: /* literal_func ::= NOW */ yytestcase(yyruleno==345); +{ yylhsminor.yy442 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy442 = yylhsminor.yy442; + break; + case 334: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy241)))); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 340: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 341: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==341); +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy241, yymsp[-1].minor.yy368)); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; + break; + case 342: /* function_expression ::= CAST NK_LP expression AS type_name NK_RP */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy442), yymsp[-1].minor.yy392)); } + yymsp[-5].minor.yy442 = yylhsminor.yy442; + break; + case 344: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy241, NULL)); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 353: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy368 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy368 = yylhsminor.yy368; + break; + case 358: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 416: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==416); +{ yylhsminor.yy442 = createColumnNode(pCxt, &yymsp[-2].minor.yy241, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; + break; + case 359: /* predicate ::= expression compare_op expression */ + case 364: /* predicate ::= expression in_op in_predicate_value */ yytestcase(yyruleno==364); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy324, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy412, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 352: /* predicate ::= expression BETWEEN expression AND expression */ + case 360: /* predicate ::= expression BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy442), releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-4].minor.yy392 = yylhsminor.yy392; + yymsp[-4].minor.yy442 = yylhsminor.yy442; break; - case 353: /* predicate ::= expression NOT BETWEEN expression AND expression */ + case 361: /* predicate ::= expression NOT BETWEEN expression AND expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy392), releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy442), releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; + yymsp[-5].minor.yy442 = yylhsminor.yy442; break; - case 354: /* predicate ::= expression IS NULL */ + case 362: /* predicate ::= expression IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), NULL)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 355: /* predicate ::= expression IS NOT NULL */ + case 363: /* predicate ::= expression IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy442), NULL)); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 357: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy324 = OP_TYPE_LOWER_THAN; } + case 365: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy412 = OP_TYPE_LOWER_THAN; } break; - case 358: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy324 = OP_TYPE_GREATER_THAN; } + case 366: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy412 = OP_TYPE_GREATER_THAN; } break; - case 359: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy324 = OP_TYPE_LOWER_EQUAL; } + case 367: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy412 = OP_TYPE_LOWER_EQUAL; } break; - case 360: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy324 = OP_TYPE_GREATER_EQUAL; } + case 368: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy412 = OP_TYPE_GREATER_EQUAL; } break; - case 361: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy324 = OP_TYPE_NOT_EQUAL; } + case 369: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy412 = OP_TYPE_NOT_EQUAL; } break; - case 362: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy324 = OP_TYPE_EQUAL; } + case 370: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy412 = OP_TYPE_EQUAL; } break; - case 363: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy324 = OP_TYPE_LIKE; } + case 371: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy412 = OP_TYPE_LIKE; } break; - case 364: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy324 = OP_TYPE_NOT_LIKE; } + case 372: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy412 = OP_TYPE_NOT_LIKE; } break; - case 365: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy324 = OP_TYPE_MATCH; } + case 373: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy412 = OP_TYPE_MATCH; } break; - case 366: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy324 = OP_TYPE_NMATCH; } + case 374: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy412 = OP_TYPE_NMATCH; } break; - case 367: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy324 = OP_TYPE_JSON_CONTAINS; } + case 375: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy412 = OP_TYPE_JSON_CONTAINS; } break; - case 368: /* in_op ::= IN */ -{ yymsp[0].minor.yy324 = OP_TYPE_IN; } + case 376: /* in_op ::= IN */ +{ yymsp[0].minor.yy412 = OP_TYPE_IN; } break; - case 369: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy324 = OP_TYPE_NOT_IN; } + case 377: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy412 = OP_TYPE_NOT_IN; } break; - case 370: /* in_predicate_value ::= NK_LP expression_list NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy600)); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 378: /* in_predicate_value ::= NK_LP expression_list NK_RP */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy368)); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 372: /* boolean_value_expression ::= NOT boolean_primary */ + case 380: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy392), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy442), NULL)); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 373: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 381: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 374: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 382: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy392); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy392); - yylhsminor.yy392 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy442); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy442); + yylhsminor.yy442 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 381: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy392 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy392, yymsp[0].minor.yy392, NULL); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 390: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy442 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy442, yymsp[0].minor.yy442, NULL); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 384: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy57, &yymsp[0].minor.yy57); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 393: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy442 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 385: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy392 = createRealTableNode(pCxt, &yymsp[-3].minor.yy57, &yymsp[-1].minor.yy57, &yymsp[0].minor.yy57); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 394: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy442 = createRealTableNode(pCxt, &yymsp[-3].minor.yy241, &yymsp[-1].minor.yy241, &yymsp[0].minor.yy241); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 386: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy392 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392), &yymsp[0].minor.yy57); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 395: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy442 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy442), &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 388: /* alias_opt ::= */ -{ yymsp[1].minor.yy57 = nil_token; } + case 397: /* alias_opt ::= */ +{ yymsp[1].minor.yy241 = nil_token; } break; - case 389: /* alias_opt ::= table_alias */ -{ yylhsminor.yy57 = yymsp[0].minor.yy57; } - yymsp[0].minor.yy57 = yylhsminor.yy57; + case 398: /* alias_opt ::= table_alias */ +{ yylhsminor.yy241 = yymsp[0].minor.yy241; } + yymsp[0].minor.yy241 = yylhsminor.yy241; break; - case 390: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy57 = yymsp[0].minor.yy57; } + case 399: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy241 = yymsp[0].minor.yy241; } break; - case 391: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 392: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==392); -{ yymsp[-2].minor.yy392 = yymsp[-1].minor.yy392; } + case 400: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 401: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==401); +{ yymsp[-2].minor.yy442 = yymsp[-1].minor.yy442; } break; - case 393: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy392 = createJoinTableNode(pCxt, yymsp[-4].minor.yy204, yymsp[-5].minor.yy392, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-5].minor.yy392 = yylhsminor.yy392; + case 402: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy442 = createJoinTableNode(pCxt, yymsp[-4].minor.yy208, yymsp[-5].minor.yy442, yymsp[-2].minor.yy442, yymsp[0].minor.yy442); } + yymsp[-5].minor.yy442 = yylhsminor.yy442; break; - case 394: /* join_type ::= */ -{ yymsp[1].minor.yy204 = JOIN_TYPE_INNER; } + case 403: /* join_type ::= */ +{ yymsp[1].minor.yy208 = JOIN_TYPE_INNER; } break; - case 395: /* join_type ::= INNER */ -{ yymsp[0].minor.yy204 = JOIN_TYPE_INNER; } + case 404: /* join_type ::= INNER */ +{ yymsp[0].minor.yy208 = JOIN_TYPE_INNER; } break; - case 396: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 405: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy392 = createSelectStmt(pCxt, yymsp[-10].minor.yy481, yymsp[-9].minor.yy600, yymsp[-8].minor.yy392); - yymsp[-11].minor.yy392 = addWhereClause(pCxt, yymsp[-11].minor.yy392, yymsp[-7].minor.yy392); - yymsp[-11].minor.yy392 = addPartitionByClause(pCxt, yymsp[-11].minor.yy392, yymsp[-6].minor.yy600); - yymsp[-11].minor.yy392 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy392, yymsp[-2].minor.yy392); - yymsp[-11].minor.yy392 = addGroupByClause(pCxt, yymsp[-11].minor.yy392, yymsp[-1].minor.yy600); - yymsp[-11].minor.yy392 = addHavingClause(pCxt, yymsp[-11].minor.yy392, yymsp[0].minor.yy392); - yymsp[-11].minor.yy392 = addRangeClause(pCxt, yymsp[-11].minor.yy392, yymsp[-5].minor.yy392); - yymsp[-11].minor.yy392 = addEveryClause(pCxt, yymsp[-11].minor.yy392, yymsp[-4].minor.yy392); - yymsp[-11].minor.yy392 = addFillClause(pCxt, yymsp[-11].minor.yy392, yymsp[-3].minor.yy392); + yymsp[-11].minor.yy442 = createSelectStmt(pCxt, yymsp[-10].minor.yy197, yymsp[-9].minor.yy368, yymsp[-8].minor.yy442); + yymsp[-11].minor.yy442 = addWhereClause(pCxt, yymsp[-11].minor.yy442, yymsp[-7].minor.yy442); + yymsp[-11].minor.yy442 = addPartitionByClause(pCxt, yymsp[-11].minor.yy442, yymsp[-6].minor.yy368); + yymsp[-11].minor.yy442 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy442, yymsp[-2].minor.yy442); + yymsp[-11].minor.yy442 = addGroupByClause(pCxt, yymsp[-11].minor.yy442, yymsp[-1].minor.yy368); + yymsp[-11].minor.yy442 = addHavingClause(pCxt, yymsp[-11].minor.yy442, yymsp[0].minor.yy442); + yymsp[-11].minor.yy442 = addRangeClause(pCxt, yymsp[-11].minor.yy442, yymsp[-5].minor.yy442); + yymsp[-11].minor.yy442 = addEveryClause(pCxt, yymsp[-11].minor.yy442, yymsp[-4].minor.yy442); + yymsp[-11].minor.yy442 = addFillClause(pCxt, yymsp[-11].minor.yy442, yymsp[-3].minor.yy442); } break; - case 399: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy481 = false; } + case 408: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy197 = false; } break; - case 400: /* select_list ::= NK_STAR */ -{ yymsp[0].minor.yy600 = NULL; } + case 409: /* select_list ::= NK_STAR */ +{ yymsp[0].minor.yy368 = NULL; } break; - case 405: /* select_item ::= common_expression column_alias */ -{ yylhsminor.yy392 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392), &yymsp[0].minor.yy57); } - yymsp[-1].minor.yy392 = yylhsminor.yy392; + case 414: /* select_item ::= common_expression column_alias */ +{ yylhsminor.yy442 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy442), &yymsp[0].minor.yy241); } + yymsp[-1].minor.yy442 = yylhsminor.yy442; break; - case 406: /* select_item ::= common_expression AS column_alias */ -{ yylhsminor.yy392 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), &yymsp[0].minor.yy57); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 415: /* select_item ::= common_expression AS column_alias */ +{ yylhsminor.yy442 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), &yymsp[0].minor.yy241); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 411: /* partition_by_clause_opt ::= PARTITION BY expression_list */ - case 428: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==428); - case 444: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==444); -{ yymsp[-2].minor.yy600 = yymsp[0].minor.yy600; } + case 420: /* partition_by_clause_opt ::= PARTITION BY expression_list */ + case 437: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==437); + case 453: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==453); +{ yymsp[-2].minor.yy368 = yymsp[0].minor.yy368; } break; - case 413: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy392 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } + case 422: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy442 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy442), releaseRawExprNode(pCxt, yymsp[-1].minor.yy442)); } break; - case 414: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ -{ yymsp[-3].minor.yy392 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } + case 423: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expression NK_RP */ +{ yymsp[-3].minor.yy442 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy442)); } break; - case 415: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy392 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), NULL, yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + case 424: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy442 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy442), NULL, yymsp[-1].minor.yy442, yymsp[0].minor.yy442); } break; - case 416: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy392 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy392), releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), yymsp[-1].minor.yy392, yymsp[0].minor.yy392); } + case 425: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy442 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy442), releaseRawExprNode(pCxt, yymsp[-3].minor.yy442), yymsp[-1].minor.yy442, yymsp[0].minor.yy442); } break; - case 418: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - case 436: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==436); -{ yymsp[-3].minor.yy392 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy392); } + case 427: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + case 445: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==445); +{ yymsp[-3].minor.yy442 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy442); } break; - case 420: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy392 = createFillNode(pCxt, yymsp[-1].minor.yy270, NULL); } + case 429: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy442 = createFillNode(pCxt, yymsp[-1].minor.yy388, NULL); } break; - case 421: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy392 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy600)); } + case 430: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ +{ yymsp[-5].minor.yy442 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy368)); } break; - case 422: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy270 = FILL_MODE_NONE; } + case 431: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy388 = FILL_MODE_NONE; } break; - case 423: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy270 = FILL_MODE_PREV; } + case 432: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy388 = FILL_MODE_PREV; } break; - case 424: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy270 = FILL_MODE_NULL; } + case 433: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy388 = FILL_MODE_NULL; } break; - case 425: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy270 = FILL_MODE_LINEAR; } + case 434: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy388 = FILL_MODE_LINEAR; } break; - case 426: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy270 = FILL_MODE_NEXT; } + case 435: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy388 = FILL_MODE_NEXT; } break; - case 429: /* group_by_list ::= expression */ -{ yylhsminor.yy600 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[0].minor.yy600 = yylhsminor.yy600; + case 438: /* group_by_list ::= expression */ +{ yylhsminor.yy368 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } + yymsp[0].minor.yy368 = yylhsminor.yy368; break; - case 430: /* group_by_list ::= group_by_list NK_COMMA expression */ -{ yylhsminor.yy600 = addNodeToList(pCxt, yymsp[-2].minor.yy600, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy392))); } - yymsp[-2].minor.yy600 = yylhsminor.yy600; + case 439: /* group_by_list ::= group_by_list NK_COMMA expression */ +{ yylhsminor.yy368 = addNodeToList(pCxt, yymsp[-2].minor.yy368, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy442))); } + yymsp[-2].minor.yy368 = yylhsminor.yy368; break; - case 434: /* range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ -{ yymsp[-5].minor.yy392 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy392), releaseRawExprNode(pCxt, yymsp[-1].minor.yy392)); } + case 443: /* range_opt ::= RANGE NK_LP expression NK_COMMA expression NK_RP */ +{ yymsp[-5].minor.yy442 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy442), releaseRawExprNode(pCxt, yymsp[-1].minor.yy442)); } break; - case 437: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 446: /* query_expression ::= query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy392 = addOrderByClause(pCxt, yymsp[-3].minor.yy392, yymsp[-2].minor.yy600); - yylhsminor.yy392 = addSlimitClause(pCxt, yylhsminor.yy392, yymsp[-1].minor.yy392); - yylhsminor.yy392 = addLimitClause(pCxt, yylhsminor.yy392, yymsp[0].minor.yy392); + yylhsminor.yy442 = addOrderByClause(pCxt, yymsp[-3].minor.yy442, yymsp[-2].minor.yy368); + yylhsminor.yy442 = addSlimitClause(pCxt, yylhsminor.yy442, yymsp[-1].minor.yy442); + yylhsminor.yy442 = addLimitClause(pCxt, yylhsminor.yy442, yymsp[0].minor.yy442); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 439: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ -{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-3].minor.yy392 = yylhsminor.yy392; + case 448: /* query_expression_body ::= query_expression_body UNION ALL query_expression_body */ +{ yylhsminor.yy442 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy442, yymsp[0].minor.yy442); } + yymsp[-3].minor.yy442 = yylhsminor.yy442; break; - case 440: /* query_expression_body ::= query_expression_body UNION query_expression_body */ -{ yylhsminor.yy392 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy392, yymsp[0].minor.yy392); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 449: /* query_expression_body ::= query_expression_body UNION query_expression_body */ +{ yylhsminor.yy442 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy442, yymsp[0].minor.yy442); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 442: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ -{ yymsp[-5].minor.yy392 = yymsp[-4].minor.yy392; } - yy_destructor(yypParser,356,&yymsp[-3].minor); - yy_destructor(yypParser,357,&yymsp[-2].minor); - yy_destructor(yypParser,358,&yymsp[-1].minor); + case 451: /* query_primary ::= NK_LP query_expression_body order_by_clause_opt slimit_clause_opt limit_clause_opt NK_RP */ +{ yymsp[-5].minor.yy442 = yymsp[-4].minor.yy442; } + yy_destructor(yypParser,359,&yymsp[-3].minor); + yy_destructor(yypParser,360,&yymsp[-2].minor); + yy_destructor(yypParser,361,&yymsp[-1].minor); break; - case 446: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 450: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==450); -{ yymsp[-1].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 455: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 459: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==459); +{ yymsp[-1].minor.yy442 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 447: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 451: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==451); -{ yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 456: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 460: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==460); +{ yymsp[-3].minor.yy442 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 448: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 452: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==452); -{ yymsp[-3].minor.yy392 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 457: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 461: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==461); +{ yymsp[-3].minor.yy442 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 453: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy392 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy392); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 462: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy442 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy442); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 457: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy392 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy392), yymsp[-1].minor.yy162, yymsp[0].minor.yy529); } - yymsp[-2].minor.yy392 = yylhsminor.yy392; + case 466: /* sort_specification ::= expression ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy442 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy442), yymsp[-1].minor.yy362, yymsp[0].minor.yy417); } + yymsp[-2].minor.yy442 = yylhsminor.yy442; break; - case 458: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy162 = ORDER_ASC; } + case 467: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy362 = ORDER_ASC; } break; - case 459: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy162 = ORDER_ASC; } + case 468: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy362 = ORDER_ASC; } break; - case 460: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy162 = ORDER_DESC; } + case 469: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy362 = ORDER_DESC; } break; - case 461: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy529 = NULL_ORDER_DEFAULT; } + case 470: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy417 = NULL_ORDER_DEFAULT; } break; - case 462: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy529 = NULL_ORDER_FIRST; } + case 471: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy417 = NULL_ORDER_FIRST; } break; - case 463: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy529 = NULL_ORDER_LAST; } + case 472: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy417 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index b0c6181264c85aab6f732e24d2b46d33f2aa0502..8802f91bdf5d164f6ce5a39c09eb6eeff475599f 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -93,6 +93,16 @@ void generateInformationSchema(MockCatalogService* mcs) { .addColumn("db_name", TSDB_DATA_TYPE_BINARY, TSDB_DB_NAME_LEN); builder.done(); } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "configs", TSDB_SYSTEM_TABLE, 1) + .addColumn("name", TSDB_DATA_TYPE_BINARY, TSDB_CONFIG_OPTION_LEN); + builder.done(); + } + { + ITableBuilder& builder = mcs->createTableBuilder("information_schema", "dnode_variables", TSDB_SYSTEM_TABLE, 1) + .addColumn("dnode_id", TSDB_DATA_TYPE_INT); + builder.done(); + } } void generatePerformanceSchema(MockCatalogService* mcs) { @@ -187,6 +197,12 @@ void generateFunctions(MockCatalogService* mcs) { 8); } +void generateDnodes(MockCatalogService* mcs) { + mcs->createDnode(1, "host1", 7030); + mcs->createDnode(2, "host2", 7030); + mcs->createDnode(3, "host3", 7030); +} + } // namespace int32_t __catalogGetHandle(const char* clusterId, struct SCatalog** catalogHandle) { return 0; } @@ -241,6 +257,15 @@ int32_t __catalogGetTableIndex(SCatalog* pCtg, void* pTrans, const SEpSet* pMgmt return g_mockCatalogService->catalogGetTableIndex(pName, pRes); } +int32_t __catalogGetDnodeList(SCatalog* pCatalog, SRequestConnInfo* pConn, SArray** pDnodeList) { + return g_mockCatalogService->catalogGetDnodeList(pDnodeList); +} + +int32_t __catalogRefreshGetTableCfg(SCatalog* pCtg, SRequestConnInfo *pConn, const SName* pTableName, STableCfg** pCfg) { + *pCfg = (STableCfg*)taosMemoryCalloc(1, sizeof(STableCfg)); + return 0; +} + void initMetaDataEnv() { g_mockCatalogService.reset(new MockCatalogService()); @@ -258,6 +283,8 @@ void initMetaDataEnv() { stub.set(catalogRefreshGetTableMeta, __catalogRefreshGetTableMeta); stub.set(catalogRemoveTableMeta, __catalogRemoveTableMeta); stub.set(catalogGetTableIndex, __catalogGetTableIndex); + stub.set(catalogGetDnodeList, __catalogGetDnodeList); + stub.set(catalogRefreshGetTableCfg, __catalogRefreshGetTableCfg); // { // AddrAny any("libcatalog.so"); // std::map result; @@ -307,6 +334,7 @@ void generateMetaData() { generateTestST1(g_mockCatalogService.get()); generateTestST2(g_mockCatalogService.get()); generateFunctions(g_mockCatalogService.get()); + generateDnodes(g_mockCatalogService.get()); g_mockCatalogService->showTables(); } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 9758da78991adccfffedaa06ba4b0f0d7f58ac2f..4263b59cc3bdd2d7d69b51dd5b655213d5e4645a 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -165,6 +165,17 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } + int32_t catalogGetDnodeList(SArray** pDnodes) const { + SMetaRes res = {0}; + res.pRes = taosArrayInit(dnode_.size(), sizeof(SEpSet)); + for (const auto& dnode : dnode_) { + taosArrayPush((SArray*)res.pRes, &dnode.second); + } + *pDnodes = taosArrayInit(1, sizeof(SMetaRes)); + taosArrayPush(*pDnodes, &res); + return TSDB_CODE_SUCCESS; + } + int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { int32_t code = getAllTableMeta(pCatalogReq->pTableMeta, &pMetaData->pTableMeta); if (TSDB_CODE_SUCCESS == code) { @@ -188,6 +199,12 @@ class MockCatalogServiceImpl { if (TSDB_CODE_SUCCESS == code) { code = getAllTableIndex(pCatalogReq->pTableIndex, &pMetaData->pTableIndex); } + if (TSDB_CODE_SUCCESS == code && pCatalogReq->dNodeRequired) { + code = catalogGetDnodeList(&pMetaData->pDnodeList); + } + if (TSDB_CODE_SUCCESS == code) { + code = getAllTableCfg(pCatalogReq->pTableCfg, &pMetaData->pTableCfg); + } return code; } @@ -303,11 +320,18 @@ class MockCatalogServiceImpl { } } + void createDnode(int32_t dnodeId, const std::string& host, int16_t port) { + SEpSet epSet = {0}; + addEpIntoEpSet(&epSet, host.c_str(), port); + dnode_.insert(std::make_pair(dnodeId, epSet)); + } + private: typedef std::map> TableMetaCache; typedef std::map DbMetaCache; typedef std::map> UdfMetaCache; typedef std::map> IndexMetaCache; + typedef std::map DnodeCache; uint64_t getNextId() { return id_++; } @@ -527,11 +551,26 @@ class MockCatalogServiceImpl { return TSDB_CODE_SUCCESS; } + int32_t getAllTableCfg(SArray* pTableCfgReq, SArray** pTableCfgData) const { + if (NULL != pTableCfgReq) { + int32_t ntables = taosArrayGetSize(pTableCfgReq); + *pTableCfgData = taosArrayInit(ntables, sizeof(SMetaRes)); + for (int32_t i = 0; i < ntables; ++i) { + SMetaRes res = {0}; + res.pRes = taosMemoryCalloc(1, sizeof(STableCfg)); + res.code = TSDB_CODE_SUCCESS; + taosArrayPush(*pTableCfgData, &res); + } + } + return TSDB_CODE_SUCCESS; + } + uint64_t id_; std::unique_ptr builder_; DbMetaCache meta_; UdfMetaCache udf_; IndexMetaCache index_; + DnodeCache dnode_; }; MockCatalogService::MockCatalogService() : impl_(new MockCatalogServiceImpl()) {} @@ -557,6 +596,10 @@ void MockCatalogService::createFunction(const std::string& func, int8_t funcType void MockCatalogService::createSmaIndex(const SMCreateSmaReq* pReq) { impl_->createSmaIndex(pReq); } +void MockCatalogService::createDnode(int32_t dnodeId, const std::string& host, int16_t port) { + impl_->createDnode(dnodeId, host, port); +} + int32_t MockCatalogService::catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const { return impl_->catalogGetTableMeta(pTableName, pTableMeta); } @@ -581,6 +624,8 @@ int32_t MockCatalogService::catalogGetTableIndex(const SName* pTableName, SArray return impl_->catalogGetTableIndex(pTableName, pIndexes); } +int32_t MockCatalogService::catalogGetDnodeList(SArray** pDnodes) const { return impl_->catalogGetDnodeList(pDnodes); } + int32_t MockCatalogService::catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const { return impl_->catalogGetAllMeta(pCatalogReq, pMetaData); } diff --git a/source/libs/parser/test/mockCatalogService.h b/source/libs/parser/test/mockCatalogService.h index 1c98a7d9d52f47804d56fd144e9228b61f679ed1..932424823c226160d033430a2a95ac8c99d705b2 100644 --- a/source/libs/parser/test/mockCatalogService.h +++ b/source/libs/parser/test/mockCatalogService.h @@ -62,6 +62,7 @@ class MockCatalogService { void showTables() const; void createFunction(const std::string& func, int8_t funcType, int8_t outputType, int32_t outputLen, int32_t bufSize); void createSmaIndex(const SMCreateSmaReq* pReq); + void createDnode(int32_t dnodeId, const std::string& host, int16_t port); int32_t catalogGetTableMeta(const SName* pTableName, STableMeta** pTableMeta) const; int32_t catalogGetTableHashVgroup(const SName* pTableName, SVgroupInfo* vgInfo) const; @@ -69,6 +70,7 @@ class MockCatalogService { int32_t catalogGetDBVgInfo(const char* pDbFName, SArray** pVgList) const; int32_t catalogGetUdfInfo(const std::string& funcName, SFuncInfo* pInfo) const; int32_t catalogGetTableIndex(const SName* pTableName, SArray** pIndexes) const; + int32_t catalogGetDnodeList(SArray** pDnodes) const; int32_t catalogGetAllMeta(const SCatalogReq* pCatalogReq, SMetaData* pMetaData) const; private: diff --git a/source/libs/parser/test/parInitialATest.cpp b/source/libs/parser/test/parInitialATest.cpp index 021348dcc7310c819a1745fa244bbe158385d3c0..ee9dae2a0114ac38ea464d5f9dd0f30651284b88 100644 --- a/source/libs/parser/test/parInitialATest.cpp +++ b/source/libs/parser/test/parInitialATest.cpp @@ -121,7 +121,7 @@ TEST_F(ParserInitialATest, alterSTable) { int32_t len = snprintf(expect.name, sizeof(expect.name), "0.test.%s", pTbname); expect.name[len] = '\0'; expect.alterType = alterType; -// expect.ttl = ttl; + // expect.ttl = ttl; if (nullptr != pComment) { expect.comment = strdup(pComment); expect.commentLen = strlen(pComment); @@ -180,9 +180,9 @@ TEST_F(ParserInitialATest, alterSTable) { tFreeSMAltertbReq(&req); }); -// setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10); -// run("ALTER TABLE st1 TTL 10"); -// clearAlterStbReq(); + // setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, nullptr, 10); + // run("ALTER TABLE st1 TTL 10"); + // clearAlterStbReq(); setAlterStbReqFunc("st1", TSDB_ALTER_TABLE_UPDATE_OPTIONS, 0, nullptr, 0, 0, nullptr, "test"); run("ALTER TABLE st1 COMMENT 'test'"); @@ -381,9 +381,48 @@ TEST_F(ParserInitialATest, alterTableSemanticCheck) { TEST_F(ParserInitialATest, alterUser) { useDb("root", "test"); - run("ALTER user wxy PASS '123456'"); + SAlterUserReq expect = {0}; - run("ALTER user wxy privilege 'write'"); + auto clearAlterUserReq = [&]() { memset(&expect, 0, sizeof(SAlterUserReq)); }; + + auto setAlterUserReq = [&](const char* pUser, int8_t alterType, const char* pPass = nullptr, int8_t sysInfo = 0, + int8_t enable = 0) { + strcpy(expect.user, pUser); + expect.alterType = alterType; + expect.superUser = 0; + expect.sysInfo = sysInfo; + expect.enable = enable; + if (nullptr != pPass) { + strcpy(expect.pass, pPass); + } + strcpy(expect.dbname, "test"); + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_ALTER_USER_STMT); + SAlterUserReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSAlterUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.alterType, expect.alterType); + ASSERT_EQ(req.superUser, expect.superUser); + ASSERT_EQ(req.sysInfo, expect.sysInfo); + ASSERT_EQ(req.enable, expect.enable); + ASSERT_EQ(std::string(req.user), std::string(expect.user)); + ASSERT_EQ(std::string(req.pass), std::string(expect.pass)); + ASSERT_EQ(std::string(req.dbname), std::string(expect.dbname)); + }); + + setAlterUserReq("wxy", TSDB_ALTER_USER_PASSWD, "123456"); + run("ALTER USER wxy PASS '123456'"); + clearAlterUserReq(); + + setAlterUserReq("wxy", TSDB_ALTER_USER_ENABLE, nullptr, 0, 1); + run("ALTER USER wxy ENABLE 1"); + clearAlterUserReq(); + + setAlterUserReq("wxy", TSDB_ALTER_USER_SYSINFO, nullptr, 1); + run("ALTER USER wxy SYSINFO 1"); + clearAlterUserReq(); } TEST_F(ParserInitialATest, balanceVgroup) { diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index d188744adf57385980279d6490f1d0e86db7ddac..7de92bba0c71671f52c70ed8dfe01e92eede41b0 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -370,7 +370,7 @@ TEST_F(ParserInitialCTest, createStable) { expect.delay2 = delay2; expect.watermark1 = watermark1; expect.watermark2 = watermark2; -// expect.ttl = ttl; + // expect.ttl = ttl; if (nullptr != pComment) { expect.comment = strdup(pComment); expect.commentLen = strlen(pComment); @@ -414,7 +414,7 @@ TEST_F(ParserInitialCTest, createStable) { ASSERT_EQ(req.ttl, expect.ttl); ASSERT_EQ(req.numOfColumns, expect.numOfColumns); ASSERT_EQ(req.numOfTags, expect.numOfTags); -// ASSERT_EQ(req.commentLen, expect.commentLen); + // ASSERT_EQ(req.commentLen, expect.commentLen); ASSERT_EQ(req.ast1Len, expect.ast1Len); ASSERT_EQ(req.ast2Len, expect.ast2Len); @@ -621,10 +621,11 @@ TEST_F(ParserInitialCTest, createTopic) { auto clearCreateTopicReq = [&]() { memset(&expect, 0, sizeof(SCMCreateTopicReq)); }; auto setCreateTopicReqFunc = [&](const char* pTopicName, int8_t igExists, const char* pSql, const char* pAst, - const char* pDbName = nullptr, const char* pTbname = nullptr) { + const char* pDbName = nullptr, const char* pTbname = nullptr, int8_t withMeta = 0) { snprintf(expect.name, sizeof(expect.name), "0.%s", pTopicName); expect.igExists = igExists; expect.sql = (char*)pSql; + expect.withMeta = withMeta; if (nullptr != pTbname) { expect.subType = TOPIC_SUB_TYPE__TABLE; snprintf(expect.subStbName, sizeof(expect.subStbName), "0.%s.%s", pDbName, pTbname); @@ -647,6 +648,7 @@ TEST_F(ParserInitialCTest, createTopic) { ASSERT_EQ(req.igExists, expect.igExists); ASSERT_EQ(req.subType, expect.subType); ASSERT_EQ(std::string(req.sql), std::string(expect.sql)); + ASSERT_EQ(req.withMeta, expect.withMeta); switch (expect.subType) { case TOPIC_SUB_TYPE__DB: ASSERT_EQ(std::string(req.subDbName), std::string(expect.subDbName)); @@ -675,15 +677,55 @@ TEST_F(ParserInitialCTest, createTopic) { run("CREATE TOPIC tp1 AS DATABASE test"); clearCreateTopicReq(); + setCreateTopicReqFunc("tp1", 0, "create topic tp1 with meta as database test", nullptr, "test", nullptr, 1); + run("CREATE TOPIC tp1 WITH META AS DATABASE test"); + clearCreateTopicReq(); + setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 as stable st1", nullptr, "test", "st1"); run("CREATE TOPIC IF NOT EXISTS tp1 AS STABLE st1"); clearCreateTopicReq(); + + setCreateTopicReqFunc("tp1", 1, "create topic if not exists tp1 with meta as stable st1", nullptr, "test", "st1", 1); + run("CREATE TOPIC IF NOT EXISTS tp1 WITH META AS STABLE st1"); + clearCreateTopicReq(); } TEST_F(ParserInitialCTest, createUser) { useDb("root", "test"); + SCreateUserReq expect = {0}; + + auto clearCreateUserReq = [&]() { memset(&expect, 0, sizeof(SCreateUserReq)); }; + + auto setCreateUserReq = [&](const char* pUser, const char* pPass, int8_t sysInfo = 1) { + strcpy(expect.user, pUser); + strcpy(expect.pass, pPass); + expect.createType = 0; + expect.superUser = 0; + expect.sysInfo = sysInfo; + expect.enable = 1; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_CREATE_USER_STMT); + SCreateUserReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSCreateUserReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(req.createType, expect.createType); + ASSERT_EQ(req.superUser, expect.superUser); + ASSERT_EQ(req.sysInfo, expect.sysInfo); + ASSERT_EQ(req.enable, expect.enable); + ASSERT_EQ(std::string(req.user), std::string(expect.user)); + ASSERT_EQ(std::string(req.pass), std::string(expect.pass)); + }); + + setCreateUserReq("wxy", "123456"); run("CREATE USER wxy PASS '123456'"); + clearCreateUserReq(); + + setCreateUserReq("wxy1", "a123456", 1); + run("CREATE USER wxy1 PASS 'a123456' SYSINFO 1"); + clearCreateUserReq(); } } // namespace ParserTest diff --git a/source/libs/parser/test/parInitialDTest.cpp b/source/libs/parser/test/parInitialDTest.cpp index 4ecbb7b6d80990c38b37eb1106a5e8d0bf140b28..870afa694a21623f7bd1816c5eeb7a769ce9a6e8 100644 --- a/source/libs/parser/test/parInitialDTest.cpp +++ b/source/libs/parser/test/parInitialDTest.cpp @@ -132,7 +132,31 @@ TEST_F(ParserInitialDTest, dropDnode) { TEST_F(ParserInitialDTest, dropIndex) { useDb("root", "test"); - run("DROP index index1 on t1"); + SMDropSmaReq expect = {0}; + + auto clearDropSmaReq = [&]() { memset(&expect, 0, sizeof(SMDropSmaReq)); }; + + auto setDropSmaReq = [&](const char* pName, int8_t igNotExists = 0) { + sprintf(expect.name, "0.test.%s", pName); + expect.igNotExists = igNotExists; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_INDEX_STMT); + SMDropSmaReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMDropSmaReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.igNotExists, expect.igNotExists); + }); + + setDropSmaReq("index1"); + run("DROP INDEX index1"); + clearDropSmaReq(); + + setDropSmaReq("index2", 1); + run("DROP INDEX IF EXISTS index2"); + clearDropSmaReq(); } TEST_F(ParserInitialDTest, dropMnode) { @@ -159,7 +183,35 @@ TEST_F(ParserInitialDTest, dropSTable) { run("DROP STABLE st1"); } -// todo DROP stream +TEST_F(ParserInitialDTest, dropStream) { + useDb("root", "test"); + + SMDropStreamReq expect = {0}; + + auto clearDropStreamReq = [&]() { memset(&expect, 0, sizeof(SMDropStreamReq)); }; + + auto setDropStreamReq = [&](const char* pStream, int8_t igNotExists = 0) { + sprintf(expect.name, "0.%s", pStream); + expect.igNotExists = igNotExists; + }; + + setCheckDdlFunc([&](const SQuery* pQuery, ParserStage stage) { + ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_DROP_STREAM_STMT); + SMDropStreamReq req = {0}; + ASSERT_TRUE(TSDB_CODE_SUCCESS == tDeserializeSMDropStreamReq(pQuery->pCmdMsg->pMsg, pQuery->pCmdMsg->msgLen, &req)); + + ASSERT_EQ(std::string(req.name), std::string(expect.name)); + ASSERT_EQ(req.igNotExists, expect.igNotExists); + }); + + setDropStreamReq("s1"); + run("DROP STREAM s1"); + clearDropStreamReq(); + + setDropStreamReq("s2", 1); + run("DROP STREAM IF EXISTS s2"); + clearDropStreamReq(); +} TEST_F(ParserInitialDTest, dropTable) { useDb("root", "test"); diff --git a/source/libs/parser/test/parSelectTest.cpp b/source/libs/parser/test/parSelectTest.cpp index 11e09cd83d89f5ba02df18f5a439786ffe6d83c0..dd154427c61ae6a5834e65a72b4267d1f70aa553 100644 --- a/source/libs/parser/test/parSelectTest.cpp +++ b/source/libs/parser/test/parSelectTest.cpp @@ -58,6 +58,8 @@ TEST_F(ParserSelectTest, expression) { run("SELECT ts > 0, c1 < 20 and c2 = 'qaz' FROM t1"); run("SELECT ts > 0, c1 between 10 and 20 and c2 = 'qaz' FROM t1"); + + run("SELECT c1 | 10, c2 & 20, c4 | c5 FROM t1"); } TEST_F(ParserSelectTest, condition) { @@ -388,4 +390,10 @@ TEST_F(ParserSelectTest, informationSchema) { run("SELECT * FROM information_schema.user_databases WHERE name = 'information_schema'"); } +TEST_F(ParserSelectTest, withoutFrom) { + useDb("root", "test"); + + run("SELECT 1"); +} + } // namespace ParserTest diff --git a/source/libs/parser/test/parShowToUse.cpp b/source/libs/parser/test/parShowToUse.cpp index 940b6ea8ac101bff71416d0abd4e20e2d2cd908b..81a09114e3343c35da1cd972ef8f0ae63bb592a2 100644 --- a/source/libs/parser/test/parShowToUse.cpp +++ b/source/libs/parser/test/parShowToUse.cpp @@ -45,7 +45,7 @@ TEST_F(ParserShowToUseTest, showCreateSTable) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_SHOW_CREATE_STABLE_STMT); ASSERT_EQ(pQuery->execMode, QUERY_EXEC_MODE_LOCAL); ASSERT_TRUE(pQuery->haveResultSet); - ASSERT_NE(((SShowCreateTableStmt*)pQuery->pRoot)->pMeta, nullptr); + ASSERT_NE(((SShowCreateTableStmt*)pQuery->pRoot)->pCfg, nullptr); }); run("SHOW CREATE STABLE st1"); @@ -58,7 +58,7 @@ TEST_F(ParserShowToUseTest, showCreateTable) { ASSERT_EQ(nodeType(pQuery->pRoot), QUERY_NODE_SHOW_CREATE_TABLE_STMT); ASSERT_EQ(pQuery->execMode, QUERY_EXEC_MODE_LOCAL); ASSERT_TRUE(pQuery->haveResultSet); - ASSERT_NE(((SShowCreateTableStmt*)pQuery->pRoot)->pMeta, nullptr); + ASSERT_NE(((SShowCreateTableStmt*)pQuery->pRoot)->pCfg, nullptr); }); run("SHOW CREATE TABLE t1"); @@ -76,6 +76,12 @@ TEST_F(ParserShowToUseTest, showDnodes) { run("SHOW dnodes"); } +TEST_F(ParserShowToUseTest, showDnodeVariables) { + useDb("root", "test"); + + run("SHOW DNODE 1 VARIABLES"); +} + TEST_F(ParserShowToUseTest, showFunctions) { useDb("root", "test"); @@ -84,6 +90,12 @@ TEST_F(ParserShowToUseTest, showFunctions) { // todo SHOW licence +TEST_F(ParserShowToUseTest, showLocalVariables) { + useDb("root", "test"); + + run("SHOW LOCAL VARIABLES"); +} + TEST_F(ParserShowToUseTest, showIndexes) { useDb("root", "test"); @@ -157,7 +169,11 @@ TEST_F(ParserShowToUseTest, showUsers) { run("SHOW users"); } -// todo SHOW variables +TEST_F(ParserShowToUseTest, showVariables) { + useDb("root", "test"); + + run("SHOW VARIABLES"); +} TEST_F(ParserShowToUseTest, showVgroups) { useDb("root", "test"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c7490faa0c28524830f16515e2eb16278b894706..9dbd49e66c82e196d1277d6237cc7a6a9bcc9159 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -56,8 +56,12 @@ static EDealRes doRewriteExpr(SNode** pNode, void* pContext) { pCol->node.resType = pToBeRewrittenExpr->resType; strcpy(pCol->node.aliasName, pToBeRewrittenExpr->aliasName); strcpy(pCol->colName, ((SExprNode*)pExpr)->aliasName); - if (QUERY_NODE_FUNCTION == nodeType(pExpr) && FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pExpr)->funcType) { - pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + if (QUERY_NODE_FUNCTION == nodeType(pExpr)) { + if (FUNCTION_TYPE_WSTARTTS == ((SFunctionNode*)pExpr)->funcType) { + pCol->colId = PRIMARYKEY_TIMESTAMP_COL_ID; + } else if (FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pExpr)->funcType) { + pCol->colType = COLUMN_TYPE_TBNAME; + } } nodesDestroyNode(*pNode); *pNode = (SNode*)pCol; @@ -292,6 +296,59 @@ static int32_t createScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect return code; } +static int32_t createColumnByLastRow(SNodeList* pFuncs, SNodeList** pOutput) { + int32_t code = TSDB_CODE_SUCCESS; + SNodeList* pCols = NULL; + SNode* pFunc = NULL; + FOREACH(pFunc, pFuncs) { + SFunctionNode* pLastRow = (SFunctionNode*)pFunc; + SColumnNode* pCol = (SColumnNode*)nodesListGetNode(pLastRow->pParameterList, 0); + snprintf(pCol->colName, sizeof(pCol->colName), "%s", pLastRow->node.aliasName); + snprintf(pCol->node.aliasName, sizeof(pCol->colName), "%s", pLastRow->node.aliasName); + NODES_CLEAR_LIST(pLastRow->pParameterList); + code = nodesListMakeStrictAppend(&pCols, (SNode*)pCol); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + if (TSDB_CODE_SUCCESS == code) { + *pOutput = pCols; + } else { + nodesDestroyList(pCols); + } + return code; +} + +static int32_t createLastRowScanLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SRealTableNode* pRealTable, + SLogicNode** pLogicNode) { + SScanLogicNode* pScan = NULL; + int32_t code = makeScanLogicNode(pCxt, pRealTable, false, (SLogicNode**)&pScan); + + SNodeList* pFuncs = NULL; + if (TSDB_CODE_SUCCESS == code) { + pScan->scanType = SCAN_TYPE_LAST_ROW; + code = nodesCollectFuncs(pSelect, SQL_CLAUSE_FROM, fmIsLastRowFunc, &pFuncs); + } + if (TSDB_CODE_SUCCESS == code) { + code = rewriteExprsForSelect(pFuncs, pSelect, SQL_CLAUSE_FROM); + } + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByLastRow(pFuncs, &pScan->pScanCols); + } + if (TSDB_CODE_SUCCESS == code) { + code = createColumnByRewriteExprs(pScan->pScanCols, &pScan->node.pTargets); + } + + if (TSDB_CODE_SUCCESS == code) { + *pLogicNode = (SLogicNode*)pScan; + } else { + nodesDestroyNode((SNode*)pScan); + } + nodesDestroyList(pFuncs); + + return code; +} + static int32_t createSubqueryLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, STempTableNode* pTable, SLogicNode** pLogicNode) { return createQueryLogicNode(pCxt, pTable->pSubquery, pLogicNode); @@ -363,7 +420,11 @@ static int32_t doCreateLogicNodeByTable(SLogicPlanContext* pCxt, SSelectStmt* pS SLogicNode** pLogicNode) { switch (nodeType(pTable)) { case QUERY_NODE_REAL_TABLE: - return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode); + if (pSelect->hasLastRowFunc) { + return createLastRowScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode); + } else { + return createScanLogicNode(pCxt, pSelect, (SRealTableNode*)pTable, pLogicNode); + } case QUERY_NODE_TEMP_TABLE: return createSubqueryLogicNode(pCxt, pSelect, (STempTableNode*)pTable, pLogicNode); case QUERY_NODE_JOIN_TABLE: @@ -840,7 +901,12 @@ static int32_t createDistinctLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSe return code; } -static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { +static int32_t createSelectWithoutFromLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, + SLogicNode** pLogicNode) { + return createProjectLogicNode(pCxt, pSelect, pLogicNode); +} + +static int32_t createSelectFromLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { SLogicNode* pRoot = NULL; int32_t code = createLogicNodeByTable(pCxt, pSelect, pSelect->pFromTable, &pRoot); if (TSDB_CODE_SUCCESS == code) { @@ -880,6 +946,14 @@ static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSele return code; } +static int32_t createSelectLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pSelect, SLogicNode** pLogicNode) { + if (NULL == pSelect->pFromTable) { + return createSelectWithoutFromLogicNode(pCxt, pSelect, pLogicNode); + } else { + return createSelectFromLogicNode(pCxt, pSelect, pLogicNode); + } +} + static int32_t createSetOpRootLogicNode(SLogicPlanContext* pCxt, SSetOperator* pSetOperator, FCreateSetOpLogicNode func, SLogicNode** pRoot) { return createRootLogicNode(pCxt, pSetOperator, pSetOperator->precision, (FCreateLogicNode)func, pRoot); @@ -1086,8 +1160,8 @@ static int32_t createDeleteScanLogicNode(SLogicPlanContext* pCxt, SDeleteStmt* p } } - if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagIndexCond) { - pScan->pTagCond = nodesCloneNode(pDelete->pTagIndexCond); + if (TSDB_CODE_SUCCESS == code && NULL != pDelete->pTagCond) { + pScan->pTagCond = nodesCloneNode(pDelete->pTagCond); if (NULL == pScan->pTagCond) { code = TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 5ed49f6aaed632245456a0c64e9145becca23965..5b9dbe83883a0de9fb46b9312396140ed18e6bb8 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -15,7 +15,6 @@ #include "filter.h" #include "functionMgt.h" -#include "index.h" #include "planInt.h" #include "ttime.h" @@ -79,7 +78,8 @@ static SLogicNode* optFindPossibleNode(SLogicNode* pNode, FMayBeOptimized func) EDealRes osdHaveNormalColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType); + // *((bool*)pContext) = (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType); + *((bool*)pContext) = true; return *((bool*)pContext) ? DEAL_RES_END : DEAL_RES_IGNORE_CHILD; } return DEAL_RES_CONTINUE; @@ -98,11 +98,6 @@ static bool osdMayBeOptimized(SLogicNode* pNode) { if (QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(pNode)) { return false; } - // todo: release after function splitting - if (TSDB_SUPER_TABLE == ((SScanLogicNode*)pNode)->tableType && - SCAN_TYPE_STREAM != ((SScanLogicNode*)pNode)->scanType) { - return false; - } if (NULL == pNode->pParent || (QUERY_NODE_LOGIC_PLAN_WINDOW != nodeType(pNode->pParent) && QUERY_NODE_LOGIC_PLAN_AGG != nodeType(pNode->pParent) && QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode->pParent))) { @@ -293,42 +288,22 @@ static int32_t cpdCondAppend(SNode** pCond, SNode** pAdditionalCond) { return code; } -static int32_t cpdCalcTimeRange(SScanLogicNode* pScan, SNode** pPrimaryKeyCond, SNode** pOtherCond) { - bool isStrict = false; - int32_t code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); - if (TSDB_CODE_SUCCESS == code) { - if (isStrict) { - nodesDestroyNode(*pPrimaryKeyCond); - } else { - code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); - } - *pPrimaryKeyCond = NULL; - } - return code; -} - -static int32_t cpdApplyTagIndex(SScanLogicNode* pScan, SNode** pTagCond, SNode** pOtherCond) { - int32_t code = TSDB_CODE_SUCCESS; - SIdxFltStatus idxStatus = idxGetFltStatus(*pTagCond); - switch (idxStatus) { - case SFLT_NOT_INDEX: - code = cpdCondAppend(pOtherCond, pTagCond); - break; - case SFLT_COARSE_INDEX: - pScan->pTagCond = nodesCloneNode(*pTagCond); - if (NULL == pScan->pTagCond) { - code = TSDB_CODE_OUT_OF_MEMORY; - break; +static int32_t cpdCalcTimeRange(SOptimizeContext* pCxt, SScanLogicNode* pScan, SNode** pPrimaryKeyCond, + SNode** pOtherCond) { + int32_t code = TSDB_CODE_SUCCESS; + if (pCxt->pPlanCxt->topicQuery || pCxt->pPlanCxt->streamQuery) { + code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); + } else { + bool isStrict = false; + code = filterGetTimeRange(*pPrimaryKeyCond, &pScan->scanRange, &isStrict); + if (TSDB_CODE_SUCCESS == code) { + if (isStrict) { + nodesDestroyNode(*pPrimaryKeyCond); + } else { + code = cpdCondAppend(pOtherCond, pPrimaryKeyCond); } - code = cpdCondAppend(pOtherCond, pTagCond); - break; - case SFLT_ACCURATE_INDEX: - pScan->pTagCond = *pTagCond; - *pTagCond = NULL; - break; - default: - code = TSDB_CODE_FAILED; - break; + *pPrimaryKeyCond = NULL; + } } return code; } @@ -340,14 +315,11 @@ static int32_t cpdOptimizeScanCondition(SOptimizeContext* pCxt, SScanLogicNode* } SNode* pPrimaryKeyCond = NULL; - SNode* pTagCond = NULL; SNode* pOtherCond = NULL; - int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pTagCond, &pOtherCond); + int32_t code = nodesPartitionCond(&pScan->node.pConditions, &pPrimaryKeyCond, &pScan->pTagIndexCond, &pScan->pTagCond, + &pOtherCond); if (TSDB_CODE_SUCCESS == code && NULL != pPrimaryKeyCond) { - code = cpdCalcTimeRange(pScan, &pPrimaryKeyCond, &pOtherCond); - } - if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) { - code = cpdApplyTagIndex(pScan, &pTagCond, &pOtherCond); + code = cpdCalcTimeRange(pCxt, pScan, &pPrimaryKeyCond, &pOtherCond); } if (TSDB_CODE_SUCCESS == code) { pScan->node.pConditions = pOtherCond; @@ -769,7 +741,7 @@ static bool smaOptMayBeOptimized(SLogicNode* pNode) { } SScanLogicNode* pScan = (SScanLogicNode*)pNode; - if (0 == pScan->interval || NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) { + if (NULL == pScan->pSmaIndexes || NULL != pScan->node.pConditions) { return false; } @@ -1042,7 +1014,7 @@ static int32_t smaOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) static EDealRes partTagsOptHasColImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { - if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType) { + if (COLUMN_TYPE_TAG != ((SColumnNode*)pNode)->colType && COLUMN_TYPE_TBNAME != ((SColumnNode*)pNode)->colType) { *(bool*)pContext = true; return DEAL_RES_END; } @@ -1056,49 +1028,102 @@ static bool partTagsOptHasCol(SNodeList* pPartKeys) { return hasCol; } +static bool partTagsIsOptimizableNode(SLogicNode* pNode) { + return ((QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) || + (QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) && NULL != ((SAggLogicNode*)pNode)->pGroupKeys && + NULL != ((SAggLogicNode*)pNode)->pAggFuncs)) && + 1 == LIST_LENGTH(pNode->pChildren) && + QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(nodesListGetNode(pNode->pChildren, 0))); +} + +static SNodeList* partTagsGetPartKeys(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + return ((SPartitionLogicNode*)pNode)->pPartitionKeys; + } else { + return ((SAggLogicNode*)pNode)->pGroupKeys; + } +} + static bool partTagsOptMayBeOptimized(SLogicNode* pNode) { - if (QUERY_NODE_LOGIC_PLAN_PARTITION != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren) || - QUERY_NODE_LOGIC_PLAN_SCAN != nodeType(nodesListGetNode(pNode->pChildren, 0))) { + if (!partTagsIsOptimizableNode(pNode)) { return false; } - return !partTagsOptHasCol(((SPartitionLogicNode*)pNode)->pPartitionKeys); + return !partTagsOptHasCol(partTagsGetPartKeys(pNode)); +} + +static EDealRes partTagsOptRebuildTbanmeImpl(SNode** pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(*pNode) && COLUMN_TYPE_TBNAME == ((SColumnNode*)*pNode)->colType) { + SFunctionNode* pFunc = (SFunctionNode*)nodesMakeNode(QUERY_NODE_FUNCTION); + if (NULL == pFunc) { + *(int32_t*)pContext = TSDB_CODE_OUT_OF_MEMORY; + return DEAL_RES_ERROR; + } + strcpy(pFunc->functionName, "tbname"); + pFunc->funcType = FUNCTION_TYPE_TBNAME; + nodesDestroyNode(*pNode); + *pNode = (SNode*)pFunc; + return DEAL_RES_IGNORE_CHILD; + } + return DEAL_RES_CONTINUE; +} + +static int32_t partTagsOptRebuildTbanme(SNodeList* pPartKeys) { + int32_t code = TSDB_CODE_SUCCESS; + nodesRewriteExprs(pPartKeys, partTagsOptRebuildTbanmeImpl, &code); + return code; } static int32_t partTagsOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { - SPartitionLogicNode* pPart = - (SPartitionLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); - if (NULL == pPart) { + SLogicNode* pNode = optFindPossibleNode(pLogicSubplan->pNode, partTagsOptMayBeOptimized); + if (NULL == pNode) { return TSDB_CODE_SUCCESS; } - SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pPart->node.pChildren, 0); - TSWAP(pPart->pPartitionKeys, pScan->pPartTags); - int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pPart, (SLogicNode*)pScan); + int32_t code = TSDB_CODE_SUCCESS; + SScanLogicNode* pScan = (SScanLogicNode*)nodesListGetNode(pNode->pChildren, 0); + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode)) { + TSWAP(((SPartitionLogicNode*)pNode)->pPartitionKeys, pScan->pPartTags); + int32_t code = replaceLogicNode(pLogicSubplan, pNode, (SLogicNode*)pScan); + if (TSDB_CODE_SUCCESS == code) { + NODES_CLEAR_LIST(pNode->pChildren); + nodesDestroyNode((SNode*)pNode); + } + } else { + SNode* pGroupKey = NULL; + FOREACH(pGroupKey, ((SAggLogicNode*)pNode)->pGroupKeys) { + code = nodesListMakeStrictAppend( + &pScan->pPartTags, nodesCloneNode(nodesListGetNode(((SGroupingSetNode*)pGroupKey)->pParameterList, 0))); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + NODES_DESTORY_LIST(((SAggLogicNode*)pNode)->pGroupKeys); + } if (TSDB_CODE_SUCCESS == code) { - NODES_CLEAR_LIST(pPart->node.pChildren); - nodesDestroyNode((SNode*)pPart); + code = partTagsOptRebuildTbanme(pScan->pPartTags); } return code; } static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { - //TODO: enable this optimization after new mechanising that map projection and targets of project node + // TODO: enable this optimization after new mechanising that map projection and targets of project node if (NULL != pNode->pParent) { return false; } - + if (QUERY_NODE_LOGIC_PLAN_PROJECT != nodeType(pNode) || 1 != LIST_LENGTH(pNode->pChildren)) { return false; } SProjectLogicNode* pProjectNode = (SProjectLogicNode*)pNode; - if (-1 != pProjectNode->limit || -1 != pProjectNode->slimit || -1 != pProjectNode->offset || -1 != pProjectNode->soffset) { + if (-1 != pProjectNode->limit || -1 != pProjectNode->slimit || -1 != pProjectNode->offset || + -1 != pProjectNode->soffset) { return false; } SHashObj* pProjColNameHash = taosHashInit(16, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - SNode* pProjection; + SNode* pProjection; FOREACH(pProjection, pProjectNode->pProjections) { SExprNode* pExprNode = (SExprNode*)pProjection; if (QUERY_NODE_COLUMN != nodeType(pExprNode)) { @@ -1106,7 +1131,7 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return false; } - char* projColumnName = ((SColumnNode*)pProjection)->colName; + char* projColumnName = ((SColumnNode*)pProjection)->colName; int32_t* pExist = taosHashGet(pProjColNameHash, projColumnName, strlen(projColumnName)); if (NULL != pExist) { taosHashCleanup(pProjColNameHash); @@ -1121,9 +1146,10 @@ static bool eliminateProjOptMayBeOptimized(SLogicNode* pNode) { return true; } -static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, SProjectLogicNode* pProjectNode) { +static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan, + SProjectLogicNode* pProjectNode) { SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pProjectNode->node.pChildren, 0); - SNodeList* pNewChildTargets = nodesMakeList(); + SNodeList* pNewChildTargets = nodesMakeList(); SNode* pProjection = NULL; FOREACH(pProjection, pProjectNode->pProjections) { @@ -1137,7 +1163,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* } nodesDestroyList(pChild->pTargets); pChild->pTargets = pNewChildTargets; - + int32_t code = replaceLogicNode(pLogicSubplan, (SLogicNode*)pProjectNode, pChild); if (TSDB_CODE_SUCCESS == code) { NODES_CLEAR_LIST(pProjectNode->node.pChildren); @@ -1148,7 +1174,7 @@ static int32_t eliminateProjOptimizeImpl(SOptimizeContext* pCxt, SLogicSubplan* static int32_t eliminateProjOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SProjectLogicNode* pProjectNode = - (SProjectLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, eliminateProjOptMayBeOptimized); + (SProjectLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, eliminateProjOptMayBeOptimized); if (NULL == pProjectNode) { return TSDB_CODE_SUCCESS; @@ -1163,7 +1189,7 @@ static const SOptimizeRule optimizeRuleSet[] = { {.pName = "ConditionPushDown", .optimizeFunc = cpdOptimize}, {.pName = "OrderByPrimaryKey", .optimizeFunc = opkOptimize}, {.pName = "SmaIndex", .optimizeFunc = smaOptimize}, - {.pName = "PartitionByTags", .optimizeFunc = partTagsOptimize}, + // {.pName = "PartitionTags", .optimizeFunc = partTagsOptimize}, {.pName = "EliminateProject", .optimizeFunc = eliminateProjOptimize} }; // clang-format on diff --git a/source/libs/planner/src/planPhysiCreater.c b/source/libs/planner/src/planPhysiCreater.c index 6af5a15147b4805eec1c374020cdb93c1f67abf8..5f8fd4ac4ead1bef6f3017d8a1deb90a8121e04b 100644 --- a/source/libs/planner/src/planPhysiCreater.c +++ b/source/libs/planner/src/planPhysiCreater.c @@ -436,6 +436,15 @@ static int32_t createScanPhysiNodeFinalize(SPhysiPlanContext* pCxt, SSubplan* pS } } + if (TSDB_CODE_SUCCESS == code) { + if (NULL != pScanLogicNode->pTagIndexCond) { + pSubplan->pTagIndexCond = nodesCloneNode(pScanLogicNode->pTagIndexCond); + if (NULL == pSubplan->pTagIndexCond) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + } + if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pScanPhysiNode; } else { @@ -462,6 +471,8 @@ static ENodeType getScanOperatorType(EScanType scanType) { return QUERY_NODE_PHYSICAL_PLAN_TABLE_MERGE_SCAN; case SCAN_TYPE_BLOCK_INFO: return QUERY_NODE_PHYSICAL_PLAN_BLOCK_DIST_SCAN; + case SCAN_TYPE_LAST_ROW: + return QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN; default: break; } @@ -534,7 +545,11 @@ static int32_t createSystemTableScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* } SQueryNodeLoad node = {.addr = {.nodeId = MNODE_HANDLE, .epSet = pCxt->pPlanCxt->mgmtEpSet}, .load = 0}; taosArrayPush(pCxt->pExecNodeList, &node); - pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; + if (0 == strcmp(pScanLogicNode->tableName.tname, TSDB_INS_TABLE_DNODE_VARIABLES)) { + pScan->mgmtEpSet = pScanLogicNode->pVgroupList->vgroups->epSet; + } else { + pScan->mgmtEpSet = pCxt->pPlanCxt->mgmtEpSet; + } tNameGetFullDbName(&pScanLogicNode->tableName, pSubplan->dbFName); return createScanPhysiNodeFinalize(pCxt, pSubplan, pScanLogicNode, (SScanPhysiNode*)pScan, pPhyNode); @@ -555,6 +570,7 @@ static int32_t createScanPhysiNode(SPhysiPlanContext* pCxt, SSubplan* pSubplan, switch (pScanLogicNode->scanType) { case SCAN_TYPE_TAG: case SCAN_TYPE_BLOCK_INFO: + case SCAN_TYPE_LAST_ROW: return createSimpleScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); case SCAN_TYPE_TABLE: return createTableScanPhysiNode(pCxt, pSubplan, pScanLogicNode, pPhyNode); @@ -728,7 +744,7 @@ static int32_t rewritePrecalcExprs(SPhysiPlanContext* pCxt, SNodeList* pList, SN SRewritePrecalcExprsCxt cxt = {.errCode = TSDB_CODE_SUCCESS, .pPrecalcExprs = *pPrecalcExprs}; nodesRewriteExprs(*pRewrittenList, doRewritePrecalcExprs, &cxt); if (0 == LIST_LENGTH(cxt.pPrecalcExprs) || TSDB_CODE_SUCCESS != cxt.errCode) { - DESTORY_LIST(*pPrecalcExprs); + NODES_DESTORY_LIST(*pPrecalcExprs); } return cxt.errCode; } @@ -879,12 +895,15 @@ static int32_t createInterpFuncPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pCh pInterpFunc->interval = pFuncLogicNode->interval; pInterpFunc->fillMode = pFuncLogicNode->fillMode; pInterpFunc->pFillValues = nodesCloneNode(pFuncLogicNode->pFillValues); - pInterpFunc->pTimeSeries = nodesCloneNode(pFuncLogicNode->pTimeSeries); - if (NULL == pInterpFunc->pTimeSeries || (NULL != pFuncLogicNode->pFillValues && NULL == pInterpFunc->pFillValues)) { + if (NULL != pFuncLogicNode->pFillValues && NULL == pInterpFunc->pFillValues) { code = TSDB_CODE_OUT_OF_MEMORY; } } + if (TSDB_CODE_SUCCESS == code) { + code = setNodeSlotId(pCxt, pChildTupe->dataBlockId, -1, pFuncLogicNode->pTimeSeries, &pInterpFunc->pTimeSeries); + } + if (TSDB_CODE_SUCCESS == code) { *pPhyNode = (SPhysiNode*)pInterpFunc; } else { @@ -907,8 +926,16 @@ static int32_t createProjectPhysiNode(SPhysiPlanContext* pCxt, SNodeList* pChild pProject->slimit = pProjectLogicNode->slimit; pProject->soffset = pProjectLogicNode->soffset; - int32_t code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, - -1, pProjectLogicNode->pProjections, &pProject->pProjections); + int32_t code = TSDB_CODE_SUCCESS; + if (0 == LIST_LENGTH(pChildren)) { + pProject->pProjections = nodesCloneList(pProjectLogicNode->pProjections); + if (NULL == pProject->pProjections) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } else { + code = setListSlotId(pCxt, ((SPhysiNode*)nodesListGetNode(pChildren, 0))->pOutputDataBlockDesc->dataBlockId, -1, + pProjectLogicNode->pProjections, &pProject->pProjections); + } if (TSDB_CODE_SUCCESS == code) { code = addDataBlockSlotsForProject(pCxt, pProjectLogicNode->stmtName, pProject->pProjections, pProject->node.pOutputDataBlockDesc); @@ -1031,7 +1058,7 @@ static ENodeType getIntervalOperatorType(EWindowAlgorithm windowAlgo) { case INTERVAL_ALGO_HASH: return QUERY_NODE_PHYSICAL_PLAN_HASH_INTERVAL; case INTERVAL_ALGO_MERGE: - return QUERY_NODE_PHYSICAL_PLAN_MERGE_INTERVAL; + return QUERY_NODE_PHYSICAL_PLAN_MERGE_ALIGNED_INTERVAL; case INTERVAL_ALGO_STREAM_FINAL: return QUERY_NODE_PHYSICAL_PLAN_STREAM_FINAL_INTERVAL; case INTERVAL_ALGO_STREAM_SEMI: diff --git a/source/libs/planner/src/planSpliter.c b/source/libs/planner/src/planSpliter.c index 390d665760bcda5d5fd0acfecad27ca3bb82193b..9847755a7a21f69377089300545b1e34f53a144e 100644 --- a/source/libs/planner/src/planSpliter.c +++ b/source/libs/planner/src/planSpliter.c @@ -166,6 +166,31 @@ static bool stbSplHasMultiTbScan(bool streamQuery, SLogicNode* pNode) { return (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pChild) && stbSplIsMultiTbScan(streamQuery, (SScanLogicNode*)pChild)); } +static bool stbSplNeedSplitWindow(bool streamQuery, SLogicNode* pNode) { + SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode; + if (WINDOW_TYPE_INTERVAL == pWindow->winType) { + return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); + } + + if (WINDOW_TYPE_SESSION == pWindow->winType) { + if (!streamQuery) { + return stbSplHasMultiTbScan(streamQuery, pNode); + } else { + return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); + } + } + + if (WINDOW_TYPE_STATE == pWindow->winType) { + if (!streamQuery) { + return stbSplHasMultiTbScan(streamQuery, pNode); + } else { + return false; + } + } + + return false; +} + static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: @@ -174,13 +199,8 @@ static bool stbSplNeedSplit(bool streamQuery, SLogicNode* pNode) { return !(((SJoinLogicNode*)pNode)->isSingleTableJoin); case QUERY_NODE_LOGIC_PLAN_AGG: return !stbSplHasGatherExecFunc(((SAggLogicNode*)pNode)->pAggFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); - case QUERY_NODE_LOGIC_PLAN_WINDOW: { - SWindowLogicNode* pWindow = (SWindowLogicNode*)pNode; - if (WINDOW_TYPE_STATE == pWindow->winType || (!streamQuery && WINDOW_TYPE_SESSION == pWindow->winType) ) { - return false; - } - return !stbSplHasGatherExecFunc(pWindow->pFuncs) && stbSplHasMultiTbScan(streamQuery, pNode); - } + case QUERY_NODE_LOGIC_PLAN_WINDOW: + return stbSplNeedSplitWindow(streamQuery, pNode); case QUERY_NODE_LOGIC_PLAN_SORT: return stbSplHasMultiTbScan(streamQuery, pNode); default: @@ -380,6 +400,7 @@ static int32_t stbSplCreateExchangeNode(SSplitContext* pCxt, SLogicNode* pParent SExchangeLogicNode* pExchange = NULL; int32_t code = splCreateExchangeNode(pCxt, pPartChild, &pExchange); if (TSDB_CODE_SUCCESS == code) { + pExchange->node.pParent = pParent; code = nodesListMakeAppend(&pParent->pChildren, (SNode*)pExchange); } return code; @@ -476,26 +497,129 @@ static int32_t stbSplSplitSessionForStream(SSplitContext* pCxt, SStableSplitInfo return code; } +static void splSetTableScanType(SLogicNode* pNode, EScanType scanType) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + ((SScanLogicNode*)pNode)->scanType = scanType; + } else { + if (1 == LIST_LENGTH(pNode->pChildren)) { + splSetTableScanType((SLogicNode*)nodesListGetNode(pNode->pChildren, 0), scanType); + } + } +} + +static int32_t stbSplSplitSessionOrStateForBatch(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + SLogicNode* pWindow = pInfo->pSplitNode; + SLogicNode* pChild = (SLogicNode*)nodesListGetNode(pWindow->pChildren, 0); + + SNodeList* pMergeKeys = NULL; + int32_t code = stbSplCreateMergeKeysByPrimaryKey(((SWindowLogicNode*)pWindow)->pTspk, &pMergeKeys); + + if (TSDB_CODE_SUCCESS == code) { + code = stbSplCreateMergeNode(pCxt, pInfo->pSubplan, pChild, pMergeKeys, (SLogicNode*)pChild); + } + + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + (SNode*)splCreateScanSubplan(pCxt, pChild, SPLIT_FLAG_STABLE_SPLIT)); + } + + if (TSDB_CODE_SUCCESS == code) { + splSetTableScanType(pChild, SCAN_TYPE_TABLE_MERGE); + ++(pCxt->groupId); + } + + if (TSDB_CODE_SUCCESS == code) { + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); + } else { + nodesDestroyList(pMergeKeys); + } + + return code; +} + static int32_t stbSplSplitSession(SSplitContext* pCxt, SStableSplitInfo* pInfo) { if (pCxt->pPlanCxt->streamQuery) { return stbSplSplitSessionForStream(pCxt, pInfo); } else { - return TSDB_CODE_PLAN_INTERNAL_ERROR; + return stbSplSplitSessionOrStateForBatch(pCxt, pInfo); } } -static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { +static int32_t stbSplSplitStateForStream(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + return TSDB_CODE_PLAN_INTERNAL_ERROR; +} + +static int32_t stbSplSplitState(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (pCxt->pPlanCxt->streamQuery) { + return stbSplSplitStateForStream(pCxt, pInfo); + } else { + return stbSplSplitSessionOrStateForBatch(pCxt, pInfo); + } +} + +static SNodeList* stbSplGetPartKeys(SLogicNode* pNode) { + if (QUERY_NODE_LOGIC_PLAN_SCAN == nodeType(pNode)) { + return ((SScanLogicNode*)pNode)->pPartTags; + } else { + return NULL; + } +} + +static bool stbSplIsPartTbanme(SNodeList* pPartKeys) { + if (NULL == pPartKeys || 1 != LIST_LENGTH(pPartKeys)) { + return false; + } + SNode* pPartKey = nodesListGetNode(pPartKeys, 0); + return QUERY_NODE_FUNCTION == nodeType(pPartKey) && FUNCTION_TYPE_TBNAME == ((SFunctionNode*)pPartKey)->funcType; +} + +static bool stbSplIsMultiTableWinodw(SWindowLogicNode* pWindow) { + return stbSplIsPartTbanme(stbSplGetPartKeys((SLogicNode*)nodesListGetNode(pWindow->node.pChildren, 0))); +} + +static int32_t stbSplSplitWindowForMergeTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { switch (((SWindowLogicNode*)pInfo->pSplitNode)->winType) { case WINDOW_TYPE_INTERVAL: return stbSplSplitInterval(pCxt, pInfo); case WINDOW_TYPE_SESSION: return stbSplSplitSession(pCxt, pInfo); + case WINDOW_TYPE_STATE: + return stbSplSplitState(pCxt, pInfo); default: break; } return TSDB_CODE_PLAN_INTERNAL_ERROR; } +static int32_t stbSplSplitWindowForMultiTable(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (pCxt->pPlanCxt->streamQuery) { + SPLIT_FLAG_SET_MASK(pInfo->pSubplan->splitFlag, SPLIT_FLAG_STABLE_SPLIT); + return TSDB_CODE_SUCCESS; + } + + SExchangeLogicNode* pExchange = NULL; + int32_t code = splCreateExchangeNode(pCxt, pInfo->pSplitNode, &pExchange); + if (TSDB_CODE_SUCCESS == code) { + code = replaceLogicNode(pInfo->pSubplan, pInfo->pSplitNode, (SLogicNode*)pExchange); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeStrictAppend(&pInfo->pSubplan->pChildren, + (SNode*)splCreateScanSubplan(pCxt, pInfo->pSplitNode, SPLIT_FLAG_STABLE_SPLIT)); + } + pInfo->pSubplan->subplanType = SUBPLAN_TYPE_MERGE; + ++(pCxt->groupId); + return code; +} + +static int32_t stbSplSplitWindowNode(SSplitContext* pCxt, SStableSplitInfo* pInfo) { + if (stbSplIsMultiTableWinodw((SWindowLogicNode*)pInfo->pSplitNode)) { + return stbSplSplitWindowForMultiTable(pCxt, pInfo); + } else { + return stbSplSplitWindowForMergeTable(pCxt, pInfo); + } +} + static int32_t stbSplCreatePartAggNode(SAggLogicNode* pMergeAgg, SLogicNode** pOutput) { SNodeList* pFunc = pMergeAgg->pAggFuncs; pMergeAgg->pAggFuncs = NULL; @@ -885,7 +1009,7 @@ static int32_t unionSplitSubplan(SSplitContext* pCxt, SLogicSubplan* pUnionSubpl } if (TSDB_CODE_SUCCESS == code) { nodesDestroyList(pSubplanChildren); - DESTORY_LIST(pSplitNode->pChildren); + NODES_DESTORY_LIST(pSplitNode->pChildren); } return code; } diff --git a/source/libs/planner/test/planBasicTest.cpp b/source/libs/planner/test/planBasicTest.cpp index f74c7df355ad74b1db1192caf83c60fca7673f79..790c418b17a307a216d7b21735221ceb811e8710 100644 --- a/source/libs/planner/test/planBasicTest.cpp +++ b/source/libs/planner/test/planBasicTest.cpp @@ -83,3 +83,21 @@ TEST_F(PlanBasicTest, interpFunc) { run("SELECT INTERP(c1) FROM t1 RANGE('2017-7-14 18:00:00', '2017-7-14 19:00:00') EVERY(5s) FILL(LINEAR)"); } + +TEST_F(PlanBasicTest, lastRowFunc) { + useDb("root", "test"); + + run("SELECT LAST_ROW(c1) FROM t1"); + + run("SELECT LAST_ROW(*) FROM t1"); + + run("SELECT LAST_ROW(c1, c2) FROM t1"); + + run("SELECT LAST_ROW(c1) FROM st1"); +} + +TEST_F(PlanBasicTest, withoutFrom) { + useDb("root", "test"); + + run("SELECT 1"); +} diff --git a/source/libs/planner/test/planOptimizeTest.cpp b/source/libs/planner/test/planOptimizeTest.cpp index 6b514ef2c327757eab7af92a177e46a757a9f189..8b3d263d66dc2e9b94c0cd041cc510a0cc7b138e 100644 --- a/source/libs/planner/test/planOptimizeTest.cpp +++ b/source/libs/planner/test/planOptimizeTest.cpp @@ -40,19 +40,28 @@ TEST_F(PlanOptimizeTest, ConditionPushDown) { run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 or tag1 < 2"); run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello'"); + + run("SELECT ts, c1 FROM st1 WHERE tag1 > 4 AND tag2 = 'hello' AND c1 > 10"); } TEST_F(PlanOptimizeTest, orderByPrimaryKey) { useDb("root", "test"); - run("SELECT * FROM t1 ORDER BY ts"); - run("SELECT * FROM t1 ORDER BY ts DESC"); run("SELECT c1 FROM t1 ORDER BY ts"); + run("SELECT c1 FROM t1 ORDER BY ts DESC"); run("SELECT COUNT(*) FROM t1 INTERVAL(10S) ORDER BY _WSTARTTS DESC"); } +TEST_F(PlanOptimizeTest, PartitionTags) { + useDb("root", "test"); + + run("SELECT c1 FROM st1 PARTITION BY tag1"); + + run("SELECT SUM(c1) FROM st1 GROUP BY tag1"); +} + TEST_F(PlanOptimizeTest, eliminateProjection) { useDb("root", "test"); @@ -60,5 +69,5 @@ TEST_F(PlanOptimizeTest, eliminateProjection) { run("SELECT c1 FROM t1"); run("SELECT * FROM st1"); run("SELECT c1 FROM st1s3"); - //run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) order by 1 nulls first"); + // run("select 1-abs(c1) from (select unique(c1) c1 from st1s3) order by 1 nulls first"); } diff --git a/source/libs/planner/test/planOtherTest.cpp b/source/libs/planner/test/planOtherTest.cpp index 0f2540924ca5d9bd59ab259cce2ee5f8d5e80186..b8963c29f986823057f7f8e38919c56f3bfe2967 100644 --- a/source/libs/planner/test/planOtherTest.cpp +++ b/source/libs/planner/test/planOtherTest.cpp @@ -37,7 +37,9 @@ TEST_F(PlanOtherTest, createStream) { TEST_F(PlanOtherTest, createStreamUseSTable) { useDb("root", "test"); - run("create stream if not exists s1 as select count(*) from st1 interval(10s)"); + run("CREATE STREAM IF NOT EXISTS s1 as SELECT COUNT(*) FROM st1 INTERVAL(10s)"); + + run("CREATE STREAM IF NOT EXISTS s1 as SELECT COUNT(*) FROM st1 PARTITION BY TBNAME INTERVAL(10s)"); } TEST_F(PlanOtherTest, createSmaIndex) { @@ -74,6 +76,8 @@ TEST_F(PlanOtherTest, show) { run("SHOW TABLE DISTRIBUTED t1"); run("SHOW TABLE DISTRIBUTED st1"); + + run("SHOW DNODE 1 VARIABLES"); } TEST_F(PlanOtherTest, delete) { diff --git a/source/libs/planner/test/planSessionTest.cpp b/source/libs/planner/test/planSessionTest.cpp index 7d5d82692557207ad032e4a9172c52da86b75f44..f445bb5ffced5430aa7b6f0544c5d85c63bf8b49 100644 --- a/source/libs/planner/test/planSessionTest.cpp +++ b/source/libs/planner/test/planSessionTest.cpp @@ -34,3 +34,13 @@ TEST_F(PlanSessionTest, selectFunc) { // select function along with the columns of select row, and with SESSION clause run("SELECT MAX(c1), c2 FROM t1 SESSION(ts, 10s)"); } + +TEST_F(PlanSessionTest, stable) { + useDb("root", "test"); + + // select function for SESSION clause + run("SELECT MAX(c1), MIN(c1) FROM st1 SESSION(ts, 10s)"); + // select function along with the columns of select row, and with SESSION clause + run("SELECT MAX(c1), c2 FROM st1 SESSION(ts, 10s)"); + run("SELECT count(ts) FROM st1 PARTITION BY c1 SESSION(ts, 10s)"); +} diff --git a/source/libs/planner/test/planStateTest.cpp b/source/libs/planner/test/planStateTest.cpp index 9ff035e1480c5ccbf17c7ab889976c4a739d951f..6985bc88070e2340af28b51c716b1da3db54f0cb 100644 --- a/source/libs/planner/test/planStateTest.cpp +++ b/source/libs/planner/test/planStateTest.cpp @@ -40,3 +40,12 @@ TEST_F(PlanStateTest, selectFunc) { // select function along with the columns of select row, and with STATE_WINDOW clause run("SELECT MAX(c1), c2 FROM t1 STATE_WINDOW(c3)"); } + +TEST_F(PlanStateTest, stable) { + useDb("root", "test"); + + // select function for STATE_WINDOW clause + run("SELECT MAX(c1), MIN(c1) FROM st1 STATE_WINDOW(c2)"); + // select function along with the columns of select row, and with STATE_WINDOW clause + run("SELECT MAX(c1), c2 FROM st1 STATE_WINDOW(c2)"); +} diff --git a/source/libs/qcom/src/queryUtil.c b/source/libs/qcom/src/queryUtil.c index c92ab9b008cb3588dc8f054e940ec7ac2922021a..bee0461e58d30ca5933c20642f0c2a727ceab900 100644 --- a/source/libs/qcom/src/queryUtil.c +++ b/source/libs/qcom/src/queryUtil.c @@ -20,6 +20,8 @@ #include "trpc.h" #include "tsched.h" // clang-format off +#include "cJSON.h" + #define VALIDNUMOFCOLS(x) ((x) >= TSDB_MIN_COLUMNS && (x) <= TSDB_MAX_COLUMNS) #define VALIDNUMOFTAGS(x) ((x) >= 0 && (x) <= TSDB_MAX_TAGS) @@ -223,3 +225,207 @@ void destroyQueryExecRes(SQueryExecRes* pRes) { } } // clang-format on + +int32_t dataConverToStr(char* str, int type, void* buf, int32_t bufSize, int32_t* len) { + int32_t n = 0; + + switch (type) { + case TSDB_DATA_TYPE_NULL: + n = sprintf(str, "null"); + break; + + case TSDB_DATA_TYPE_BOOL: + n = sprintf(str, (*(int8_t*)buf) ? "true" : "false"); + break; + + case TSDB_DATA_TYPE_TINYINT: + n = sprintf(str, "%d", *(int8_t*)buf); + break; + + case TSDB_DATA_TYPE_SMALLINT: + n = sprintf(str, "%d", *(int16_t*)buf); + break; + + case TSDB_DATA_TYPE_INT: + n = sprintf(str, "%d", *(int32_t*)buf); + break; + + case TSDB_DATA_TYPE_BIGINT: + case TSDB_DATA_TYPE_TIMESTAMP: + n = sprintf(str, "%" PRId64, *(int64_t*)buf); + break; + + case TSDB_DATA_TYPE_FLOAT: + n = sprintf(str, "%e", GET_FLOAT_VAL(buf)); + break; + + case TSDB_DATA_TYPE_DOUBLE: + n = sprintf(str, "%e", GET_DOUBLE_VAL(buf)); + break; + + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + if (bufSize < 0) { + // tscError("invalid buf size"); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *str = '"'; + memcpy(str + 1, buf, bufSize); + *(str + bufSize + 1) = '"'; + n = bufSize + 2; + break; + + case TSDB_DATA_TYPE_UTINYINT: + n = sprintf(str, "%d", *(uint8_t*)buf); + break; + + case TSDB_DATA_TYPE_USMALLINT: + n = sprintf(str, "%d", *(uint16_t*)buf); + break; + + case TSDB_DATA_TYPE_UINT: + n = sprintf(str, "%u", *(uint32_t*)buf); + break; + + case TSDB_DATA_TYPE_UBIGINT: + n = sprintf(str, "%" PRIu64, *(uint64_t*)buf); + break; + + default: + // tscError("unsupported type:%d", type); + return TSDB_CODE_TSC_INVALID_VALUE; + } + + *len = n; + + return TSDB_CODE_SUCCESS; +} + +char* parseTagDatatoJson(void* p) { + char* string = NULL; + cJSON* json = cJSON_CreateObject(); + if (json == NULL) { + goto end; + } + + SArray* pTagVals = NULL; + if (tTagToValArray((const STag*)p, &pTagVals) != 0) { + goto end; + } + + int16_t nCols = taosArrayGetSize(pTagVals); + char tagJsonKey[256] = {0}; + for (int j = 0; j < nCols; ++j) { + STagVal* pTagVal = (STagVal*)taosArrayGet(pTagVals, j); + // json key encode by binary + memset(tagJsonKey, 0, sizeof(tagJsonKey)); + memcpy(tagJsonKey, pTagVal->pKey, strlen(pTagVal->pKey)); + // json value + char type = pTagVal->type; + if (type == TSDB_DATA_TYPE_NULL) { + cJSON* value = cJSON_CreateNull(); + if (value == NULL) { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + } else if (type == TSDB_DATA_TYPE_NCHAR) { + cJSON* value = NULL; + if (pTagVal->nData > 0) { + char* tagJsonValue = taosMemoryCalloc(pTagVal->nData, 1); + int32_t length = taosUcs4ToMbs((TdUcs4*)pTagVal->pData, pTagVal->nData, tagJsonValue); + if (length < 0) { + qError("charset:%s to %s. val:%s convert json value failed.", DEFAULT_UNICODE_ENCODEC, tsCharset, + pTagVal->pData); + taosMemoryFree(tagJsonValue); + goto end; + } + value = cJSON_CreateString(tagJsonValue); + taosMemoryFree(tagJsonValue); + if (value == NULL) { + goto end; + } + } else if (pTagVal->nData == 0) { + value = cJSON_CreateString(""); + } else { + ASSERT(0); + } + + cJSON_AddItemToObject(json, tagJsonKey, value); + } else if (type == TSDB_DATA_TYPE_DOUBLE) { + double jsonVd = *(double*)(&pTagVal->i64); + cJSON* value = cJSON_CreateNumber(jsonVd); + if (value == NULL) { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + } else if (type == TSDB_DATA_TYPE_BOOL) { + char jsonVd = *(char*)(&pTagVal->i64); + cJSON* value = cJSON_CreateBool(jsonVd); + if (value == NULL) { + goto end; + } + cJSON_AddItemToObject(json, tagJsonKey, value); + } else { + ASSERT(0); + } + } + string = cJSON_PrintUnformatted(json); +end: + cJSON_Delete(json); + return string; +} + +int32_t cloneTableMeta(STableMeta* pSrc, STableMeta** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + int32_t metaSize = (pSrc->tableInfo.numOfColumns + pSrc->tableInfo.numOfTags) * sizeof(SSchema); + *pDst = taosMemoryMalloc(metaSize); + if (NULL == *pDst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(*pDst, pSrc, metaSize); + return TSDB_CODE_SUCCESS; +} + +int32_t cloneDbVgInfo(SDBVgInfo* pSrc, SDBVgInfo** pDst) { + if (NULL == pSrc) { + *pDst = NULL; + return TSDB_CODE_SUCCESS; + } + + *pDst = taosMemoryMalloc(sizeof(*pSrc)); + if (NULL == *pDst) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + memcpy(*pDst, pSrc, sizeof(*pSrc)); + if (pSrc->vgHash) { + (*pDst)->vgHash = taosHashInit(taosHashGetSize(pSrc->vgHash), taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, + HASH_ENTRY_LOCK); + if (NULL == (*pDst)->vgHash) { + return TSDB_CODE_TSC_OUT_OF_MEMORY; + } + + SVgroupInfo* vgInfo = NULL; + void* pIter = taosHashIterate(pSrc->vgHash, NULL); + while (pIter) { + vgInfo = pIter; + int32_t* vgId = taosHashGetKey(pIter, NULL); + + if (0 != taosHashPut((*pDst)->vgHash, vgId, sizeof(*vgId), vgInfo, sizeof(*vgInfo))) { + qError("taosHashPut failed, vgId:%d", vgInfo->vgId); + taosHashCancelIterate(pSrc->vgHash, pIter); + taosHashCleanup((*pDst)->vgHash); + taosMemoryFreeClear(*pDst); + return TSDB_CODE_CTG_MEM_ERROR; + } + + pIter = taosHashIterate(pSrc->vgHash, pIter); + } + } + + return TSDB_CODE_SUCCESS; +} diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index b89c220519c60c3c3b67cc10f001ff3447448bbd..df608412b03164b590b62e37aca3cf9f9f7d7e83 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -63,7 +63,7 @@ int32_t queryBuildUseDbOutput(SUseDbOutput *pOut, SUseDbRsp *usedbRsp) { } int32_t queryBuildTableMetaReqMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int32_t)) { - SBuildTableMetaInput *pInput = input; + SBuildTableInput *pInput = input; if (NULL == input || NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; } @@ -126,6 +126,25 @@ int32_t queryBuildQnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t return TSDB_CODE_SUCCESS; } +int32_t queryBuildDnodeListMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int32_t)) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SDnodeListReq dnodeListReq = {0}; + dnodeListReq.rowNum = -1; + + int32_t bufLen = tSerializeSDnodeListReq(NULL, 0, &dnodeListReq); + void *pBuf = (*mallcFp)(bufLen); + tSerializeSDnodeListReq(pBuf, bufLen, &dnodeListReq); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + + int32_t queryBuildGetDBCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int32_t)) { if (NULL == msg || NULL == msgLen) { return TSDB_CODE_TSC_INVALID_INPUT; @@ -221,6 +240,27 @@ int32_t queryBuildGetTbIndexMsg(void *input, char **msg, int32_t msgSize, int32_ return TSDB_CODE_SUCCESS; } +int32_t queryBuildGetTbCfgMsg(void *input, char **msg, int32_t msgSize, int32_t *msgLen, void*(*mallcFp)(int32_t)) { + if (NULL == msg || NULL == msgLen) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + SBuildTableInput *pInput = input; + STableCfgReq cfgReq = {0}; + cfgReq.header.vgId = pInput->vgId; + strcpy(cfgReq.dbFName, pInput->dbFName); + strcpy(cfgReq.tbName, pInput->tbName); + + int32_t bufLen = tSerializeSTableCfgReq(NULL, 0, &cfgReq); + void *pBuf = (*mallcFp)(bufLen); + tSerializeSTableCfgReq(pBuf, bufLen, &cfgReq); + + *msg = pBuf; + *msgLen = bufLen; + + return TSDB_CODE_SUCCESS; +} + int32_t queryProcessUseDBRsp(void *output, char *msg, int32_t msgSize) { SUseDbOutput *pOut = output; @@ -407,6 +447,27 @@ int32_t queryProcessQnodeListRsp(void *output, char *msg, int32_t msgSize) { return code; } +int32_t queryProcessDnodeListRsp(void *output, char *msg, int32_t msgSize) { + SDnodeListRsp out = {0}; + int32_t code = 0; + + if (NULL == output || NULL == msg || msgSize <= 0) { + code = TSDB_CODE_TSC_INVALID_INPUT; + return code; + } + + if (tDeserializeSDnodeListRsp(msg, msgSize, &out) != 0) { + qError("invalid dnode list rsp msg, msgSize:%d", msgSize); + code = TSDB_CODE_INVALID_MSG; + return code; + } + + *(SArray**)output = out.dnodeList; + + return code; +} + + int32_t queryProcessGetDbCfgRsp(void *output, char *msg, int32_t msgSize) { SDbCfgRsp out = {0}; @@ -493,27 +554,48 @@ int32_t queryProcessGetTbIndexRsp(void *output, char *msg, int32_t msgSize) { return TSDB_CODE_SUCCESS; } +int32_t queryProcessGetTbCfgRsp(void *output, char *msg, int32_t msgSize) { + if (NULL == output || NULL == msg || msgSize <= 0) { + return TSDB_CODE_TSC_INVALID_INPUT; + } + + STableCfgRsp *out = taosMemoryCalloc(1, sizeof(STableCfgRsp)); + if (tDeserializeSTableCfgRsp(msg, msgSize, out) != 0) { + qError("tDeserializeSTableCfgRsp failed, msgSize:%d", msgSize); + return TSDB_CODE_INVALID_MSG; + } + + *(STableCfgRsp**)output = out; + + return TSDB_CODE_SUCCESS; +} void initQueryModuleMsgHandle() { queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryBuildTableMetaReqMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_USE_DB)] = queryBuildUseDbMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryBuildQnodeListMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_DNODE_LIST)] = queryBuildDnodeListMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryBuildGetDBCfgMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryBuildGetIndexMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryBuildRetrieveFuncMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryBuildGetUserAuthMsg; queryBuildMsg[TMSG_INDEX(TDMT_MND_GET_TABLE_INDEX)] = queryBuildGetTbIndexMsg; + queryBuildMsg[TMSG_INDEX(TDMT_VND_TABLE_CFG)] = queryBuildGetTbCfgMsg; + queryBuildMsg[TMSG_INDEX(TDMT_MND_TABLE_CFG)] = queryBuildGetTbCfgMsg; queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_META)] = queryProcessTableMetaRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_USE_DB)] = queryProcessUseDBRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_QNODE_LIST)] = queryProcessQnodeListRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_DNODE_LIST)] = queryProcessDnodeListRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_DB_CFG)] = queryProcessGetDbCfgRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_INDEX)] = queryProcessGetIndexRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_RETRIEVE_FUNC)] = queryProcessRetrieveFuncRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_USER_AUTH)] = queryProcessGetUserAuthRsp; queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_GET_TABLE_INDEX)] = queryProcessGetTbIndexRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_VND_TABLE_CFG)] = queryProcessGetTbCfgRsp; + queryProcessMsgRsp[TMSG_INDEX(TDMT_MND_TABLE_CFG)] = queryProcessGetTbCfgRsp; } #pragma GCC diagnostic pop diff --git a/source/libs/qworker/src/qworker.c b/source/libs/qworker/src/qworker.c index 28181d7e11291a984ae53b0bff2bc46282977548..4250785f7cca6a041776d55a992276fc316d74b6 100644 --- a/source/libs/qworker/src/qworker.c +++ b/source/libs/qworker/src/qworker.c @@ -271,7 +271,7 @@ int32_t qwGetDeleteResFromSink(QW_FPARAMS_DEF, SQWTaskCtx *ctx, int32_t *dataLen SDeleterRes* pDelRes = (SDeleterRes*)output.pData; rsp.affectedRows = pDelRes->affectedRows; - pRes->uid = pDelRes->uid; + pRes->suid = pDelRes->suid; pRes->uidList = pDelRes->uidList; pRes->skey = pDelRes->skey; pRes->ekey = pDelRes->ekey; diff --git a/source/libs/scalar/inc/sclInt.h b/source/libs/scalar/inc/sclInt.h index 4dd16098ff3df97185c4a6f249a12882456d111d..c2d984c7ddb20cc12dae22ec836d7ccb53b9eab3 100644 --- a/source/libs/scalar/inc/sclInt.h +++ b/source/libs/scalar/inc/sclInt.h @@ -57,7 +57,7 @@ typedef struct SScalarCtx { #define SCL_ERR_JRET(c) do { code = c; if (code != TSDB_CODE_SUCCESS) { terrno = code; goto _return; } } while (0) int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out); -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows); +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows); int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode); #define GET_PARAM_TYPE(_c) ((_c)->columnData ? (_c)->columnData->info.type : (_c)->hashValueType) diff --git a/source/libs/scalar/src/filter.c b/source/libs/scalar/src/filter.c index a446aa7bdb5a35ad1cfaaa1e356ead7148804ef1..b8bcae161b84df58ee8986dbdd35e900860d524a 100644 --- a/source/libs/scalar/src/filter.c +++ b/source/libs/scalar/src/filter.c @@ -3821,7 +3821,7 @@ bool filterExecute(SFilterInfo *info, SSDataBlock *pSrc, int8_t** p, SColumnData SScalarParam output = {0}; SDataType type = {.type = TSDB_DATA_TYPE_BOOL, .bytes = sizeof(bool)}; - output.columnData = createColumnInfoData(&type, pSrc->info.rows); + output.columnData = sclCreateColumnInfoData(&type, pSrc->info.rows); SArray *pList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(pList, &pSrc); diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index 66e4af5ded0404b9f5b2ec7e9b8097178613c2c6..75a9bc3809632f88601f8628add9aed4b8365cf4 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -36,7 +36,7 @@ int32_t sclConvertToTsValueNode(int8_t precision, SValueNode* valueNode) { } -SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { +SColumnInfoData* sclCreateColumnInfoData(SDataType* pType, int32_t numOfRows) { SColumnInfoData* pColumnData = taosMemoryCalloc(1, sizeof(SColumnInfoData)); if (pColumnData == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -48,7 +48,7 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { pColumnData->info.scale = pType->scale; pColumnData->info.precision = pType->precision; - int32_t code = colInfoDataEnsureCapacity(pColumnData, 0, numOfRows); + int32_t code = colInfoDataEnsureCapacity(pColumnData, numOfRows); if (code != TSDB_CODE_SUCCESS) { terrno = TSDB_CODE_OUT_OF_MEMORY; taosMemoryFree(pColumnData); @@ -60,10 +60,10 @@ SColumnInfoData* createColumnInfoData(SDataType* pType, int32_t numOfRows) { int32_t doConvertDataType(SValueNode* pValueNode, SScalarParam* out) { SScalarParam in = {.numOfRows = 1}; - in.columnData = createColumnInfoData(&pValueNode->node.resType, 1); + in.columnData = sclCreateColumnInfoData(&pValueNode->node.resType, 1); colDataAppend(in.columnData, 0, nodesGetValueFromNode(pValueNode), false); - colInfoDataEnsureCapacity(out->columnData, 0, 1); + colInfoDataEnsureCapacity(out->columnData, 1); int32_t code = vectorConvertImpl(&in, out); sclFreeParam(&in); @@ -193,7 +193,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t SValueNode *valueNode = (SValueNode *)node; param->numOfRows = 1; - param->columnData = createColumnInfoData(&valueNode->node.resType, 1); + param->columnData = sclCreateColumnInfoData(&valueNode->node.resType, 1); if (TSDB_DATA_TYPE_NULL == valueNode->node.resType.type) { colDataAppendNULL(param->columnData, 0); } else { @@ -245,7 +245,7 @@ int32_t sclInitParam(SNode* node, SScalarParam *param, SScalarCtx *ctx, int32_t } SSDataBlock *block = *(SSDataBlock **)taosArrayGet(ctx->pBlockList, index); - if (NULL == block || ref->slotId >= block->info.numOfCols) { + if (NULL == block || ref->slotId >= taosArrayGetSize(block->pDataBlock)) { sclError("column slotId is too big, slodId:%d, dataBlockSize:%d", ref->slotId, (int32_t)taosArrayGetSize(block->pDataBlock)); SCL_ERR_RET(TSDB_CODE_QRY_INVALID_INPUT); } @@ -431,7 +431,7 @@ int32_t sclExecFunction(SFunctionNode *node, SScalarCtx *ctx, SScalarParam *outp SCL_ERR_JRET(code); } - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * output->columnData->info.bytes)); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -484,7 +484,7 @@ int32_t sclExecLogic(SLogicConditionNode *node, SScalarCtx *ctx, SScalarParam *o output->numOfRows = rowNum; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - output->columnData = createColumnInfoData(&t, rowNum); + output->columnData = sclCreateColumnInfoData(&t, rowNum); if (output->columnData == NULL) { sclError("calloc %d failed", (int32_t)(rowNum * sizeof(bool))); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -539,7 +539,7 @@ int32_t sclExecOperator(SOperatorNode *node, SScalarCtx *ctx, SScalarParam *outp int32_t code = 0; SCL_ERR_RET(sclInitOperatorParams(¶ms, node, ctx, &rowNum)); - output->columnData = createColumnInfoData(&node->node.resType, rowNum); + output->columnData = sclCreateColumnInfoData(&node->node.resType, rowNum); if (output->columnData == NULL) { sclError("calloc failed, size:%d", (int32_t)rowNum * node->node.resType.bytes); SCL_ERR_JRET(TSDB_CODE_QRY_OUT_OF_MEMORY); @@ -901,7 +901,7 @@ EDealRes sclWalkTarget(SNode* pNode, SScalarCtx *ctx) { return DEAL_RES_ERROR; } - colDataAssign(col, res->columnData, res->numOfRows); + colDataAssign(col, res->columnData, res->numOfRows, NULL); block->info.rows = res->numOfRows; sclFreeParam(res); @@ -982,8 +982,9 @@ int32_t scalarCalculate(SNode *pNode, SArray *pBlockList, SScalarParam *pDst) { sclError("no valid res in hash, node:%p, type:%d", pNode, nodeType(pNode)); SCL_ERR_JRET(TSDB_CODE_QRY_APP_ERROR); } - - colDataAssign(pDst->columnData, res->columnData, res->numOfRows); + + colInfoDataEnsureCapacity(pDst->columnData, res->numOfRows); + colDataAssign(pDst->columnData, res->columnData, res->numOfRows, NULL); pDst->numOfRows = res->numOfRows; taosHashRemove(ctx.pRes, (void *)&pNode, POINTER_BYTES); } diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 2369779105f06685be2120907531ee0a32869c84..7ea0457592824b7f26fe823f8f016fd98a90d4f7 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -120,7 +120,7 @@ int32_t absFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutpu } default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } @@ -273,7 +273,7 @@ static int32_t doScalarFunction(SScalarParam *pInput, int32_t inputNum, SScalarP } default: { - colDataAssign(pOutputData, pInputData, pInput->numOfRows); + colDataAssign(pOutputData, pInputData, pInput->numOfRows, NULL); } } @@ -856,10 +856,13 @@ int32_t castFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutp int32_t toISO8601Function(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); + bool tzPresent = (inputNum == 2) ? true : false; char* tz; int32_t tzLen; - tz = varDataVal(pInput[1].columnData->pData); - tzLen = varDataLen(pInput[1].columnData->pData); + if (tzPresent) { + tz = varDataVal(pInput[1].columnData->pData); + tzLen = varDataLen(pInput[1].columnData->pData); + } for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { @@ -1415,11 +1418,19 @@ int32_t roundFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOut } int32_t lowerFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +#ifdef WINDOWS + return doCaseConvFunction(pInput, inputNum, pOutput, towlower); +#else return doCaseConvFunction(pInput, inputNum, pOutput, tolower); +#endif } int32_t upperFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { +#ifdef WINDOWS + return doCaseConvFunction(pInput, inputNum, pOutput, towupper); +#else return doCaseConvFunction(pInput, inputNum, pOutput, toupper); +#endif } int32_t ltrimFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { diff --git a/source/libs/scalar/src/sclvector.c b/source/libs/scalar/src/sclvector.c index bff49c2ca91d3b03ee0d832bfc7b996959576f18..f9a9ec0f7223e138147c13102e7bb56b1a497fc1 100644 --- a/source/libs/scalar/src/sclvector.c +++ b/source/libs/scalar/src/sclvector.c @@ -23,6 +23,7 @@ #include "sclvector.h" #include "tcompare.h" #include "tdatablock.h" +#include "tdataformat.h" #include "ttypes.h" #include "ttime.h" @@ -506,6 +507,16 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t } } + // if types can not comparable + if((IS_NUMERIC_TYPE(typeLeft) && !IS_NUMERIC_TYPE(typeRight)) || + (IS_NUMERIC_TYPE(typeRight) && !IS_NUMERIC_TYPE(typeLeft)) || + (IS_VAR_DATA_TYPE(typeLeft) && !IS_VAR_DATA_TYPE(typeRight)) || + (IS_VAR_DATA_TYPE(typeRight) && !IS_VAR_DATA_TYPE(typeLeft)) || + ((typeLeft == TSDB_DATA_TYPE_BOOL) && (typeRight != TSDB_DATA_TYPE_BOOL)) || + ((typeRight == TSDB_DATA_TYPE_BOOL) && (typeLeft != TSDB_DATA_TYPE_BOOL))) + return false; + + if(typeLeft == TSDB_DATA_TYPE_NULL || typeRight == TSDB_DATA_TYPE_NULL){ *isNull = true; return true; @@ -519,24 +530,28 @@ bool convertJsonValue(__compar_fn_t *fp, int32_t optr, int8_t typeLeft, int8_t t *fp = filterGetCompFunc(type, optr); - if(IS_NUMERIC_TYPE(type) || IS_FLOAT_TYPE(type)){ + if(IS_NUMERIC_TYPE(type)){ if(typeLeft == TSDB_DATA_TYPE_NCHAR) { - convertNcharToDouble(*pLeftData, pLeftOut); - *pLeftData = pLeftOut; + ASSERT(0); +// convertNcharToDouble(*pLeftData, pLeftOut); +// *pLeftData = pLeftOut; } else if(typeLeft == TSDB_DATA_TYPE_BINARY) { - convertBinaryToDouble(*pLeftData, pLeftOut); - *pLeftData = pLeftOut; + ASSERT(0); +// convertBinaryToDouble(*pLeftData, pLeftOut); +// *pLeftData = pLeftOut; } else if(typeLeft != type) { convertNumberToNumber(*pLeftData, pLeftOut, typeLeft, type); *pLeftData = pLeftOut; } if(typeRight == TSDB_DATA_TYPE_NCHAR) { - convertNcharToDouble(*pRightData, pRightOut); - *pRightData = pRightOut; + ASSERT(0); +// convertNcharToDouble(*pRightData, pRightOut); +// *pRightData = pRightOut; } else if(typeRight == TSDB_DATA_TYPE_BINARY) { - convertBinaryToDouble(*pRightData, pRightOut); - *pRightData = pRightOut; + ASSERT(0); +// convertBinaryToDouble(*pRightData, pRightOut); +// *pRightData = pRightOut; } else if(typeRight != type) { convertNumberToNumber(*pRightData, pRightOut, typeRight, type); *pRightData = pRightOut; @@ -837,7 +852,7 @@ int32_t vectorConvertScalarParam(SScalarParam *input, SScalarParam *output, int3 SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; output->numOfRows = input->numOfRows; - output->columnData = createColumnInfoData(&t, input->numOfRows); + output->columnData = sclCreateColumnInfoData(&t, input->numOfRows); if (output->columnData == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -908,7 +923,7 @@ static int32_t doConvertHelper(SScalarParam* pDest, int32_t* convert, const SSca pDest->numOfRows = pParam->numOfRows; SDataType t = {.type = type, .bytes = tDataTypes[type].bytes}; - pDest->columnData = createColumnInfoData(&t, pParam->numOfRows); + pDest->columnData = sclCreateColumnInfoData(&t, pParam->numOfRows); if (pDest->columnData == NULL) { sclError("malloc %d failed", (int32_t)(pParam->numOfRows * sizeof(double))); return TSDB_CODE_OUT_OF_MEMORY; @@ -1693,6 +1708,13 @@ void vectorIsTrue(SScalarParam* pLeft, SScalarParam* pRight, SScalarParam *pOut, STagVal getJsonValue(char *json, char *key, bool *isExist) { STagVal val = {.pKey = key}; + if (tTagIsJson((const STag *)json) == false){ + if(isExist){ + *isExist = false; + } + return val; + } + bool find = tTagGet(((const STag *)json), &val); // json value is null and not exist is different if(isExist){ *isExist = find; diff --git a/source/libs/scalar/test/filter/filterTests.cpp b/source/libs/scalar/test/filter/filterTests.cpp index 7fb1ffbd64aecb2fee9d7c862f295070dbea8e09..8f9332cef802e30521de8fc7515e8cc0b73d2125 100644 --- a/source/libs/scalar/test/filter/filterTests.cpp +++ b/source/libs/scalar/test/filter/filterTests.cpp @@ -105,29 +105,14 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_NULL, 10, 1+i); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + blockDataAppendColInfo(res, &idata); blockDataEnsureCapacity(res, rowNum); SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); @@ -142,23 +127,18 @@ void flttMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in rnode->slotId = 2; rnode->colId = 3; + res->info.rows = rowNum; *block = res; } else { SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1+idx); + blockDataAppendColInfo(res, &idata); + blockDataEnsureCapacity(res, rowNum); + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); - - colInfoDataEnsureCapacity(pColumn, 0, rowNum); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -399,7 +379,7 @@ TEST(columnTest, smallint_column_greater_double_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, true); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -407,9 +387,9 @@ TEST(columnTest, smallint_column_greater_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -430,7 +410,7 @@ TEST(columnTest, int_column_greater_smallint_value) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_SMALLINT, &rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); @@ -454,7 +434,7 @@ TEST(columnTest, int_column_greater_smallint_value) { keep = filterRangeExecute(filter, &stat, 1, rowNum); ASSERT_EQ(keep, false); - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -462,9 +442,9 @@ TEST(columnTest, int_column_greater_smallint_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -478,11 +458,11 @@ TEST(columnTest, int_column_in_double_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; int32_t leftv[5] = {1, 2, 3, 4, 5}; double rightv1 = 1.1,rightv2 = 2.2,rightv3 = 3.3; - bool eRes[5] = {true, true, true, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, true, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv1); nodesListAppend(list, pRight); @@ -498,7 +478,7 @@ TEST(columnTest, int_column_in_double_list) { ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -506,9 +486,9 @@ TEST(columnTest, int_column_in_double_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -521,8 +501,8 @@ TEST(columnTest, int_column_in_double_list) { TEST(columnTest, binary_column_in_binary_list) { SNode *pLeft = NULL, *pRight = NULL, *listNode = NULL, *opNode = NULL; - bool eRes[5] = {true, true, false, false, false}; - SSDataBlock *src = NULL; + bool eRes[5] = {true, true, false, false, false}; + SSDataBlock *src = NULL; SScalarParam res = {0}; char leftv[5][5]= {0}; char rightv[3][5]= {0}; @@ -531,22 +511,22 @@ TEST(columnTest, binary_column_in_binary_list) { leftv[i][3] = 'b' + i; leftv[i][4] = '0' + i; varDataSetLen(leftv[i], 3); - } + } for (int32_t i = 0; i < 2; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'b' + i; rightv[i][4] = '0' + i; varDataSetLen(rightv[i], 3); - } + } for (int32_t i = 2; i < 3; ++i) { rightv[i][2] = 'a' + i; rightv[i][3] = 'a' + i; rightv[i][4] = 'a' + i; varDataSetLen(rightv[i], 3); } - + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SNodeList* list = nodesMakeList(); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv[0]); nodesListAppend(list, pRight); @@ -556,13 +536,13 @@ TEST(columnTest, binary_column_in_binary_list) { nodesListAppend(list, pRight); flttMakeListNode(&listNode,list, TSDB_DATA_TYPE_BINARY); flttMakeOpNode(&opNode, OP_TYPE_IN, TSDB_DATA_TYPE_BOOL, pLeft, listNode); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -570,9 +550,9 @@ TEST(columnTest, binary_column_in_binary_list) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -586,31 +566,31 @@ TEST(columnTest, binary_column_like_binary) { SNode *pLeft = NULL, *pRight = NULL, *opNode = NULL; char rightv[64] = {0}; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, false, true, false, true}; - + bool eRes[5] = {true, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = 'a'; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); sprintf(&rightv[2], "%s", "__0"); varDataSetLen(rightv, strlen(&rightv[2])); flttMakeValueNode(&pRight, TSDB_DATA_TYPE_BINARY, rightv); flttMakeOpNode(&opNode, OP_TYPE_LIKE, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -618,9 +598,9 @@ TEST(columnTest, binary_column_like_binary) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -633,31 +613,31 @@ TEST(columnTest, binary_column_like_binary) { TEST(columnTest, binary_column_is_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, false, true, false, true}; - + bool eRes[5] = {false, false, true, false, true}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 2, NULL, true); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -665,9 +645,9 @@ TEST(columnTest, binary_column_is_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -680,31 +660,31 @@ TEST(columnTest, binary_column_is_null) { TEST(columnTest, binary_column_is_not_null) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {true, true, true, true, false}; - + bool eRes[5] = {true, true, true, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); SColumnInfoData *pcolumn = (SColumnInfoData *)taosArrayGetLast(src->pDataBlock); colDataAppend(pcolumn, 4, NULL, true); flttMakeOpNode(&opNode, OP_TYPE_IS_NOT_NULL, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -712,9 +692,9 @@ TEST(columnTest, binary_column_is_not_null) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -735,13 +715,13 @@ TEST(opTest, smallint_column_greater_int_column) { flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, leftv); flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft, pRight); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -749,9 +729,9 @@ TEST(opTest, smallint_column_greater_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -773,13 +753,13 @@ TEST(opTest, smallint_value_add_int_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_SMALLINT, sizeof(int16_t), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_ADD, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -787,9 +767,9 @@ TEST(opTest, smallint_value_add_int_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -816,13 +796,13 @@ TEST(opTest, bigint_column_multi_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_MULTI, TSDB_DATA_TYPE_DOUBLE, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -830,9 +810,9 @@ TEST(opTest, bigint_column_multi_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -859,13 +839,13 @@ TEST(opTest, smallint_column_and_binary_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_BINARY, 5, rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_AND, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -873,9 +853,9 @@ TEST(opTest, smallint_column_and_binary_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, (int32_t) taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -897,13 +877,13 @@ TEST(opTest, smallint_column_or_float_column) { flttMakeColumnNode(&pRight, &src, TSDB_DATA_TYPE_FLOAT, sizeof(float), rowNum, rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -911,9 +891,9 @@ TEST(opTest, smallint_column_or_float_column) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -935,13 +915,13 @@ TEST(opTest, smallint_column_or_double_value) { flttMakeValueNode(&pRight, TSDB_DATA_TYPE_DOUBLE, &rightv); flttMakeOpNode(&opNode, OP_TYPE_BIT_OR, TSDB_DATA_TYPE_BIGINT, pLeft, pRight); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, opNode, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -949,9 +929,9 @@ TEST(opTest, smallint_column_or_double_value) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, true); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -964,28 +944,28 @@ TEST(opTest, smallint_column_or_double_value) { TEST(opTest, binary_column_is_true) { SNode *pLeft = NULL, *opNode = NULL; char leftv[5][5]= {0}; - SSDataBlock *src = NULL; + SSDataBlock *src = NULL; SScalarParam res = {0}; - bool eRes[5] = {false, true, false, true, false}; - + bool eRes[5] = {false, true, false, true, false}; + for (int32_t i = 0; i < 5; ++i) { leftv[i][2] = '0' + i % 2; leftv[i][3] = 'a'; leftv[i][4] = '0' + i % 2; varDataSetLen(leftv[i], 3); - } - + } + int32_t rowNum = sizeof(leftv)/sizeof(leftv[0]); - flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); + flttMakeColumnNode(&pLeft, &src, TSDB_DATA_TYPE_BINARY, 3, rowNum, leftv); flttMakeOpNode(&opNode, OP_TYPE_IS_TRUE, TSDB_DATA_TYPE_BOOL, pLeft, NULL); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(opNode, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -993,9 +973,9 @@ TEST(opTest, binary_column_is_true) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1027,12 +1007,12 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1042,7 +1022,7 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1050,13 +1030,13 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1064,9 +1044,9 @@ TEST(filterModelogicTest, diff_columns_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1096,13 +1076,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); - flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); nodesListAppend(list, opNode1); @@ -1111,7 +1091,7 @@ TEST(filterModelogicTest, same_column_and_or_and) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_AND, list); list = nodesMakeList(); @@ -1119,13 +1099,13 @@ TEST(filterModelogicTest, same_column_and_or_and) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1133,9 +1113,9 @@ TEST(filterModelogicTest, same_column_and_or_and) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1165,12 +1145,12 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv1); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1180,7 +1160,7 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1188,13 +1168,13 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t) taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1202,9 +1182,9 @@ TEST(filterModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1234,12 +1214,12 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeValueNode(&pRight1, TSDB_DATA_TYPE_INT, &rightv3); flttMakeOpNode(&opNode1, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1249,7 +1229,7 @@ TEST(filterModelogicTest, same_column_or_and_or) { flttMakeValueNode(&pRight2, TSDB_DATA_TYPE_INT, &rightv4); flttMakeOpNode(&opNode2, OP_TYPE_GREATER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft1, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1257,13 +1237,13 @@ TEST(filterModelogicTest, same_column_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1271,9 +1251,9 @@ TEST(filterModelogicTest, same_column_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); - + for (int32_t i = 0; i < rowNum; ++i) { ASSERT_EQ(*((int8_t *)rowRes + i), eRes[i]); } @@ -1305,12 +1285,12 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_THAN, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); - + flttMakeColumnNode(&pLeft1, &src, TSDB_DATA_TYPE_DOUBLE, sizeof(double), rowNum, leftv1); flttMakeColumnNode(&pRight1, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv1); flttMakeOpNode(&opNode1, OP_TYPE_GREATER_THAN, TSDB_DATA_TYPE_BOOL, pLeft1, pRight1); @@ -1320,7 +1300,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { flttMakeColumnNode(&pRight2, &src, TSDB_DATA_TYPE_INT, sizeof(int32_t), rowNum, rightv2); flttMakeOpNode(&opNode2, OP_TYPE_LOWER_EQUAL, TSDB_DATA_TYPE_BOOL, pLeft2, pRight2); nodesListAppend(list, opNode2); - + flttMakeLogicNodeFromList(&logicNode2, LOGIC_COND_TYPE_OR, list); list = nodesMakeList(); @@ -1328,13 +1308,13 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { nodesListAppend(list, logicNode2); flttMakeLogicNodeFromList(&logicNode1, LOGIC_COND_TYPE_AND, list); - + SFilterInfo *filter = NULL; int32_t code = filterInitFromNode(logicNode1, &filter, 0); ASSERT_EQ(code, 0); SColumnDataAgg stat = {0}; - SFilterColumnParam param = { src->info.numOfCols, src->pDataBlock }; + SFilterColumnParam param = { (int32_t)taosArrayGetSize(src->pDataBlock), src->pDataBlock }; code = filterSetDataFromSlotId(filter, ¶m); ASSERT_EQ(code, 0); @@ -1342,7 +1322,7 @@ TEST(scalarModelogicTest, diff_columns_or_and_or) { stat.min = 1; stat.numOfNull = 0; int8_t *rowRes = NULL; - bool keep = filterExecute(filter, src, &rowRes, &stat, src->info.numOfCols); + bool keep = filterExecute(filter, src, &rowRes, &stat, taosArrayGetSize(src->pDataBlock)); ASSERT_EQ(keep, false); for (int32_t i = 0; i < rowNum; ++i) { diff --git a/source/libs/scalar/test/scalar/scalarTests.cpp b/source/libs/scalar/test/scalar/scalarTests.cpp index d8791502008755b6ac85949ca07be8b5225a774c..07440e743553aa9756a389b1e32a92e8d156d537 100644 --- a/source/libs/scalar/test/scalar/scalarTests.cpp +++ b/source/libs/scalar/test/scalar/scalarTests.cpp @@ -84,30 +84,29 @@ void scltInitLogFile() { void scltAppendReservedSlot(SArray *pBlockList, int16_t *dataBlockId, int16_t *slotId, bool newBlock, int32_t rows, SColumnInfo *colInfo) { if (newBlock) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 1; - res->info.rows = rows; - res->pDataBlock = taosArrayInit(1, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); + SColumnInfoData idata = {0}; - idata.info = *colInfo; + idata.info = *colInfo; + colInfoDataEnsureCapacity(&idata, rows); - taosArrayPush(res->pDataBlock, &idata); - taosArrayPush(pBlockList, &res); + blockDataAppendColInfo(res, &idata); - blockDataEnsureCapacity(res, rows); + res->info.capacity = rows; + res->info.rows = rows; + SColumnInfoData* p = static_cast(taosArrayGet(res->pDataBlock, 0)); + ASSERT(p->pData != NULL && p->nullbitmap != NULL); + taosArrayPush(pBlockList, &res); *dataBlockId = taosArrayGetSize(pBlockList) - 1; res->info.blockId = *dataBlockId; *slotId = 0; } else { SSDataBlock *res = *(SSDataBlock **)taosArrayGetLast(pBlockList); - res->info.numOfCols++; SColumnInfoData idata = {0}; idata.info = *colInfo; - - colInfoDataEnsureCapacity(&idata, 0, rows); - - taosArrayPush(res->pDataBlock, &idata); + colInfoDataEnsureCapacity(&idata, rows); + blockDataAppendColInfo(res, &idata); *dataBlockId = taosArrayGetSize(pBlockList) - 1; *slotId = taosArrayGetSize(res->pDataBlock) - 1; @@ -144,29 +143,19 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in } if (NULL == *block) { - SSDataBlock *res = (SSDataBlock *)taosMemoryCalloc(1, sizeof(SSDataBlock)); - res->info.numOfCols = 3; - res->info.rows = rowNum; - res->pDataBlock = taosArrayInit(3, sizeof(SColumnInfoData)); + SSDataBlock *res = createDataBlock(); for (int32_t i = 0; i < 2; ++i) { - SColumnInfoData idata = {{0}}; - idata.info.type = TSDB_DATA_TYPE_NULL; - idata.info.bytes = 10; - idata.info.colId = i + 1; - - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(TSDB_DATA_TYPE_INT, 10, i + 1); + colInfoDataEnsureCapacity(&idata, rowNum); + blockDataAppendColInfo(res, &idata); } - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 3; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - colInfoDataEnsureCapacity(&idata, 0, rowNum); - taosArrayPush(res->pDataBlock, &idata); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 3); + colInfoDataEnsureCapacity(&idata, rowNum); + blockDataAppendColInfo(res, &idata); + res->info.capacity = rowNum; + + res->info.rows = rowNum; SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -185,17 +174,13 @@ void scltMakeColumnNode(SNode **pNode, SSDataBlock **block, int32_t dataType, in SSDataBlock *res = *block; int32_t idx = taosArrayGetSize(res->pDataBlock); - SColumnInfoData idata = {{0}}; - idata.info.type = dataType; - idata.info.bytes = dataBytes; - idata.info.colId = 1 + idx; - int32_t size = idata.info.bytes * rowNum; - idata.pData = (char *)taosMemoryCalloc(1, size); - taosArrayPush(res->pDataBlock, &idata); - res->info.numOfCols++; - SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); + SColumnInfoData idata = createColumnInfoData(dataType, dataBytes, 1 + idx); + colInfoDataEnsureCapacity(&idata, rowNum); - colInfoDataEnsureCapacity(pColumn, 0, rowNum); + res->info.capacity = rowNum; + blockDataAppendColInfo(res, &idata); + + SColumnInfoData *pColumn = (SColumnInfoData *)taosArrayGetLast(res->pDataBlock); for (int32_t i = 0; i < rowNum; ++i) { colDataAppend(pColumn, i, (const char *)value, false); @@ -279,9 +264,6 @@ void scltMakeTargetNode(SNode **pNode, int16_t dataBlockId, int16_t slotId, SNod *pNode = (SNode *)onode; } - - - } TEST(constantTest, bigint_add_bigint) { @@ -1310,11 +1292,11 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json string--0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k2"; - bool eRes1[len+len1] = {false, false, true, true, false, false, false, true, false, true, false, true, true}; + bool eRes1[len+len1] = {false, false, false, false, false, false, false, true, false, true, false, true, true}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes1[i], op[i], false); } - bool eRes_1[len0] = {true, true, false, false, false, false}; + bool eRes_1[len0] = {false, false, false, false, false, false}; for(int i = 0; i < len0; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_1[i], op[i], true); } @@ -1346,11 +1328,11 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json bool--1 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k4"; - bool eRes3[len+len1] = {false, false, true, true, false, true, false, true, true, false, false, false, false}; + bool eRes3[len+len1] = {false, false, false, false, false, false, false, true, true, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes3[i], op[i], false); } - bool eRes_3[len0] = {false, true, false, false, false, true}; + bool eRes_3[len0] = {false, false, false, false, false, false}; for(int i = 0; i < len0; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_3[i], op[i], true); } @@ -1419,11 +1401,11 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json bool-- 0 {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k8"; - bool eRes7[len+len1] = {false, false, true, true, false, false, false, true, false, false, false, false, false}; + bool eRes7[len+len1] = {false, false, false, false, false, false, false, true, false, false, false, false, false}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes7[i], op[i], false); } - bool eRes_7[len0] = {true, true, false, false, false, false}; + bool eRes_7[len0] = {false, false, false, false, false, false}; for(int i = 0; i < len0; i++) { makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_7[i], op[i], true); } @@ -1438,11 +1420,11 @@ TEST(columnTest, json_column_logic_op) { printf("--------------------json string-- 6.6hello {1, 8, 2, 2, 3, 0, 0, 0, 0}-------------------\n"); key = "k9"; - bool eRes8[len+len1] = {true, false, false, false, false, true, false, true, true, false, true, false, true}; + bool eRes8[len+len1] = {false, false, false, false, false, false, false, true, true, false, true, false, true}; for(int i = 0; i < len; i++){ makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes8[i], op[i], false); } - bool eRes_8[len0] = {false, true, true, true, false, true}; + bool eRes_8[len0] = {false, false, false, false, false, false}; for(int i = 0; i < len0; i++) { makeCalculate(row, key, TSDB_DATA_TYPE_INT, &input[i], eRes_8[i], op[i], true); } @@ -1529,6 +1511,7 @@ TEST(columnTest, bigint_column_multi_binary_column) { SArray *blockList = taosArrayInit(1, POINTER_BYTES); taosArrayPush(blockList, &src); + SColumnInfo colInfo = createColumnInfo(1, TSDB_DATA_TYPE_DOUBLE, sizeof(double)); int16_t dataBlockId = 0, slotId = 0; scltAppendReservedSlot(blockList, &dataBlockId, &slotId, false, rowNum, &colInfo); @@ -2033,7 +2016,7 @@ void scltMakeDataBlock(SScalarParam **pInput, int32_t type, void *pVal, int32_t input->numOfRows = num; input->columnData->info = createColumnInfo(0, type, bytes); - colInfoDataEnsureCapacity(input->columnData, 0, num); + colInfoDataEnsureCapacity(input->columnData, num); if (setVal) { for (int32_t i = 0; i < num; ++i) { diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 8aaf4953def5d8fcb1fe354124c28f595649d8e1..9654e21d24a3768a69d8452715fd56fac73b2f2d 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -33,8 +33,13 @@ static SStreamGlobalEnv streamEnv; int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatch(SStreamTask* pTask, SMsgCb* pMsgCb); int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData); +int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData); int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcMsg* pMsg, SEpSet** ppEpSet); +int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock); + +int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq); + #ifdef __cplusplus } #endif diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index 38a1ad14b1b4abf124a3badf905f2b30a15a0f80..9d92865e4780fc5f0f9ea78ab9fe7356e0c0c661 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -50,6 +50,10 @@ void streamCleanUp() { void streamTriggerByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; + if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + return; + } + if (atomic_load_8(&pTask->triggerStatus) == TASK_TRIGGER_STATUS__ACTIVE) { SStreamTrigger* trigger = taosAllocateQitem(sizeof(SStreamTrigger), DEF_QITEM); if (trigger == NULL) return; @@ -82,8 +86,8 @@ int32_t streamSetupTrigger(SStreamTask* pTask) { } int32_t streamLaunchByWrite(SStreamTask* pTask, int32_t vgId) { - int8_t execStatus = atomic_load_8(&pTask->status); - if (execStatus == TASK_STATUS__IDLE || execStatus == TASK_STATUS__CLOSING) { + int8_t execStatus = atomic_load_8(&pTask->execStatus); + if (execStatus == TASK_EXEC_STATUS__IDLE || execStatus == TASK_EXEC_STATUS__CLOSING) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) return -1; @@ -108,7 +112,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* // enqueue if (pData != NULL) { pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK; - pData->sourceVg = pReq->sourceVg; + pData->srcVgId = pReq->dataSrcVgId; // decode /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ @@ -129,7 +133,42 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* SStreamDispatchRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead)); pCont->inputStatus = status; pCont->streamId = pReq->streamId; - pCont->taskId = pReq->sourceTaskId; + pCont->taskId = pReq->upstreamTaskId; + pRsp->pCont = buf; + pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); + tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; +} + +int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { + SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); + int8_t status = TASK_INPUT_STATUS__NORMAL; + + // enqueue + if (pData != NULL) { + pData->type = STREAM_DATA_TYPE_SSDATA_BLOCK; + pData->srcVgId = 0; + // decode + /*pData->blocks = pReq->data;*/ + /*pBlock->sourceVer = pReq->sourceVer;*/ + streamRetrieveReqToData(pReq, pData); + if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + status = TASK_INPUT_STATUS__NORMAL; + } else { + status = TASK_INPUT_STATUS__FAILED; + } + } else { + /*streamTaskInputFail(pTask);*/ + /*status = TASK_INPUT_STATUS__FAILED;*/ + } + + // rsp by input status + void* buf = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamRetrieveRsp)); + ((SMsgHead*)buf)->vgId = htonl(pReq->srcNodeId); + SStreamRetrieveRsp* pCont = POINTER_SHIFT(buf, sizeof(SMsgHead)); + pCont->streamId = pReq->streamId; + pCont->rspToTaskId = pReq->srcTaskId; + pCont->rspFromTaskId = pReq->dstTaskId; pRsp->pCont = buf; pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); @@ -137,7 +176,7 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* } int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp) { - qInfo("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->sourceTaskId); + qInfo("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, pReq->upstreamTaskId); // 1. handle input streamTaskEnqueue(pTask, pReq, pRsp); @@ -188,6 +227,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp) { int32_t streamProcessRunReq(SStreamTask* pTask) { streamExec(pTask, pTask->pMsgCb); + if (pTask->dispatchType != TASK_DISPATCH__NONE) { streamDispatch(pTask, pTask->pMsgCb); } @@ -203,3 +243,22 @@ int32_t streamProcessRecoverRsp(SStreamTask* pTask, SStreamTaskRecoverRsp* pRsp) // return 0; } + +int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { + qInfo("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId); + + streamTaskEnqueueRetrieve(pTask, pReq, pRsp); + + ASSERT(pTask->execType != TASK_EXEC__NONE); + streamExec(pTask, pTask->pMsgCb); + + ASSERT(pTask->dispatchType != TASK_DISPATCH__NONE); + streamDispatch(pTask, pTask->pMsgCb); + + return 0; +} + +int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) { + // + return 0; +} diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 6699e86b1e6137cd9f7a5f873d207c23a0cc09f9..ef328ecf848de7d0fdde0ab8780c9cefd62943bc 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -15,27 +15,6 @@ #include "streamInc.h" -#if 0 -int32_t streamDataBlockEncode(void** buf, const SStreamDataBlock* pOutput) { - int32_t tlen = 0; - tlen += taosEncodeFixedI8(buf, pOutput->type); - tlen += taosEncodeFixedI32(buf, pOutput->sourceVg); - tlen += taosEncodeFixedI64(buf, pOutput->sourceVer); - ASSERT(pOutput->type == STREAM_INPUT__DATA_BLOCK); - tlen += tEncodeDataBlocks(buf, pOutput->blocks); - return tlen; -} - -void* streamDataBlockDecode(const void* buf, SStreamDataBlock* pInput) { - buf = taosDecodeFixedI8(buf, &pInput->type); - buf = taosDecodeFixedI32(buf, &pInput->sourceVg); - buf = taosDecodeFixedI64(buf, &pInput->sourceVer); - ASSERT(pInput->type == STREAM_INPUT__DATA_BLOCK); - buf = tDecodeDataBlocks(buf, &pInput->blocks); - return (void*)buf; -} -#endif - int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock* pData) { int32_t blockNum = pReq->blockNum; SArray* pArray = taosArrayInit(blockNum, sizeof(SSDataBlock)); @@ -54,8 +33,21 @@ int32_t streamDispatchReqToData(const SStreamDispatchReq* pReq, SStreamDataBlock blockCompressDecode(pDataBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); // TODO: refactor pDataBlock->info.type = pRetrieve->streamBlockType; - pDataBlock->info.childId = pReq->sourceChildId; + pDataBlock->info.childId = pReq->upstreamChildId; + } + pData->blocks = pArray; + return 0; +} + +int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock* pData) { + SArray* pArray = taosArrayInit(1, sizeof(SSDataBlock)); + if (pArray == NULL) { + return -1; } + taosArraySetSize(pArray, 1); + SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; + SSDataBlock* pBlock = taosArrayGet(pArray, 0); + blockCompressDecode(pBlock, htonl(pRetrieve->numOfCols), htonl(pRetrieve->numOfRows), pRetrieve->data); pData->blocks = pArray; return 0; } diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 1894f697c01d550fd4877d2dc533d527b4d51eb4..b5a9e221a74195e4799efab35ae3d2e52bf09efa 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -19,9 +19,9 @@ int32_t tEncodeStreamDispatchReq(SEncoder* pEncoder, const SStreamDispatchReq* p if (tStartEncode(pEncoder) < 0) return -1; if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->taskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->sourceTaskId) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->sourceVg) < 0) return -1; - if (tEncodeI32(pEncoder, pReq->sourceChildId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->dataSrcVgId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->upstreamChildId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->upstreamNodeId) < 0) return -1; if (tEncodeI32(pEncoder, pReq->blockNum) < 0) return -1; ASSERT(taosArrayGetSize(pReq->data) == pReq->blockNum); @@ -40,9 +40,9 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { if (tStartDecode(pDecoder) < 0) return -1; if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->taskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->sourceTaskId) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->sourceVg) < 0) return -1; - if (tDecodeI32(pDecoder, &pReq->sourceChildId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->dataSrcVgId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->upstreamChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->upstreamNodeId) < 0) return -1; if (tDecodeI32(pDecoder, &pReq->blockNum) < 0) return -1; ASSERT(pReq->blockNum > 0); @@ -62,6 +62,103 @@ int32_t tDecodeStreamDispatchReq(SDecoder* pDecoder, SStreamDispatchReq* pReq) { return 0; } +int32_t tEncodeStreamRetrieveReq(SEncoder* pEncoder, const SStreamRetrieveReq* pReq) { + // + if (tStartEncode(pEncoder) < 0) return -1; + if (tEncodeI64(pEncoder, pReq->streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->dstNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->dstTaskId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->srcNodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pReq->srcTaskId) < 0) return -1; + if (tEncodeBinary(pEncoder, (const uint8_t*)&pReq->pRetrieve, pReq->retrieveLen) < 0) return -1; + tEndEncode(pEncoder); + return pEncoder->pos; +} + +int32_t tDecodeStreamRetrieveReq(SDecoder* pDecoder, SStreamRetrieveReq* pReq) { + int32_t tlen = 0; + if (tStartDecode(pDecoder) < 0) return -1; + if (tDecodeI64(pDecoder, &pReq->streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->dstNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->dstTaskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->srcNodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pReq->srcTaskId) < 0) return -1; + if (tDecodeBinary(pDecoder, (uint8_t**)&pReq->pRetrieve, &pReq->retrieveLen) < 0) return -1; + tEndDecode(pDecoder); + return 0; +} + +int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) { + SRetrieveTableRsp* pRetrieve = NULL; + void* buf = NULL; + int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); + + pRetrieve = taosMemoryCalloc(1, dataStrLen); + if (pRetrieve == NULL) return -1; + + int32_t numOfCols = taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->useconds = 0; + pRetrieve->precision = TSDB_DEFAULT_PRECISION; + pRetrieve->compressed = 0; + pRetrieve->completed = 1; + pRetrieve->streamBlockType = pBlock->info.type; + pRetrieve->numOfRows = htonl(pBlock->info.rows); + pRetrieve->numOfCols = htonl(numOfCols); + + int32_t actualLen = 0; + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); + + SStreamRetrieveReq req = { + .streamId = pTask->streamId, + .srcNodeId = pTask->nodeId, + .srcTaskId = pTask->taskId, + .pRetrieve = pRetrieve, + }; + + int32_t sz = taosArrayGetSize(pTask->childEpInfo); + ASSERT(sz > 0); + for (int32_t i = 0; i < sz; i++) { + SStreamChildEpInfo* pEpInfo = taosArrayGetP(pTask->childEpInfo, i); + req.dstNodeId = pEpInfo->nodeId; + req.dstTaskId = pEpInfo->taskId; + int32_t code; + int32_t len; + tEncodeSize(tEncodeStreamRetrieveReq, &req, len, code); + if (code < 0) { + ASSERT(0); + return -1; + } + + buf = rpcMallocCont(sizeof(SMsgHead) + len); + if (buf == NULL) { + goto FAIL; + } + + ((SMsgHead*)buf)->vgId = htonl(pEpInfo->nodeId); + void* abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); + SEncoder encoder; + tEncoderInit(&encoder, abuf, len); + tEncodeStreamRetrieveReq(&encoder, &req); + + SRpcMsg rpcMsg = { + .code = 0, + .msgType = TDMT_STREAM_RETRIEVE, + .pCont = buf, + .contLen = len, + }; + + if (tmsgSendReq(&pEpInfo->epSet, &rpcMsg) < 0) { + ASSERT(0); + return -1; + } + } + return 0; +FAIL: + if (pRetrieve) taosMemoryFree(pRetrieve); + if (buf) taosMemoryFree(buf); + return -1; +} + static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDispatchReq* pReq) { int32_t dataStrLen = sizeof(SRetrieveTableRsp) + blockGetEncodeSize(pBlock); void* buf = taosMemoryCalloc(1, dataStrLen); @@ -74,10 +171,12 @@ static int32_t streamAddBlockToDispatchMsg(const SSDataBlock* pBlock, SStreamDis pRetrieve->completed = 1; pRetrieve->streamBlockType = pBlock->info.type; pRetrieve->numOfRows = htonl(pBlock->info.rows); - pRetrieve->numOfCols = htonl(pBlock->info.numOfCols); + + int32_t numOfCols = (int32_t) taosArrayGetSize(pBlock->pDataBlock); + pRetrieve->numOfCols = htonl(numOfCols); int32_t actualLen = 0; - blockCompressEncode(pBlock, pRetrieve->data, &actualLen, pBlock->info.numOfCols, false); + blockCompressEncode(pBlock, pRetrieve->data, &actualLen, numOfCols, false); actualLen += sizeof(SRetrieveTableRsp); ASSERT(actualLen <= dataStrLen); taosArrayPush(pReq->dataLen, &actualLen); @@ -94,9 +193,9 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM SStreamDispatchReq req = { .streamId = pTask->streamId, - .sourceTaskId = pTask->taskId, - .sourceVg = data->sourceVg, - .sourceChildId = pTask->childId, + .dataSrcVgId = data->srcVgId, + .upstreamTaskId = pTask->taskId, + .upstreamChildId = pTask->selfChildId, .upstreamNodeId = pTask->nodeId, .blockNum = blockNum, }; @@ -147,7 +246,7 @@ int32_t streamBuildDispatchMsg(SStreamTask* pTask, SStreamDataBlock* data, SRpcM ASSERT(vgId > 0 || vgId == SNODE_HANDLE); req.taskId = downstreamTaskId; - qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->childId, + qInfo("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, downstreamTaskId, vgId); // serialize diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 5a71fccab886f56ddc5c9de178879a0f1feb145a..6fada59c84e38392db20e6862bb847468de46eb2 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -13,8 +13,7 @@ * along with this program. If not, see . */ -#include "executor.h" -#include "tstream.h" +#include "streamInc.h" static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) { void* exec = pTask->exec.executor; @@ -25,14 +24,16 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) SStreamTrigger* pTrigger = (SStreamTrigger*)data; qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_DATA_TYPE_SSDATA_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { + ASSERT(pTask->isDataScan); SStreamDataSubmit* pSubmit = (SStreamDataSubmit*)data; - ASSERT(pTask->inputType == STREAM_INPUT__DATA_SUBMIT); qSetStreamInput(exec, pSubmit->data, STREAM_DATA_TYPE_SUBMIT_BLOCK, false); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)data; - ASSERT(pTask->inputType == STREAM_INPUT__DATA_BLOCK); - SArray* blocks = pBlock->blocks; + SArray* blocks = pBlock->blocks; qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_DATA_TYPE_SSDATA_BLOCK, false); + } else if (pItem->type == STREAM_INPUT__DROP) { + // TODO exec drop + return 0; } // exec @@ -43,9 +44,17 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, void* data, SArray* pRes) ASSERT(false); } if (output == NULL) break; + + if (output->info.type == STREAM_RETRIEVE) { + if (streamBroadcastToChildren(pTask, output) < 0) { + // TODO + } + continue; + } + // TODO: do we need free memory? SSDataBlock* outputCopy = createOneDataBlock(output, true); - outputCopy->info.childId = pTask->childId; + outputCopy->info.childId = pTask->selfChildId; taosArrayPush(pRes, outputCopy); } return 0; @@ -58,6 +67,10 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { streamTaskExecImpl(pTask, data, pRes); + if (pTask->taskStatus == TASK_STATUS__DROPPING) { + return NULL; + } + if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM); if (qRes == NULL) { @@ -67,7 +80,6 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { } qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - /*qRes->sourceVg = pTask->nodeId;*/ if (streamTaskOutput(pTask, qRes) < 0) { streamQueueProcessFail(pTask->inputQueue); taosArrayDestroy(pRes); @@ -75,12 +87,17 @@ static SArray* streamExecForQall(SStreamTask* pTask, SArray* pRes) { return NULL; } - if (pTask->inputType == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit*)data); + int8_t type = ((SStreamQueueItem*)data)->type; + if (type == STREAM_INPUT__TRIGGER) { + blockDataDestroy(((SStreamTrigger*)data)->pBlock); taosFreeQitem(data); - } else { + } else if (type == STREAM_INPUT__DATA_BLOCK) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)tDeleteSSDataBlock); taosFreeQitem(data); + } else if (type == STREAM_INPUT__DATA_SUBMIT) { + ASSERT(pTask->isDataScan); + streamDataSubmitRefDec((SStreamDataSubmit*)data); + taosFreeQitem(data); } streamQueueProcessSuccess(pTask->inputQueue); return taosArrayInit(0, sizeof(SSDataBlock)); @@ -94,25 +111,26 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); if (pRes == NULL) return -1; while (1) { - int8_t execStatus = atomic_val_compare_exchange_8(&pTask->status, TASK_STATUS__IDLE, TASK_STATUS__EXECUTING); - if (execStatus == TASK_STATUS__IDLE) { + int8_t execStatus = + atomic_val_compare_exchange_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE, TASK_EXEC_STATUS__EXECUTING); + if (execStatus == TASK_EXEC_STATUS__IDLE) { // first run pRes = streamExecForQall(pTask, pRes); if (pRes == NULL) goto FAIL; // set status closing - atomic_store_8(&pTask->status, TASK_STATUS__CLOSING); + atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__CLOSING); // second run, make sure inputQ and qall are cleared pRes = streamExecForQall(pTask, pRes); if (pRes == NULL) goto FAIL; taosArrayDestroy(pRes); - atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); return 0; - } else if (execStatus == TASK_STATUS__CLOSING) { + } else if (execStatus == TASK_EXEC_STATUS__CLOSING) { continue; - } else if (execStatus == TASK_STATUS__EXECUTING) { + } else if (execStatus == TASK_EXEC_STATUS__EXECUTING) { ASSERT(taosArrayGetSize(pRes) == 0); taosArrayDestroy(pRes); return 0; @@ -122,7 +140,7 @@ int32_t streamExec(SStreamTask* pTask, SMsgCb* pMsgCb) { } FAIL: if (pRes) taosArrayDestroy(pRes); - atomic_store_8(&pTask->status, TASK_STATUS__IDLE); + atomic_store_8(&pTask->execStatus, TASK_EXEC_STATUS__IDLE); return -1; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index b5a63d937ae6be238406839921ba8a514568001a..1daa9bd50e3dc32a56dbf93d969c01e3c12e66b9 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -23,29 +23,53 @@ SStreamTask* tNewSStreamTask(int64_t streamId) { } pTask->taskId = tGenIdPI32(); pTask->streamId = streamId; - pTask->status = TASK_STATUS__IDLE; + pTask->execStatus = TASK_EXEC_STATUS__IDLE; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; return pTask; } +int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo) { + if (tEncodeI32(pEncoder, pInfo->taskId) < 0) return -1; + if (tEncodeI32(pEncoder, pInfo->nodeId) < 0) return -1; + if (tEncodeI32(pEncoder, pInfo->childId) < 0) return -1; + if (tEncodeSEpSet(pEncoder, &pInfo->epSet) < 0) return -1; + return 0; +} + +int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { + if (tDecodeI32(pDecoder, &pInfo->taskId) < 0) return -1; + if (tDecodeI32(pDecoder, &pInfo->nodeId) < 0) return -1; + if (tDecodeI32(pDecoder, &pInfo->childId) < 0) return -1; + if (tDecodeSEpSet(pDecoder, &pInfo->epSet) < 0) return -1; + return 0; +} + int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { /*if (tStartEncode(pEncoder) < 0) return -1;*/ if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->inputType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->status) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->isDataScan) < 0) return -1; if (tEncodeI8(pEncoder, pTask->execType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->sinkType) < 0) return -1; if (tEncodeI8(pEncoder, pTask->dispatchType) < 0) return -1; if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->dataScan) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->childId) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->execStatus) < 0) return -1; + + if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; + int32_t epSz = taosArrayGetSize(pTask->childEpInfo); + if (tEncodeI32(pEncoder, epSz) < 0) return -1; + for (int32_t i = 0; i < epSz; i++) { + SStreamChildEpInfo* pInfo = taosArrayGetP(pTask->childEpInfo, i); + if (tEncodeStreamEpInfo(pEncoder, pInfo) < 0) return -1; + } + if (pTask->execType != TASK_EXEC__NONE) { if (tEncodeCStr(pEncoder, pTask->exec.qmsg) < 0) return -1; } @@ -82,18 +106,29 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { /*if (tStartDecode(pDecoder) < 0) return -1;*/ if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->inputType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->status) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->isDataScan) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->execType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->sinkType) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->dispatchType) < 0) return -1; if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->dataScan) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->childId) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->execStatus) < 0) return -1; + + if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; + int32_t epSz; + if (tDecodeI32(pDecoder, &epSz) < 0) return -1; + pTask->childEpInfo = taosArrayInit(epSz, sizeof(void*)); + for (int32_t i = 0; i < epSz; i++) { + SStreamChildEpInfo* pInfo = taosMemoryCalloc(1, sizeof(SStreamChildEpInfo)); + if (pInfo == NULL) return -1; + if (tDecodeStreamEpInfo(pDecoder, pInfo) < 0) return -1; + taosArrayPush(pTask->childEpInfo, &pInfo); + } + if (pTask->execType != TASK_EXEC__NONE) { if (tDecodeCStrAlloc(pDecoder, &pTask->exec.qmsg) < 0) return -1; } diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 63db395425c23d2c4ac50fda93b88b974c4787ab..1bd0671fb5515d901a4271f39d1e9f05131d868d 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -197,6 +197,7 @@ int32_t syncNodeSendMsgByInfo(const SNodeInfo* nodeInfo, SSyncNode* pSyncNode, S cJSON* syncNode2Json(const SSyncNode* pSyncNode); char* syncNode2Str(const SSyncNode* pSyncNode); void syncNodeEventLog(const SSyncNode* pSyncNode, char* str); +void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str); char* syncNode2SimpleStr(const SSyncNode* pSyncNode); bool syncNodeInConfig(SSyncNode* pSyncNode, const SSyncCfg* config); void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* newConfig, SyncIndex lastConfigChangeIndex); @@ -253,6 +254,7 @@ void syncNodePrint(SSyncNode* pObj); void syncNodePrint2(char* s, SSyncNode* pObj); void syncNodeLog(SSyncNode* pObj); void syncNodeLog2(char* s, SSyncNode* pObj); +void syncNodeLog3(char* s, SSyncNode* pObj); #ifdef __cplusplus } diff --git a/source/libs/sync/inc/syncRaftCfg.h b/source/libs/sync/inc/syncRaftCfg.h index 7f45276e9f2bb5891e96b22650fd3eddd37a338b..efe8a65b776f7c00c9c54f49f7eaadddcebb153c 100644 --- a/source/libs/sync/inc/syncRaftCfg.h +++ b/source/libs/sync/inc/syncRaftCfg.h @@ -27,7 +27,7 @@ extern "C" { #include "syncInt.h" #include "taosdef.h" -#define CONFIG_FILE_LEN 1024 +#define CONFIG_FILE_LEN 2048 #define MAX_CONFIG_INDEX_COUNT 512 @@ -49,14 +49,14 @@ int32_t raftCfgClose(SRaftCfg *pRaftCfg); int32_t raftCfgPersist(SRaftCfg *pRaftCfg); int32_t raftCfgAddConfigIndex(SRaftCfg *pRaftCfg, SyncIndex configIndex); -cJSON * syncCfg2Json(SSyncCfg *pSyncCfg); -char * syncCfg2Str(SSyncCfg *pSyncCfg); -char * syncCfg2SimpleStr(SSyncCfg *pSyncCfg); +cJSON *syncCfg2Json(SSyncCfg *pSyncCfg); +char *syncCfg2Str(SSyncCfg *pSyncCfg); +char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg); int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg); int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg); -cJSON * raftCfg2Json(SRaftCfg *pRaftCfg); -char * raftCfg2Str(SRaftCfg *pRaftCfg); +cJSON *raftCfg2Json(SRaftCfg *pRaftCfg); +char *raftCfg2Str(SRaftCfg *pRaftCfg); int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg); int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg); diff --git a/source/libs/sync/inc/syncSnapshot.h b/source/libs/sync/inc/syncSnapshot.h index 069154fb93416c3b90840ed81ea8ac2bebf2d8b3..f00dfe6f50ea72dbb68eebc902d17b4bfb1924a4 100644 --- a/source/libs/sync/inc/syncSnapshot.h +++ b/source/libs/sync/inc/syncSnapshot.h @@ -39,8 +39,8 @@ typedef struct SSyncSnapshotSender { bool start; int32_t seq; int32_t ack; - void *pReader; - void *pCurrentBlock; + void * pReader; + void * pCurrentBlock; int32_t blockLen; SSnapshot snapshot; SSyncCfg lastConfig; @@ -55,19 +55,19 @@ typedef struct SSyncSnapshotSender { SSyncSnapshotSender *snapshotSenderCreate(SSyncNode *pSyncNode, int32_t replicaIndex); void snapshotSenderDestroy(SSyncSnapshotSender *pSender); bool snapshotSenderIsStart(SSyncSnapshotSender *pSender); -void snapshotSenderStart(SSyncSnapshotSender *pSender); +void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader); void snapshotSenderStop(SSyncSnapshotSender *pSender); int32_t snapshotSend(SSyncSnapshotSender *pSender); int32_t snapshotReSend(SSyncSnapshotSender *pSender); -cJSON *snapshotSender2Json(SSyncSnapshotSender *pSender); -char *snapshotSender2Str(SSyncSnapshotSender *pSender); -char *snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); +cJSON * snapshotSender2Json(SSyncSnapshotSender *pSender); +char * snapshotSender2Str(SSyncSnapshotSender *pSender); +char * snapshotSender2SimpleStr(SSyncSnapshotSender *pSender, char *event); typedef struct SSyncSnapshotReceiver { bool start; int32_t ack; - void *pWriter; + void * pWriter; SyncTerm term; SyncTerm privateTerm; SSnapshot snapshot; @@ -83,8 +83,8 @@ void snapshotReceiverStart(SSyncSnapshotReceiver *pReceiver, SyncTerm privateT bool snapshotReceiverIsStart(SSyncSnapshotReceiver *pReceiver); void snapshotReceiverStop(SSyncSnapshotReceiver *pReceiver, bool apply); cJSON *snapshotReceiver2Json(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); -char *snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); +char * snapshotReceiver2Str(SSyncSnapshotReceiver *pReceiver); +char * snapshotReceiver2SimpleStr(SSyncSnapshotReceiver *pReceiver, char *event); int32_t syncNodeOnSnapshotSendCb(SSyncNode *ths, SyncSnapshotSend *pMsg); int32_t syncNodeOnSnapshotRspCb(SSyncNode *ths, SyncSnapshotRsp *pMsg); diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index d2726201cceb50c910e0cebfff23322c6747a1ef..bc5342cc7e1fc441706f3758c50ec45ada5981b6 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -99,19 +99,25 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); } - assert(pMsg->term <= ths->pRaftStore->currentTerm); + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); // reset elect timer if (pMsg->term == ths->pRaftStore->currentTerm) { ths->leaderCache = pMsg->srcId; syncNodeResetElectTimer(ths); } - assert(pMsg->dataLen >= 0); + ASSERT(pMsg->dataLen >= 0); SyncTerm localPreLogTerm = 0; if (pMsg->prevLogIndex >= SYNC_INDEX_BEGIN && pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)) { SSyncRaftEntry* pEntry = ths->pLogStore->getEntry(ths->pLogStore, pMsg->prevLogIndex); - assert(pEntry != NULL); + if (pEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error, index:%ld, since %s", pMsg->prevLogIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } + localPreLogTerm = pEntry->term; syncEntryDestory(pEntry); } @@ -160,7 +166,7 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // accept request if (pMsg->term == ths->pRaftStore->currentTerm && ths->state == TAOS_SYNC_STATE_FOLLOWER && logOK) { // preIndex = -1, or has preIndex entry in local log - assert(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); + ASSERT(pMsg->prevLogIndex <= ths->pLogStore->getLastIndex(ths->pLogStore)); // has extra entries (> preIndex) in local log bool hasExtraEntries = pMsg->prevLogIndex < ths->pLogStore->getLastIndex(ths->pLogStore); @@ -179,13 +185,21 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { SyncIndex extraIndex = pMsg->prevLogIndex + 1; SSyncRaftEntry* pExtraEntry = ths->pLogStore->getEntry(ths->pLogStore, extraIndex); - assert(pExtraEntry != NULL); + if (pExtraEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error2, index:%ld, since %s", extraIndex, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry error"); + return -1; + } // log not match, conflict - assert(extraIndex == pAppendEntry->index); + ASSERT(extraIndex == pAppendEntry->index); if (pExtraEntry->term != pAppendEntry->term) { conflict = true; } @@ -201,7 +215,12 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { for (SyncIndex index = delEnd; index >= delBegin; --index) { if (ths->pFsm->FpRollBackCb != NULL) { SSyncRaftEntry* pRollBackEntry = ths->pLogStore->getEntry(ths->pLogStore, index); - assert(pRollBackEntry != NULL); + if (pRollBackEntry == NULL) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "getEntry error3, index:%ld, since %s", index, terrstr()); + syncNodeErrorLog(ths, logBuf); + return -1; + } // if (pRollBackEntry->msgType != TDMT_SYNC_NOOP) { if (syncUtilUserRollback(pRollBackEntry->msgType)) { @@ -257,7 +276,10 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { } else if (!hasExtraEntries && hasAppendEntries) { SSyncRaftEntry* pAppendEntry = syncEntryDeserialize(pMsg->data, pMsg->dataLen); - assert(pAppendEntry != NULL); + if (pAppendEntry == NULL) { + syncNodeErrorLog(ths, "syncEntryDeserialize pAppendEntry2 error"); + return -1; + } // append new entries ths->pLogStore->appendEntry(ths->pLogStore, pAppendEntry); @@ -287,7 +309,8 @@ int32_t syncNodeOnAppendEntriesCb(SSyncNode* ths, SyncAppendEntries* pMsg) { // do nothing } else { - assert(0); + syncNodeLog3("", ths); + ASSERT(0); } SyncAppendEntriesReply* pReply = syncAppendEntriesReplyBuild(ths->vgId); @@ -413,6 +436,11 @@ static bool syncNodeOnAppendEntriesLogOK(SSyncNode* pSyncNode, SyncAppendEntries } SyncTerm myPreLogTerm = syncNodeGetPreTerm(pSyncNode, pMsg->prevLogIndex + 1); + if (myPreLogTerm == SYNC_TERM_INVALID) { + sError("vgId:%d sync get pre term error, preindex:%ld", pSyncNode->vgId, pMsg->prevLogIndex); + return false; + } + if (pMsg->prevLogIndex <= myLastIndex && pMsg->prevLogTerm == myPreLogTerm) { if (gRaftDetailLog) { sTrace( diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index 47fd23baaef98add632cfbdd65d7ea73ef6efdfb..f13a3604dab1e776583164c0315b243a6a766665 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -67,7 +67,7 @@ int32_t syncNodeOnAppendEntriesReplyCb(SSyncNode* ths, SyncAppendEntriesReply* p return ret; } - assert(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); if (pMsg->success) { // nextIndex' = [nextIndex EXCEPT ![i][j] = m.mmatchIndex + 1] @@ -173,21 +173,44 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries // get sender SSyncSnapshotSender* pSender = syncNodeGetSnapshotSender(ths, &(pMsg->srcId)); ASSERT(pSender != NULL); - bool hasSnapshot = syncNodeHasSnapshot(ths); - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - // start sending snapshot first time - // start here, stop by receiver - if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && - pMsg->privateTerm < pSender->privateTerm) { - snapshotSenderStart(pSender); + SSnapshot snapshot; + void* pReader = NULL; + ths->pFsm->FpGetSnapshot(ths->pFsm, &snapshot, NULL, &pReader); + if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN && nextIndex <= snapshot.lastApplyIndex + 1 && + !snapshotSenderIsStart(pSender) && pMsg->privateTerm < pSender->privateTerm) { + // has snapshot + ASSERT(pReader != NULL); + snapshotSenderStart(pSender, snapshot, pReader); char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); syncNodeEventLog(ths, eventLog); taosMemoryFree(eventLog); + + } else { + // no snapshot + if (pReader != NULL) { + ths->pFsm->FpSnapshotStopRead(ths->pFsm, pReader); + } } + /* + bool hasSnapshot = syncNodeHasSnapshot(ths); + SSnapshot snapshot; + ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); + + // start sending snapshot first time + // start here, stop by receiver + if (hasSnapshot && nextIndex <= snapshot.lastApplyIndex + 1 && !snapshotSenderIsStart(pSender) && + pMsg->privateTerm < pSender->privateTerm) { + snapshotSenderStart(pSender); + + char* eventLog = snapshotSender2SimpleStr(pSender, "snapshot sender start"); + syncNodeEventLog(ths, eventLog); + taosMemoryFree(eventLog); + } + */ + SyncIndex sentryIndex = pSender->snapshot.lastApplyIndex + 1; // update nextIndex to sentryIndex @@ -207,12 +230,6 @@ int32_t syncNodeOnAppendEntriesReplySnapshotCb(SSyncNode* ths, SyncAppendEntries syncIndexMgrLog2("recv SyncAppendEntriesReply, after pNextIndex:", ths->pNextIndex); syncIndexMgrLog2("recv SyncAppendEntriesReply, after pMatchIndex:", ths->pMatchIndex); - if (gRaftDetailLog) { - SSnapshot snapshot; - ths->pFsm->FpGetSnapshotInfo(ths->pFsm, &snapshot); - sTrace("recv SyncAppendEntriesReply, after snapshot.lastApplyIndex:%ld, snapshot.lastApplyTerm:%lu", - snapshot.lastApplyIndex, snapshot.lastApplyTerm); - } return ret; } \ No newline at end of file diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index ec3e1ab2baf94deb6153f3f0b69b067e05f33a92..c92edae381ae631cacb2b2dfd20c25e8f46eb4c9 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -75,7 +75,7 @@ void syncMaybeAdvanceCommitIndex(SSyncNode* pSyncNode) { if (agree) { // term SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, index); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); // cannot commit, even if quorum agree. need check term! if (pEntry->term == pSyncNode->pRaftStore->currentTerm) { diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index fdebbe3990f32efed4b5e63a07dcc50ad2666fd8..e0f2b0fed2fb7b2c5dffccdd05db6095dbf72c71 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -32,7 +32,7 @@ // /\ UNCHANGED <> // int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); int32_t ret = 0; for (int i = 0; i < pSyncNode->peersNum; ++i) { @@ -44,7 +44,7 @@ int32_t syncNodeRequestVotePeers(SSyncNode* pSyncNode) { pMsg->lastLogTerm = pSyncNode->pLogStore->getLastTerm(pSyncNode->pLogStore); ret = syncNodeRequestVote(pSyncNode, &pSyncNode->peersId[i], pMsg); - assert(ret == 0); + ASSERT(ret == 0); syncRequestVoteDestroy(pMsg); } return ret; @@ -75,7 +75,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { syncNodeFollower2Candidate(pSyncNode); } - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); // start election raftStoreNextTerm(pSyncNode->pRaftStore); @@ -86,7 +86,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { syncNodeVoteForSelf(pSyncNode); if (voteGrantedMajority(pSyncNode->pVotesGranted)) { // only myself, to leader - assert(!pSyncNode->pVotesGranted->toLeader); + ASSERT(!pSyncNode->pVotesGranted->toLeader); syncNodeCandidate2Leader(pSyncNode); pSyncNode->pVotesGranted->toLeader = true; return ret; @@ -98,7 +98,7 @@ int32_t syncNodeElect(SSyncNode* pSyncNode) { ret = syncNodeRequestVotePeers(pSyncNode); } - assert(ret == 0); + ASSERT(ret == 0); syncNodeResetElectTimer(pSyncNode); return ret; diff --git a/source/libs/sync/src/syncEnv.c b/source/libs/sync/src/syncEnv.c index e67439f8feae8601e75e0c8df2dcc82aced32a86..fb0bfb8bef318c2f8d2d53f12cfabcfa4fb30ef2 100644 --- a/source/libs/sync/src/syncEnv.c +++ b/source/libs/sync/src/syncEnv.c @@ -14,7 +14,7 @@ */ #include "syncEnv.h" -// #include +// #include SSyncEnv *gSyncEnv = NULL; @@ -35,11 +35,12 @@ bool syncEnvIsStart() { } int32_t syncEnvStart() { - int32_t ret = 0; - taosSeedRand(taosGetTimestampSec()); + int32_t ret = 0; + uint32_t seed = (uint32_t)(taosGetTimestampNs() & 0x00000000FFFFFFFF); + taosSeedRand(seed); // gSyncEnv = doSyncEnvStart(gSyncEnv); gSyncEnv = doSyncEnvStart(); - assert(gSyncEnv != NULL); + ASSERT(gSyncEnv != NULL); sTrace("sync env start ok"); return ret; } @@ -85,7 +86,7 @@ static void syncEnvTick(void *param, void *tmrId) { static SSyncEnv *doSyncEnvStart() { SSyncEnv *pSyncEnv = (SSyncEnv *)taosMemoryMalloc(sizeof(SSyncEnv)); - assert(pSyncEnv != NULL); + ASSERT(pSyncEnv != NULL); memset(pSyncEnv, 0, sizeof(SSyncEnv)); pSyncEnv->envTickTimerCounter = 0; @@ -102,7 +103,7 @@ static SSyncEnv *doSyncEnvStart() { } static int32_t doSyncEnvStop(SSyncEnv *pSyncEnv) { - assert(pSyncEnv == gSyncEnv); + ASSERT(pSyncEnv == gSyncEnv); if (pSyncEnv != NULL) { atomic_store_8(&(pSyncEnv->isStart), 0); taosTmrCleanUp(pSyncEnv->pTimerManager); diff --git a/source/libs/sync/src/syncIO.c b/source/libs/sync/src/syncIO.c index 0b5a9685c0e86de847fbf2d4e6cc180650872ab1..038c36c417e5586ff144d173cf42fa7094724a94 100644 --- a/source/libs/sync/src/syncIO.c +++ b/source/libs/sync/src/syncIO.c @@ -30,7 +30,7 @@ static int32_t syncIODestroy(SSyncIO *io); static int32_t syncIOStartInternal(SSyncIO *io); static int32_t syncIOStopInternal(SSyncIO *io); -static void * syncIOConsumerFunc(void *param); +static void *syncIOConsumerFunc(void *param); static void syncIOProcessRequest(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static void syncIOProcessReply(void *pParent, SRpcMsg *pMsg, SEpSet *pEpSet); static int32_t syncIOAuth(void *parent, char *meterId, char *spi, char *encrypt, char *secret, char *ckey); @@ -47,11 +47,11 @@ static void syncIOTickPing(void *param, void *tmrId); int32_t syncIOStart(char *host, uint16_t port) { int32_t ret = 0; gSyncIO = syncIOCreate(host, port); - assert(gSyncIO != NULL); + ASSERT(gSyncIO != NULL); taosSeedRand(taosGetTimestampSec()); ret = syncIOStartInternal(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); sTrace("syncIOStart ok, gSyncIO:%p", gSyncIO); return ret; @@ -59,16 +59,16 @@ int32_t syncIOStart(char *host, uint16_t port) { int32_t syncIOStop() { int32_t ret = syncIOStopInternal(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); ret = syncIODestroy(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); return ret; } int32_t syncIOSendMsg(const SEpSet *pEpSet, SRpcMsg *pMsg) { - assert(pEpSet->inUse == 0); - assert(pEpSet->numOfEps == 1); + ASSERT(pEpSet->inUse == 0); + ASSERT(pEpSet->numOfEps == 1); int32_t ret = 0; { @@ -107,25 +107,25 @@ int32_t syncIOEqMsg(const SMsgCb *msgcb, SRpcMsg *pMsg) { int32_t syncIOQTimerStart() { int32_t ret = syncIOStartQ(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); return ret; } int32_t syncIOQTimerStop() { int32_t ret = syncIOStopQ(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); return ret; } int32_t syncIOPingTimerStart() { int32_t ret = syncIOStartPing(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); return ret; } int32_t syncIOPingTimerStop() { int32_t ret = syncIOStopPing(gSyncIO); - assert(ret == 0); + ASSERT(ret == 0); return ret; } @@ -151,7 +151,7 @@ static SSyncIO *syncIOCreate(char *host, uint16_t port) { static int32_t syncIODestroy(SSyncIO *io) { int32_t ret = 0; int8_t start = atomic_load_8(&io->isStart); - assert(start == 0); + ASSERT(start == 0); if (io->serverRpc != NULL) { rpcClose(io->serverRpc); @@ -242,9 +242,9 @@ static int32_t syncIOStopInternal(SSyncIO *io) { } static void *syncIOConsumerFunc(void *param) { - SSyncIO * io = param; + SSyncIO *io = param; STaosQall *qall; - SRpcMsg * pRpcMsg, rpcMsg; + SRpcMsg *pRpcMsg, rpcMsg; qall = taosAllocateQall(); while (1) { @@ -264,7 +264,7 @@ static void *syncIOConsumerFunc(void *param) { if (pRpcMsg->msgType == TDMT_SYNC_PING) { if (io->FpOnSyncPing != NULL) { SyncPing *pSyncMsg = syncPingFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncPing(io->pSyncNode, pSyncMsg); syncPingDestroy(pSyncMsg); } @@ -272,7 +272,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_PING_REPLY) { if (io->FpOnSyncPingReply != NULL) { SyncPingReply *pSyncMsg = syncPingReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncPingReply(io->pSyncNode, pSyncMsg); syncPingReplyDestroy(pSyncMsg); } @@ -280,7 +280,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_CLIENT_REQUEST) { if (io->FpOnSyncClientRequest != NULL) { SyncClientRequest *pSyncMsg = syncClientRequestFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncClientRequest(io->pSyncNode, pSyncMsg); syncClientRequestDestroy(pSyncMsg); } @@ -288,7 +288,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE) { if (io->FpOnSyncRequestVote != NULL) { SyncRequestVote *pSyncMsg = syncRequestVoteFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncRequestVote(io->pSyncNode, pSyncMsg); syncRequestVoteDestroy(pSyncMsg); } @@ -296,7 +296,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_REQUEST_VOTE_REPLY) { if (io->FpOnSyncRequestVoteReply != NULL) { SyncRequestVoteReply *pSyncMsg = syncRequestVoteReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncRequestVoteReply(io->pSyncNode, pSyncMsg); syncRequestVoteReplyDestroy(pSyncMsg); } @@ -304,7 +304,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES) { if (io->FpOnSyncAppendEntries != NULL) { SyncAppendEntries *pSyncMsg = syncAppendEntriesFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncAppendEntries(io->pSyncNode, pSyncMsg); syncAppendEntriesDestroy(pSyncMsg); } @@ -312,7 +312,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_APPEND_ENTRIES_REPLY) { if (io->FpOnSyncAppendEntriesReply != NULL) { SyncAppendEntriesReply *pSyncMsg = syncAppendEntriesReplyFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncAppendEntriesReply(io->pSyncNode, pSyncMsg); syncAppendEntriesReplyDestroy(pSyncMsg); } @@ -320,7 +320,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_TIMEOUT) { if (io->FpOnSyncTimeout != NULL) { SyncTimeout *pSyncMsg = syncTimeoutFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncTimeout(io->pSyncNode, pSyncMsg); syncTimeoutDestroy(pSyncMsg); } @@ -328,7 +328,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_SEND) { if (io->FpOnSyncSnapshotSend != NULL) { SyncSnapshotSend *pSyncMsg = syncSnapshotSendFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncSnapshotSend(io->pSyncNode, pSyncMsg); syncSnapshotSendDestroy(pSyncMsg); } @@ -336,7 +336,7 @@ static void *syncIOConsumerFunc(void *param) { } else if (pRpcMsg->msgType == TDMT_SYNC_SNAPSHOT_RSP) { if (io->FpOnSyncSnapshotRsp != NULL) { SyncSnapshotRsp *pSyncMsg = syncSnapshotRspFromRpcMsg2(pRpcMsg); - assert(pSyncMsg != NULL); + ASSERT(pSyncMsg != NULL); io->FpOnSyncSnapshotRsp(io->pSyncNode, pSyncMsg); syncSnapshotRspDestroy(pSyncMsg); } diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 18cb55b417411b46e9482409b40fc5dc477513f7..5b432aeec4e214a7c79106934965012d1b162b02 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -20,7 +20,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pSyncNode) { SSyncIndexMgr *pSyncIndexMgr = taosMemoryMalloc(sizeof(SSyncIndexMgr)); - assert(pSyncIndexMgr != NULL); + ASSERT(pSyncIndexMgr != NULL); memset(pSyncIndexMgr, 0, sizeof(SSyncIndexMgr)); pSyncIndexMgr->replicas = &(pSyncNode->replicasId); @@ -63,7 +63,12 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, } // maybe config change - assert(0); + // ASSERT(0); + + char host[128]; + uint16_t port; + syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); + sError("vgId:%d index mgr set for %s:%d, index:%" PRId64 " error", pSyncIndexMgr->pSyncNode->vgId, host, port, index); } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { @@ -73,7 +78,9 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaf return idx; } } - assert(0); + + syncNodeLog3("syncIndexMgrGetIndex", pSyncIndexMgr->pSyncNode); + ASSERT(0); } cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { @@ -119,7 +126,7 @@ cJSON *syncIndexMgr2Json(SSyncIndexMgr *pSyncIndexMgr) { char *syncIndexMgr2Str(SSyncIndexMgr *pSyncIndexMgr) { cJSON *pJson = syncIndexMgr2Json(pSyncIndexMgr); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -162,7 +169,11 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId, S } // maybe config change - assert(0); + // ASSERT(0); + char host[128]; + uint16_t port; + syncUtilU642Addr(pRaftId->addr, host, sizeof(host), &port); + sError("vgId:%d index mgr set for %s:%d, term:%lu error", pSyncIndexMgr->pSyncNode->vgId, host, port, term); } SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftId) { @@ -172,5 +183,5 @@ SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pSyncIndexMgr, const SRaftId *pRaftI return term; } } - assert(0); + ASSERT(0); } \ No newline at end of file diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 33e7a8241f92f2865e5d1e1a209237dac8c54693..732f450f4cfd41fafe1ff72503fbf4f0ac63633b 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -75,7 +75,7 @@ int32_t syncInit() { void syncCleanUp() { int32_t ret = syncEnvStop(); - assert(ret == 0); + ASSERT(ret == 0); if (tsNodeRefId != -1) { taosCloseRef(tsNodeRefId); @@ -85,7 +85,7 @@ void syncCleanUp() { int64_t syncOpen(const SSyncInfo* pSyncInfo) { SSyncNode* pSyncNode = syncNodeOpen(pSyncInfo); - assert(pSyncNode != NULL); + ASSERT(pSyncNode != NULL); if (gRaftDetailLog) { syncNodeLog2("syncNodeOpen open success", pSyncNode); @@ -156,8 +156,12 @@ int32_t syncSetStandby(int64_t rid) { if (pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); - terrno = TSDB_CODE_SYN_IS_LEADER; - sError("failed to set standby since it is not follower, rid:%" PRId64, rid); + if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { + terrno = TSDB_CODE_SYN_IS_LEADER; + } else { + terrno = TSDB_CODE_SYN_STANDBY_NOT_READY; + } + sError("failed to set standby since it is not follower, state:%s rid:%" PRId64, syncStr(pSyncNode->state), rid); return -1; } @@ -314,7 +318,7 @@ bool syncCanLeaderTransfer(int64_t rid) { if (pSyncNode == NULL) { return false; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); if (pSyncNode->replicaNum == 1) { taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -341,7 +345,7 @@ bool syncCanLeaderTransfer(int64_t rid) { return matchOK; } -int32_t syncForwardToPeer(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { +int32_t syncForwardToPeer(int64_t rid, SRpcMsg* pMsg, bool isWeak) { int32_t ret = syncPropose(rid, pMsg, isWeak); return ret; } @@ -351,7 +355,7 @@ ESyncState syncGetMyRole(int64_t rid) { if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); ESyncState state = pSyncNode->state; taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -363,7 +367,7 @@ bool syncIsReady(int64_t rid) { if (pSyncNode == NULL) { return false; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); bool b = (pSyncNode->state == TAOS_SYNC_STATE_LEADER) && pSyncNode->restoreFinish; taosReleaseRef(tsNodeRefId, pSyncNode->rid); return b; @@ -374,7 +378,7 @@ bool syncIsRestoreFinish(int64_t rid) { if (pSyncNode == NULL) { return false; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); bool b = pSyncNode->restoreFinish; taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -386,7 +390,7 @@ int32_t syncGetSnapshotMeta(int64_t rid, struct SSnapshotMeta* sMeta) { if (pSyncNode == NULL) { return -1; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); sMeta->lastConfigIndex = pSyncNode->pRaftCfg->lastConfigIndex; sTrace("vgId:%d, get snapshot meta, lastConfigIndex:%" PRId64, pSyncNode->vgId, pSyncNode->pRaftCfg->lastConfigIndex); @@ -400,7 +404,7 @@ int32_t syncGetSnapshotMetaByIndex(int64_t rid, SyncIndex snapshotIndex, struct if (pSyncNode == NULL) { return -1; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); ASSERT(pSyncNode->pRaftCfg->configIndexCount >= 1); SyncIndex lastIndex = (pSyncNode->pRaftCfg->configIndexArr)[0]; @@ -444,7 +448,7 @@ SyncTerm syncGetMyTerm(int64_t rid) { if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); SyncTerm term = pSyncNode->pRaftStore->currentTerm; taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -456,7 +460,7 @@ SyncGroupId syncGetVgId(int64_t rid) { if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); SyncGroupId vgId = pSyncNode->vgId; taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -469,7 +473,7 @@ void syncGetEpSet(int64_t rid, SEpSet* pEpSet) { memset(pEpSet, 0, sizeof(*pEpSet)); return; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); pEpSet->numOfEps = 0; for (int i = 0; i < pSyncNode->pRaftCfg->cfg.replicaNum; ++i) { snprintf(pEpSet->eps[i].fqdn, sizeof(pEpSet->eps[i].fqdn), "%s", (pSyncNode->pRaftCfg->cfg.nodeInfo)[i].nodeFqdn); @@ -490,7 +494,7 @@ int32_t syncGetRespRpc(int64_t rid, uint64_t index, SRpcMsg* msg) { if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); SRespStub stub; int32_t ret = syncRespMgrGet(pSyncNode->pSyncRespMgr, index, &stub); @@ -507,7 +511,7 @@ int32_t syncGetAndDelRespRpc(int64_t rid, uint64_t index, SRpcHandleInfo* pInfo) if (pSyncNode == NULL) { return TAOS_SYNC_STATE_ERROR; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); SRespStub stub; int32_t ret = syncRespMgrGetAndDel(pSyncNode->pSyncRespMgr, index, &stub); @@ -526,7 +530,7 @@ void syncSetMsgCb(int64_t rid, const SMsgCb* msgcb) { sTrace("syncSetQ get pSyncNode is NULL, rid:%ld", rid); return; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); pSyncNode->msgcb = msgcb; taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -538,7 +542,7 @@ char* sync2SimpleStr(int64_t rid) { sTrace("syncSetRpc get pSyncNode is NULL, rid:%ld", rid); return NULL; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); char* s = syncNode2SimpleStr(pSyncNode); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -550,7 +554,7 @@ void setPingTimerMS(int64_t rid, int32_t pingTimerMS) { if (pSyncNode == NULL) { return; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); pSyncNode->pingBaseLine = pingTimerMS; pSyncNode->pingTimerMS = pingTimerMS; @@ -562,7 +566,7 @@ void setElectTimerMS(int64_t rid, int32_t electTimerMS) { if (pSyncNode == NULL) { return; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); pSyncNode->electBaseLine = electTimerMS; taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -573,14 +577,14 @@ void setHeartbeatTimerMS(int64_t rid, int32_t hbTimerMS) { if (pSyncNode == NULL) { return; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); pSyncNode->hbBaseLine = hbTimerMS; pSyncNode->heartbeatTimerMS = hbTimerMS; taosReleaseRef(tsNodeRefId, pSyncNode->rid); } -int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { +int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak) { int32_t ret = 0; SSyncNode* pSyncNode = taosAcquireRef(tsNodeRefId, rid); @@ -588,7 +592,7 @@ int32_t syncPropose(int64_t rid, const SRpcMsg* pMsg, bool isWeak) { terrno = TSDB_CODE_SYN_INTERNAL_ERROR; return -1; } - assert(rid == pSyncNode->rid); + ASSERT(rid == pSyncNode->rid); ret = syncNodePropose(pSyncNode, pMsg, isWeak); taosReleaseRef(tsNodeRefId, pSyncNode->rid); @@ -658,7 +662,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { SSyncInfo* pSyncInfo = (SSyncInfo*)pOldSyncInfo; SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); - assert(pSyncNode != NULL); + ASSERT(pSyncNode != NULL); memset(pSyncNode, 0, sizeof(SSyncNode)); int32_t ret = 0; @@ -678,12 +682,12 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { meta.snapshotEnable = pSyncInfo->snapshotEnable; meta.lastConfigIndex = SYNC_INDEX_INVALID; ret = raftCfgCreateFile((SSyncCfg*)&(pSyncInfo->syncCfg), meta, pSyncNode->configPath); - assert(ret == 0); + ASSERT(ret == 0); } else { // update syncCfg by raft_config.json pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); - assert(pSyncNode->pRaftCfg != NULL); + ASSERT(pSyncNode->pRaftCfg != NULL); pSyncInfo->syncCfg = pSyncNode->pRaftCfg->cfg; if (gRaftDetailLog) { @@ -708,7 +712,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // init raft config pSyncNode->pRaftCfg = raftCfgOpen(pSyncNode->configPath); - assert(pSyncNode->pRaftCfg != NULL); + ASSERT(pSyncNode->pRaftCfg != NULL); // init internal pSyncNode->myNodeInfo = pSyncNode->pRaftCfg->cfg.nodeInfo[pSyncNode->pRaftCfg->cfg.myIndex]; @@ -767,23 +771,23 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // init TLA+ server vars pSyncNode->state = TAOS_SYNC_STATE_FOLLOWER; pSyncNode->pRaftStore = raftStoreOpen(pSyncNode->raftStorePath); - assert(pSyncNode->pRaftStore != NULL); + ASSERT(pSyncNode->pRaftStore != NULL); // init TLA+ candidate vars pSyncNode->pVotesGranted = voteGrantedCreate(pSyncNode); - assert(pSyncNode->pVotesGranted != NULL); + ASSERT(pSyncNode->pVotesGranted != NULL); pSyncNode->pVotesRespond = votesRespondCreate(pSyncNode); - assert(pSyncNode->pVotesRespond != NULL); + ASSERT(pSyncNode->pVotesRespond != NULL); // init TLA+ leader vars pSyncNode->pNextIndex = syncIndexMgrCreate(pSyncNode); - assert(pSyncNode->pNextIndex != NULL); + ASSERT(pSyncNode->pNextIndex != NULL); pSyncNode->pMatchIndex = syncIndexMgrCreate(pSyncNode); - assert(pSyncNode->pMatchIndex != NULL); + ASSERT(pSyncNode->pMatchIndex != NULL); // init TLA+ log vars pSyncNode->pLogStore = logStoreCreate(pSyncNode); - assert(pSyncNode->pLogStore != NULL); + ASSERT(pSyncNode->pLogStore != NULL); pSyncNode->commitIndex = SYNC_INDEX_INVALID; // timer ms init @@ -841,7 +845,7 @@ SSyncNode* syncNodeOpen(const SSyncInfo* pOldSyncInfo) { // tools pSyncNode->pSyncRespMgr = syncRespMgrCreate(pSyncNode, 0); - assert(pSyncNode->pSyncRespMgr != NULL); + ASSERT(pSyncNode->pSyncRespMgr != NULL); // restore state pSyncNode->restoreFinish = false; @@ -889,7 +893,7 @@ void syncNodeStart(SSyncNode* pSyncNode) { // int32_t ret = 0; // ret = syncNodeStartPingTimer(pSyncNode); - // assert(ret == 0); + // ASSERT(ret == 0); if (gRaftDetailLog) { syncNodeLog2("==state change become leader immediately==", pSyncNode); @@ -911,10 +915,10 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeEventLog(pSyncNode, "sync close"); int32_t ret; - assert(pSyncNode != NULL); + ASSERT(pSyncNode != NULL); ret = raftStoreClose(pSyncNode->pRaftStore); - assert(ret == 0); + ASSERT(ret == 0); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); voteGrantedDestroy(pSyncNode->pVotesGranted); @@ -976,7 +980,7 @@ int32_t syncNodePingSelf(SSyncNode* pSyncNode) { int32_t ret = 0; SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, &pSyncNode->myRaftId, pSyncNode->vgId); ret = syncNodePing(pSyncNode, &pMsg->destId, pMsg); - assert(ret == 0); + ASSERT(ret == 0); syncPingDestroy(pMsg); return ret; @@ -988,7 +992,7 @@ int32_t syncNodePingPeers(SSyncNode* pSyncNode) { SRaftId* destId = &(pSyncNode->peersId[i]); SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); ret = syncNodePing(pSyncNode, destId, pMsg); - assert(ret == 0); + ASSERT(ret == 0); syncPingDestroy(pMsg); } return ret; @@ -1000,7 +1004,7 @@ int32_t syncNodePingAll(SSyncNode* pSyncNode) { SRaftId* destId = &(pSyncNode->replicasId[i]); SyncPing* pMsg = syncPingBuild3(&pSyncNode->myRaftId, destId, pSyncNode->vgId); ret = syncNodePing(pSyncNode, destId, pMsg); - assert(ret == 0); + ASSERT(ret == 0); syncPingDestroy(pMsg); } return ret; @@ -1278,6 +1282,9 @@ cJSON* syncNode2Json(const SSyncNode* pSyncNode) { // snapshot receivers cJSON* pReceivers = cJSON_CreateArray(); cJSON_AddItemToObject(pRoot, "receiver", snapshotReceiver2Json(pSyncNode->pNewNodeReceiver)); + + // changing + cJSON_AddNumberToObject(pRoot, "changing", pSyncNode->changing); } cJSON* pJson = cJSON_CreateObject(); @@ -1300,29 +1307,92 @@ void syncNodeEventLog(const SSyncNode* pSyncNode, char* str) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + char* pCfgStr = syncCfg2SimpleStr(&(pSyncNode->pRaftCfg->cfg)); if (userStrLen < 256) { - char logBuf[128 + 256]; - snprintf(logBuf, sizeof(logBuf), - "vgId:%d, sync %s %s, term:%lu, commit:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, replica-num:%d, " - "lconfig:%ld, changing:%d", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, - pSyncNode->commitIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, - pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing); + char logBuf[256 + 256]; + if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { + snprintf(logBuf, sizeof(logBuf), + "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "replica-num:%d, " + "lconfig:%ld, changing:%d, %s", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, + pSyncNode->changing, pCfgStr); + } else { + snprintf(logBuf, sizeof(logBuf), "%s", str); + } sDebug("%s", logBuf); } else { - int len = 128 + userStrLen; + int len = 256 + userStrLen; char* s = (char*)taosMemoryMalloc(len); - snprintf(s, len, - "vgId:%d, sync %s %s, term:%lu, commit:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, replica-num:%d, " - "lconfig:%ld, changing:%d", - pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, - pSyncNode->commitIndex, logLastIndex, snapshot.lastApplyIndex, pSyncNode->pRaftCfg->isStandBy, - pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, pSyncNode->changing); + if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { + snprintf(s, len, + "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "replica-num:%d, " + "lconfig:%ld, changing:%d, %s", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, + pSyncNode->changing, pCfgStr); + } else { + snprintf(s, len, "%s", str); + } sDebug("%s", s); taosMemoryFree(s); } + + taosMemoryFree(pCfgStr); +} + +void syncNodeErrorLog(const SSyncNode* pSyncNode, char* str) { + int32_t userStrLen = strlen(str); + + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + } + SyncIndex logLastIndex = pSyncNode->pLogStore->syncLogLastIndex(pSyncNode->pLogStore); + SyncIndex logBeginIndex = pSyncNode->pLogStore->syncLogBeginIndex(pSyncNode->pLogStore); + + if (userStrLen < 256) { + char logBuf[128 + 256]; + if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { + snprintf(logBuf, sizeof(logBuf), + "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "replica-num:%d, " + "lconfig:%ld, changing:%d", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, + pSyncNode->changing); + } else { + snprintf(logBuf, sizeof(logBuf), "%s", str); + } + sError("%s", logBuf); + + } else { + int len = 128 + userStrLen; + char* s = (char*)taosMemoryMalloc(len); + if (pSyncNode != NULL && pSyncNode->pRaftCfg != NULL && pSyncNode->pRaftStore != NULL) { + snprintf(s, len, + "vgId:%d, sync %s %s, term:%lu, commit:%ld, beginlog:%ld, lastlog:%ld, lastsnapshot:%ld, standby:%d, " + "replica-num:%d, " + "lconfig:%ld, changing:%d", + pSyncNode->vgId, syncUtilState2String(pSyncNode->state), str, pSyncNode->pRaftStore->currentTerm, + pSyncNode->commitIndex, logBeginIndex, logLastIndex, snapshot.lastApplyIndex, + pSyncNode->pRaftCfg->isStandBy, pSyncNode->replicaNum, pSyncNode->pRaftCfg->lastConfigIndex, + pSyncNode->changing); + } else { + snprintf(s, len, "%s", str); + } + sError("%s", s); + taosMemoryFree(s); + } } char* syncNode2SimpleStr(const SSyncNode* pSyncNode) { @@ -1389,6 +1459,17 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde isAdd = false; } + // log begin config change + do { + char eventLog[256]; + char* pOldCfgStr = syncCfg2SimpleStr(&oldConfig); + char* pNewCfgStr = syncCfg2SimpleStr(pNewConfig); + snprintf(eventLog, sizeof(eventLog), "begin do config change, from %s to %s", pOldCfgStr, pNewCfgStr); + syncNodeEventLog(pSyncNode, eventLog); + taosMemoryFree(pOldCfgStr); + taosMemoryFree(pNewCfgStr); + } while (0); + if (IamInNew) { pSyncNode->pRaftCfg->isStandBy = 0; // change isStandBy to normal } @@ -1396,7 +1477,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde pSyncNode->pRaftCfg->isStandBy = 1; // set standby } - // persist last config index + // add last config index raftCfgAddConfigIndex(pSyncNode->pRaftCfg, lastConfigChangeIndex); if (IamInNew) { @@ -1547,6 +1628,17 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } _END: + + // log end config change + do { + char eventLog[256]; + char* pOldCfgStr = syncCfg2SimpleStr(&oldConfig); + char* pNewCfgStr = syncCfg2SimpleStr(pNewConfig); + snprintf(eventLog, sizeof(eventLog), "end do config change, from %s to %s", pOldCfgStr, pNewCfgStr); + syncNodeEventLog(pSyncNode, eventLog); + taosMemoryFree(pOldCfgStr); + taosMemoryFree(pNewCfgStr); + } while (0); return; } @@ -1690,8 +1782,8 @@ void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr) { } void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); - assert(voteGrantedMajority(pSyncNode->pVotesGranted)); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + ASSERT(voteGrantedMajority(pSyncNode->pVotesGranted)); syncNodeBecomeLeader(pSyncNode, "candidate to leader"); syncNodeLog2("==state change syncNodeCandidate2Leader==", pSyncNode); @@ -1703,21 +1795,21 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { } void syncNodeFollower2Candidate(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER); pSyncNode->state = TAOS_SYNC_STATE_CANDIDATE; syncNodeLog2("==state change syncNodeFollower2Candidate==", pSyncNode); } void syncNodeLeader2Follower(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER); syncNodeBecomeFollower(pSyncNode, "leader to follower"); syncNodeLog2("==state change syncNodeLeader2Follower==", pSyncNode); } void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_CANDIDATE); syncNodeBecomeFollower(pSyncNode, "candidate to follower"); syncNodeLog2("==state change syncNodeCandidate2Follower==", pSyncNode); @@ -1728,8 +1820,8 @@ void syncNodeCandidate2Follower(SSyncNode* pSyncNode) { // just called by syncNodeVoteForSelf // need assert void syncNodeVoteForTerm(SSyncNode* pSyncNode, SyncTerm term, SRaftId* pRaftId) { - assert(term == pSyncNode->pRaftStore->currentTerm); - assert(!raftStoreHasVoted(pSyncNode->pRaftStore)); + ASSERT(term == pSyncNode->pRaftStore->currentTerm); + ASSERT(!raftStoreHasVoted(pSyncNode->pRaftStore)); raftStoreVote(pSyncNode->pRaftStore, pRaftId); } @@ -1752,7 +1844,7 @@ void syncNodeVoteForSelf(SSyncNode* pSyncNode) { // snapshot -------------- bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { bool ret = false; - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); if (snapshot.lastApplyIndex >= SYNC_INDEX_BEGIN) { @@ -1762,6 +1854,7 @@ bool syncNodeHasSnapshot(SSyncNode* pSyncNode) { return ret; } +#if 0 bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { ASSERT(syncNodeHasSnapshot(pSyncNode)); ASSERT(pSyncNode->pFsm->FpGetSnapshotInfo != NULL); @@ -1772,6 +1865,7 @@ bool syncNodeIsIndexInSnapshot(SSyncNode* pSyncNode, SyncIndex index) { bool b = (index <= snapshot.lastApplyIndex); return b; } +#endif SyncIndex syncNodeGetLastIndex(SSyncNode* pSyncNode) { SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; @@ -1788,7 +1882,7 @@ SyncTerm syncNodeGetLastTerm(SSyncNode* pSyncNode) { SyncTerm lastTerm = 0; if (syncNodeHasSnapshot(pSyncNode)) { // has snapshot - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } @@ -1820,19 +1914,112 @@ SyncIndex syncNodeSyncStartIndex(SSyncNode* pSyncNode) { return syncStartIndex; } +SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { + SyncIndex preIndex = index - 1; + if (preIndex < SYNC_INDEX_INVALID) { + preIndex = SYNC_INDEX_INVALID; + } + + return preIndex; +} + +/* SyncIndex syncNodeGetPreIndex(SSyncNode* pSyncNode, SyncIndex index) { ASSERT(index >= SYNC_INDEX_BEGIN); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); - ASSERT(index <= syncStartIndex); + if (index > syncStartIndex) { + syncNodeLog3("syncNodeGetPreIndex", pSyncNode); + ASSERT(0); + } SyncIndex preIndex = index - 1; return preIndex; } +*/ +SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { + if (index < SYNC_INDEX_BEGIN) { + return SYNC_TERM_INVALID; + } + + if (index == SYNC_INDEX_BEGIN) { + return 0; + } + + SyncTerm preTerm = 0; + SyncIndex preIndex = index - 1; + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry); + if (code == 0) { + ASSERT(pPreEntry != NULL); + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); + return preTerm; + } else { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex == preIndex) { + return snapshot.lastApplyTerm; + } + } + } + } + + return SYNC_TERM_INVALID; +} + +#if 0 SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { ASSERT(index >= SYNC_INDEX_BEGIN); + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); - ASSERT(index <= syncStartIndex); + if (index > syncStartIndex) { + syncNodeLog3("syncNodeGetPreTerm", pSyncNode); + ASSERT(0); + } + + if (index == SYNC_INDEX_BEGIN) { + return 0; + } + + SyncTerm preTerm = 0; + SyncIndex preIndex = index - 1; + SSyncRaftEntry* pPreEntry = NULL; + int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, preIndex, &pPreEntry); + if (code == 0) { + ASSERT(pPreEntry != NULL); + preTerm = pPreEntry->term; + taosMemoryFree(pPreEntry); + return preTerm; + } else { + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; + if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { + pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); + if (snapshot.lastApplyIndex == preIndex) { + return snapshot.lastApplyTerm; + } + } + } + } + + ASSERT(0); + return -1; +} +#endif + +#if 0 +SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { + ASSERT(index >= SYNC_INDEX_BEGIN); + + SyncIndex syncStartIndex = syncNodeSyncStartIndex(pSyncNode); + if (index > syncStartIndex) { + syncNodeLog3("syncNodeGetPreTerm", pSyncNode); + ASSERT(0); + } if (index == SYNC_INDEX_BEGIN) { return 0; @@ -1841,7 +2028,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { SyncTerm preTerm = 0; if (syncNodeHasSnapshot(pSyncNode)) { // has snapshot - SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0}; + SSnapshot snapshot = {.data = NULL, .lastApplyIndex = -1, .lastApplyTerm = 0, .lastConfigIndex = -1}; if (pSyncNode->pFsm->FpGetSnapshotInfo != NULL) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); } @@ -1888,6 +2075,7 @@ SyncTerm syncNodeGetPreTerm(SSyncNode* pSyncNode, SyncIndex index) { return preTerm; } +#endif // get pre index and term of "index" int32_t syncNodeGetPreIndexTerm(SSyncNode* pSyncNode, SyncIndex index, SyncIndex* pPreIndex, SyncTerm* pPreTerm) { @@ -1925,6 +2113,12 @@ void syncNodeLog2(char* s, SSyncNode* pObj) { } } +void syncNodeLog3(char* s, SSyncNode* pObj) { + char* serialized = syncNode2Str(pObj); + sTraceLong("syncNodeLog3 | len:%lu | %s | %s", strlen(serialized), s, serialized); + taosMemoryFree(serialized); +} + // ------ local funciton --------- // enqueue message ---- static void syncNodeEqPingTimer(void* param, void* tmrId) { @@ -2036,17 +2230,17 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { static int32_t syncNodeEqNoop(SSyncNode* ths) { int32_t ret = 0; - assert(ths->state == TAOS_SYNC_STATE_LEADER); + ASSERT(ths->state == TAOS_SYNC_STATE_LEADER); - SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); uint32_t entryLen; char* serialized = syncEntrySerialize(pEntry, &entryLen); SyncClientRequest* pSyncMsg = syncClientRequestBuild(entryLen); - assert(pSyncMsg->dataLen == entryLen); + ASSERT(pSyncMsg->dataLen == entryLen); memcpy(pSyncMsg->data, serialized, entryLen); SRpcMsg rpcMsg = {0}; @@ -2066,10 +2260,10 @@ static int32_t syncNodeEqNoop(SSyncNode* ths) { static int32_t syncNodeAppendNoop(SSyncNode* ths) { int32_t ret = 0; - SyncIndex index = ths->pLogStore->getLastIndex(ths->pLogStore) + 1; + SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; SSyncRaftEntry* pEntry = syncEntryBuildNoop(term, index, ths->vgId); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); @@ -2132,7 +2326,7 @@ int32_t syncNodeOnClientRequestCb(SSyncNode* ths, SyncClientRequest* pMsg) { SyncIndex index = ths->pLogStore->syncLogWriteIndex(ths->pLogStore); SyncTerm term = ths->pRaftStore->currentTerm; SSyncRaftEntry* pEntry = syncEntryBuild2((SyncClientRequest*)pMsg, term, index); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); if (ths->state == TAOS_SYNC_STATE_LEADER) { // ths->pLogStore->appendEntry(ths->pLogStore, pEntry); @@ -2290,7 +2484,7 @@ static int32_t syncNodeConfigChangeFinish(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyn ths->pFsm->FpReConfigCb(ths->pFsm, pRpcMsg, cbMeta); } - // update changing + // clear changing ths->changing = false; char tmpbuf[512]; @@ -2309,6 +2503,9 @@ static int32_t syncNodeConfigChangeFinish(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyn static int32_t syncNodeConfigChange(SSyncNode* ths, SRpcMsg* pRpcMsg, SSyncRaftEntry* pEntry, SyncReconfigFinish* pFinish) { + // set changing + ths->changing = true; + // old config SSyncCfg oldSyncCfg = ths->pRaftCfg->cfg; diff --git a/source/libs/sync/src/syncMessage.c b/source/libs/sync/src/syncMessage.c index 454609009c8ecd3dfee235c5ea87ecd2a49c0fe2..119a178893ab66bb3b3edd5fec89389e139fabaf 100644 --- a/source/libs/sync/src/syncMessage.c +++ b/source/libs/sync/src/syncMessage.c @@ -186,18 +186,18 @@ void syncTimeoutDestroy(SyncTimeout* pMsg) { } void syncTimeoutSerialize(const SyncTimeout* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncTimeoutDeserialize(const char* buf, uint32_t len, SyncTimeout* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncTimeoutSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -208,9 +208,9 @@ char* syncTimeoutSerialize2(const SyncTimeout* pMsg, uint32_t* len) { SyncTimeout* syncTimeoutDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncTimeout* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncTimeoutDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -228,7 +228,7 @@ void syncTimeoutFromRpcMsg(const SRpcMsg* pRpcMsg, SyncTimeout* pMsg) { SyncTimeout* syncTimeoutFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncTimeout* pMsg = syncTimeoutDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -322,19 +322,19 @@ void syncPingDestroy(SyncPing* pMsg) { } void syncPingSerialize(const SyncPing* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncPingDeserialize(const char* buf, uint32_t len, SyncPing* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); - assert(pMsg->bytes == sizeof(SyncPing) + pMsg->dataLen); + ASSERT(len == pMsg->bytes); + ASSERT(pMsg->bytes == sizeof(SyncPing) + pMsg->dataLen); } char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncPingSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -345,9 +345,9 @@ char* syncPingSerialize2(const SyncPing* pMsg, uint32_t* len) { SyncPing* syncPingDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncPing* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncPingDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -406,7 +406,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) { } pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); pMsg->bytes = bytes; if (tDecodeI32(&decoder, &pMsg->vgId) < 0) { @@ -435,7 +435,7 @@ SyncPing* syncPingDeserialize3(void* buf, int32_t bufLen) { if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) { return NULL; } - assert(len = pMsg->dataLen); + ASSERT(len = pMsg->dataLen); memcpy(pMsg->data, data, len); tEndDecode(&decoder); @@ -457,7 +457,7 @@ void syncPingFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPing* pMsg) { SyncPing* syncPingFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncPing* pMsg = syncPingDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -584,19 +584,19 @@ void syncPingReplyDestroy(SyncPingReply* pMsg) { } void syncPingReplySerialize(const SyncPingReply* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncPingReplyDeserialize(const char* buf, uint32_t len, SyncPingReply* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); - assert(pMsg->bytes == sizeof(SyncPing) + pMsg->dataLen); + ASSERT(len == pMsg->bytes); + ASSERT(pMsg->bytes == sizeof(SyncPing) + pMsg->dataLen); } char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncPingReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -607,9 +607,9 @@ char* syncPingReplySerialize2(const SyncPingReply* pMsg, uint32_t* len) { SyncPingReply* syncPingReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncPingReply* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncPingReplyDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -668,7 +668,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) { } pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); pMsg->bytes = bytes; if (tDecodeI32(&decoder, &pMsg->vgId) < 0) { @@ -697,7 +697,7 @@ SyncPingReply* syncPingReplyDeserialize3(void* buf, int32_t bufLen) { if (tDecodeBinary(&decoder, (uint8_t**)(&data), &len) < 0) { return NULL; } - assert(len = pMsg->dataLen); + ASSERT(len = pMsg->dataLen); memcpy(pMsg->data, data, len); tEndDecode(&decoder); @@ -719,7 +719,7 @@ void syncPingReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncPingReply* pMsg) { SyncPingReply* syncPingReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncPingReply* pMsg = syncPingReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -844,18 +844,18 @@ void syncClientRequestDestroy(SyncClientRequest* pMsg) { } void syncClientRequestSerialize(const SyncClientRequest* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncClientRequestDeserialize(const char* buf, uint32_t len, SyncClientRequest* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncClientRequestSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -866,9 +866,9 @@ char* syncClientRequestSerialize2(const SyncClientRequest* pMsg, uint32_t* len) SyncClientRequest* syncClientRequestDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncClientRequest* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncClientRequestDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -888,7 +888,7 @@ void syncClientRequestFromRpcMsg(const SRpcMsg* pRpcMsg, SyncClientRequest* pMsg // step 3. RpcMsg => SyncClientRequest, from queue SyncClientRequest* syncClientRequestFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncClientRequest* pMsg = syncClientRequestDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -974,18 +974,18 @@ void syncRequestVoteDestroy(SyncRequestVote* pMsg) { } void syncRequestVoteSerialize(const SyncRequestVote* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncRequestVoteDeserialize(const char* buf, uint32_t len, SyncRequestVote* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncRequestVoteSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -996,9 +996,9 @@ char* syncRequestVoteSerialize2(const SyncRequestVote* pMsg, uint32_t* len) { SyncRequestVote* syncRequestVoteDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncRequestVote* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncRequestVoteDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1016,7 +1016,7 @@ void syncRequestVoteFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVote* pMsg) { SyncRequestVote* syncRequestVoteFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncRequestVote* pMsg = syncRequestVoteDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -1125,18 +1125,18 @@ void syncRequestVoteReplyDestroy(SyncRequestVoteReply* pMsg) { } void syncRequestVoteReplySerialize(const SyncRequestVoteReply* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncRequestVoteReplyDeserialize(const char* buf, uint32_t len, SyncRequestVoteReply* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncRequestVoteReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -1147,9 +1147,9 @@ char* syncRequestVoteReplySerialize2(const SyncRequestVoteReply* pMsg, uint32_t* SyncRequestVoteReply* syncRequestVoteReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncRequestVoteReply* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncRequestVoteReplyDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1167,7 +1167,7 @@ void syncRequestVoteReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncRequestVoteReply SyncRequestVoteReply* syncRequestVoteReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncRequestVoteReply* pMsg = syncRequestVoteReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -1274,19 +1274,19 @@ void syncAppendEntriesDestroy(SyncAppendEntries* pMsg) { } void syncAppendEntriesSerialize(const SyncAppendEntries* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncAppendEntriesDeserialize(const char* buf, uint32_t len, SyncAppendEntries* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); - assert(pMsg->bytes == sizeof(SyncAppendEntries) + pMsg->dataLen); + ASSERT(len == pMsg->bytes); + ASSERT(pMsg->bytes == sizeof(SyncAppendEntries) + pMsg->dataLen); } char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncAppendEntriesSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -1297,9 +1297,9 @@ char* syncAppendEntriesSerialize2(const SyncAppendEntries* pMsg, uint32_t* len) SyncAppendEntries* syncAppendEntriesDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncAppendEntries* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncAppendEntriesDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1317,7 +1317,7 @@ void syncAppendEntriesFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntries* pMsg SyncAppendEntries* syncAppendEntriesFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncAppendEntries* pMsg = syncAppendEntriesDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -1444,18 +1444,18 @@ void syncAppendEntriesReplyDestroy(SyncAppendEntriesReply* pMsg) { } void syncAppendEntriesReplySerialize(const SyncAppendEntriesReply* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncAppendEntriesReplyDeserialize(const char* buf, uint32_t len, SyncAppendEntriesReply* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncAppendEntriesReplySerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -1466,9 +1466,9 @@ char* syncAppendEntriesReplySerialize2(const SyncAppendEntriesReply* pMsg, uint3 SyncAppendEntriesReply* syncAppendEntriesReplyDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncAppendEntriesReply* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncAppendEntriesReplyDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1486,7 +1486,7 @@ void syncAppendEntriesReplyFromRpcMsg(const SRpcMsg* pRpcMsg, SyncAppendEntriesR SyncAppendEntriesReply* syncAppendEntriesReplyFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncAppendEntriesReply* pMsg = syncAppendEntriesReplyDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -1607,18 +1607,18 @@ void syncApplyMsgDestroy(SyncApplyMsg* pMsg) { } void syncApplyMsgSerialize(const SyncApplyMsg* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncApplyMsgDeserialize(const char* buf, uint32_t len, SyncApplyMsg* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncApplyMsgSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -1629,9 +1629,9 @@ char* syncApplyMsgSerialize2(const SyncApplyMsg* pMsg, uint32_t* len) { SyncApplyMsg* syncApplyMsgDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncApplyMsg* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncApplyMsgDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1752,19 +1752,19 @@ void syncSnapshotSendDestroy(SyncSnapshotSend* pMsg) { } void syncSnapshotSendSerialize(const SyncSnapshotSend* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncSnapshotSendDeserialize(const char* buf, uint32_t len, SyncSnapshotSend* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); - assert(pMsg->bytes == sizeof(SyncSnapshotSend) + pMsg->dataLen); + ASSERT(len == pMsg->bytes); + ASSERT(pMsg->bytes == sizeof(SyncSnapshotSend) + pMsg->dataLen); } char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncSnapshotSendSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -1775,9 +1775,9 @@ char* syncSnapshotSendSerialize2(const SyncSnapshotSend* pMsg, uint32_t* len) { SyncSnapshotSend* syncSnapshotSendDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncSnapshotSend* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncSnapshotSendDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1795,7 +1795,7 @@ void syncSnapshotSendFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotSend* pMsg) SyncSnapshotSend* syncSnapshotSendFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncSnapshotSend* pMsg = syncSnapshotSendDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -1925,18 +1925,18 @@ void syncSnapshotRspDestroy(SyncSnapshotRsp* pMsg) { } void syncSnapshotRspSerialize(const SyncSnapshotRsp* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncSnapshotRspDeserialize(const char* buf, uint32_t len, SyncSnapshotRsp* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncSnapshotRspSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -1947,9 +1947,9 @@ char* syncSnapshotRspSerialize2(const SyncSnapshotRsp* pMsg, uint32_t* len) { SyncSnapshotRsp* syncSnapshotRspDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncSnapshotRsp* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncSnapshotRspDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -1967,7 +1967,7 @@ void syncSnapshotRspFromRpcMsg(const SRpcMsg* pRpcMsg, SyncSnapshotRsp* pMsg) { SyncSnapshotRsp* syncSnapshotRspFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncSnapshotRsp* pMsg = syncSnapshotRspDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -2085,18 +2085,18 @@ void syncLeaderTransferDestroy(SyncLeaderTransfer* pMsg) { } void syncLeaderTransferSerialize(const SyncLeaderTransfer* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncLeaderTransferDeserialize(const char* buf, uint32_t len, SyncLeaderTransfer* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncLeaderTransferSerialize2(const SyncLeaderTransfer* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncLeaderTransferSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -2107,9 +2107,9 @@ char* syncLeaderTransferSerialize2(const SyncLeaderTransfer* pMsg, uint32_t* len SyncLeaderTransfer* syncLeaderTransferDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncLeaderTransfer* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncLeaderTransferDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -2127,7 +2127,7 @@ void syncLeaderTransferFromRpcMsg(const SRpcMsg* pRpcMsg, SyncLeaderTransfer* pM SyncLeaderTransfer* syncLeaderTransferFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncLeaderTransfer* pMsg = syncLeaderTransferDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } @@ -2247,18 +2247,18 @@ void syncReconfigFinishDestroy(SyncReconfigFinish* pMsg) { } void syncReconfigFinishSerialize(const SyncReconfigFinish* pMsg, char* buf, uint32_t bufLen) { - assert(pMsg->bytes <= bufLen); + ASSERT(pMsg->bytes <= bufLen); memcpy(buf, pMsg, pMsg->bytes); } void syncReconfigFinishDeserialize(const char* buf, uint32_t len, SyncReconfigFinish* pMsg) { memcpy(pMsg, buf, len); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); } char* syncReconfigFinishSerialize2(const SyncReconfigFinish* pMsg, uint32_t* len) { char* buf = taosMemoryMalloc(pMsg->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); syncReconfigFinishSerialize(pMsg, buf, pMsg->bytes); if (len != NULL) { *len = pMsg->bytes; @@ -2269,9 +2269,9 @@ char* syncReconfigFinishSerialize2(const SyncReconfigFinish* pMsg, uint32_t* len SyncReconfigFinish* syncReconfigFinishDeserialize2(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SyncReconfigFinish* pMsg = taosMemoryMalloc(bytes); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); syncReconfigFinishDeserialize(buf, len, pMsg); - assert(len == pMsg->bytes); + ASSERT(len == pMsg->bytes); return pMsg; } @@ -2289,7 +2289,7 @@ void syncReconfigFinishFromRpcMsg(const SRpcMsg* pRpcMsg, SyncReconfigFinish* pM SyncReconfigFinish* syncReconfigFinishFromRpcMsg2(const SRpcMsg* pRpcMsg) { SyncReconfigFinish* pMsg = syncReconfigFinishDeserialize2(pRpcMsg->pCont, pRpcMsg->contLen); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); return pMsg; } diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index 08c3e0126c143acb4196d9196ed8ee09f6678f43..2cc1eb023991e1b31d4b803fa5845fc284147054 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -24,42 +24,47 @@ SRaftCfg *raftCfgOpen(const char *path) { snprintf(pCfg->path, sizeof(pCfg->path), "%s", path); pCfg->pFile = taosOpenFile(pCfg->path, TD_FILE_READ | TD_FILE_WRITE); - assert(pCfg->pFile != NULL); + ASSERT(pCfg->pFile != NULL); taosLSeekFile(pCfg->pFile, 0, SEEK_SET); char buf[1024] = {0}; int len = taosReadFile(pCfg->pFile, buf, sizeof(buf)); - assert(len > 0); + ASSERT(len > 0); int32_t ret = raftCfgFromStr(buf, pCfg); - assert(ret == 0); + ASSERT(ret == 0); return pCfg; } int32_t raftCfgClose(SRaftCfg *pRaftCfg) { int64_t ret = taosCloseFile(&(pRaftCfg->pFile)); - assert(ret == 0); + ASSERT(ret == 0); taosMemoryFree(pRaftCfg); return 0; } int32_t raftCfgPersist(SRaftCfg *pRaftCfg) { - assert(pRaftCfg != NULL); + ASSERT(pRaftCfg != NULL); char *s = raftCfg2Str(pRaftCfg); taosLSeekFile(pRaftCfg->pFile, 0, SEEK_SET); char buf[CONFIG_FILE_LEN] = {0}; memset(buf, 0, sizeof(buf)); - ASSERT(strlen(s) + 1 <= CONFIG_FILE_LEN); + + if (strlen(s) + 1 > CONFIG_FILE_LEN) { + sError("too long config str:%s", s); + ASSERT(0); + } + snprintf(buf, sizeof(buf), "%s", s); int64_t ret = taosWriteFile(pRaftCfg->pFile, buf, sizeof(buf)); - assert(ret == sizeof(buf)); + ASSERT(ret == sizeof(buf)); // int64_t ret = taosWriteFile(pRaftCfg->pFile, s, strlen(s) + 1); - // assert(ret == strlen(s) + 1); + // ASSERT(ret == strlen(s) + 1); taosMemoryFree(s); taosFsyncFile(pRaftCfg->pFile); @@ -96,14 +101,14 @@ cJSON *syncCfg2Json(SSyncCfg *pSyncCfg) { char *syncCfg2Str(SSyncCfg *pSyncCfg) { cJSON *pJson = syncCfg2Json(pSyncCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } char *syncCfg2SimpleStr(SSyncCfg *pSyncCfg) { int32_t len = 512; - char * s = taosMemoryMalloc(len); + char *s = taosMemoryMalloc(len); memset(s, 0, len); snprintf(s, len, "{replica-num:%d, my-index:%d, ", pSyncCfg->replicaNum, pSyncCfg->myIndex); @@ -130,27 +135,27 @@ int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { const cJSON *pJson = pRoot; cJSON *pReplicaNum = cJSON_GetObjectItem(pJson, "replicaNum"); - assert(cJSON_IsNumber(pReplicaNum)); + ASSERT(cJSON_IsNumber(pReplicaNum)); pSyncCfg->replicaNum = cJSON_GetNumberValue(pReplicaNum); cJSON *pMyIndex = cJSON_GetObjectItem(pJson, "myIndex"); - assert(cJSON_IsNumber(pMyIndex)); + ASSERT(cJSON_IsNumber(pMyIndex)); pSyncCfg->myIndex = cJSON_GetNumberValue(pMyIndex); cJSON *pNodeInfoArr = cJSON_GetObjectItem(pJson, "nodeInfo"); int arraySize = cJSON_GetArraySize(pNodeInfoArr); - assert(arraySize == pSyncCfg->replicaNum); + ASSERT(arraySize == pSyncCfg->replicaNum); for (int i = 0; i < arraySize; ++i) { cJSON *pNodeInfo = cJSON_GetArrayItem(pNodeInfoArr, i); - assert(pNodeInfo != NULL); + ASSERT(pNodeInfo != NULL); cJSON *pNodePort = cJSON_GetObjectItem(pNodeInfo, "nodePort"); - assert(cJSON_IsNumber(pNodePort)); + ASSERT(cJSON_IsNumber(pNodePort)); ((pSyncCfg->nodeInfo)[i]).nodePort = cJSON_GetNumberValue(pNodePort); cJSON *pNodeFqdn = cJSON_GetObjectItem(pNodeInfo, "nodeFqdn"); - assert(cJSON_IsString(pNodeFqdn)); + ASSERT(cJSON_IsString(pNodeFqdn)); snprintf(((pSyncCfg->nodeInfo)[i]).nodeFqdn, sizeof(((pSyncCfg->nodeInfo)[i]).nodeFqdn), "%s", pNodeFqdn->valuestring); } @@ -160,10 +165,10 @@ int32_t syncCfgFromJson(const cJSON *pRoot, SSyncCfg *pSyncCfg) { int32_t syncCfgFromStr(const char *s, SSyncCfg *pSyncCfg) { cJSON *pRoot = cJSON_Parse(s); - assert(pRoot != NULL); + ASSERT(pRoot != NULL); int32_t ret = syncCfgFromJson(pRoot, pSyncCfg); - assert(ret == 0); + ASSERT(ret == 0); cJSON_Delete(pRoot); return 0; @@ -196,16 +201,16 @@ cJSON *raftCfg2Json(SRaftCfg *pRaftCfg) { char *raftCfg2Str(SRaftCfg *pRaftCfg) { cJSON *pJson = raftCfg2Json(pRaftCfg); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { - assert(pCfg != NULL); + ASSERT(pCfg != NULL); TdFilePtr pFile = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE); - assert(pFile != NULL); + ASSERT(pFile != NULL); SRaftCfg raftCfg; raftCfg.cfg = *pCfg; @@ -222,10 +227,10 @@ int32_t raftCfgCreateFile(SSyncCfg *pCfg, SRaftCfgMeta meta, const char *path) { ASSERT(strlen(s) + 1 <= CONFIG_FILE_LEN); snprintf(buf, sizeof(buf), "%s", s); int64_t ret = taosWriteFile(pFile, buf, sizeof(buf)); - assert(ret == sizeof(buf)); + ASSERT(ret == sizeof(buf)); // int64_t ret = taosWriteFile(pFile, s, strlen(s) + 1); - // assert(ret == strlen(s) + 1); + // ASSERT(ret == strlen(s) + 1); taosMemoryFree(s); taosCloseFile(&pFile); @@ -250,19 +255,19 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { cJSON *pIndexArr = cJSON_GetObjectItem(pJson, "configIndexArr"); int arraySize = cJSON_GetArraySize(pIndexArr); - assert(arraySize == pRaftCfg->configIndexCount); + ASSERT(arraySize == pRaftCfg->configIndexCount); memset(pRaftCfg->configIndexArr, 0, sizeof(pRaftCfg->configIndexArr)); for (int i = 0; i < arraySize; ++i) { cJSON *pIndexObj = cJSON_GetArrayItem(pIndexArr, i); - assert(pIndexObj != NULL); + ASSERT(pIndexObj != NULL); cJSON *pIndex = cJSON_GetObjectItem(pIndexObj, "index"); - assert(cJSON_IsString(pIndex)); + ASSERT(cJSON_IsString(pIndex)); (pRaftCfg->configIndexArr)[i] = atoll(pIndex->valuestring); } - cJSON * pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); + cJSON *pJsonSyncCfg = cJSON_GetObjectItem(pJson, "SSyncCfg"); int32_t code = syncCfgFromJson(pJsonSyncCfg, &(pRaftCfg->cfg)); ASSERT(code == 0); @@ -271,10 +276,10 @@ int32_t raftCfgFromJson(const cJSON *pRoot, SRaftCfg *pRaftCfg) { int32_t raftCfgFromStr(const char *s, SRaftCfg *pRaftCfg) { cJSON *pRoot = cJSON_Parse(s); - assert(pRoot != NULL); + ASSERT(pRoot != NULL); int32_t ret = raftCfgFromJson(pRoot, pRaftCfg); - assert(ret == 0); + ASSERT(ret == 0); cJSON_Delete(pRoot); return 0; diff --git a/source/libs/sync/src/syncRaftEntry.c b/source/libs/sync/src/syncRaftEntry.c index 05a2dbaa3f36b00cd9202844d65f97f022ba22ae..ff334a76bb67811cc99d465415f15a60b69d54d6 100644 --- a/source/libs/sync/src/syncRaftEntry.c +++ b/source/libs/sync/src/syncRaftEntry.c @@ -19,7 +19,7 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { uint32_t bytes = sizeof(SSyncRaftEntry) + dataLen; SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); memset(pEntry, 0, bytes); pEntry->bytes = bytes; pEntry->dataLen = dataLen; @@ -29,14 +29,14 @@ SSyncRaftEntry* syncEntryBuild(uint32_t dataLen) { // step 4. SyncClientRequest => SSyncRaftEntry, add term, index SSyncRaftEntry* syncEntryBuild2(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild3(pMsg, term, index); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); return pEntry; } SSyncRaftEntry* syncEntryBuild3(SyncClientRequest* pMsg, SyncTerm term, SyncIndex index) { SSyncRaftEntry* pEntry = syncEntryBuild(pMsg->dataLen); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); pEntry->msgType = pMsg->msgType; pEntry->originalRpcType = pMsg->originalRpcType; @@ -63,7 +63,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) memcpy(rpcMsg.pCont, &head, sizeof(head)); SSyncRaftEntry* pEntry = syncEntryBuild(rpcMsg.contLen); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; pEntry->originalRpcType = TDMT_SYNC_NOOP; @@ -72,7 +72,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) pEntry->term = term; pEntry->index = index; - assert(pEntry->dataLen == rpcMsg.contLen); + ASSERT(pEntry->dataLen == rpcMsg.contLen); memcpy(pEntry->data, rpcMsg.pCont, rpcMsg.contLen); rpcFreeCont(rpcMsg.pCont); @@ -88,7 +88,7 @@ void syncEntryDestory(SSyncRaftEntry* pEntry) { // step 5. SSyncRaftEntry => bin, to raft log char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { char* buf = taosMemoryMalloc(pEntry->bytes); - assert(buf != NULL); + ASSERT(buf != NULL); memcpy(buf, pEntry, pEntry->bytes); if (len != NULL) { *len = pEntry->bytes; @@ -100,9 +100,9 @@ char* syncEntrySerialize(const SSyncRaftEntry* pEntry, uint32_t* len) { SSyncRaftEntry* syncEntryDeserialize(const char* buf, uint32_t len) { uint32_t bytes = *((uint32_t*)buf); SSyncRaftEntry* pEntry = taosMemoryMalloc(bytes); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); memcpy(pEntry, buf, len); - assert(len == pEntry->bytes); + ASSERT(len == pEntry->bytes); return pEntry; } diff --git a/source/libs/sync/src/syncRaftLog.c b/source/libs/sync/src/syncRaftLog.c index 79d9b329c10bf8e2fc6db6ee7865bd31fbc764ae..701b4a7b93bd6c9db05df9152339fa8f393ca95f 100644 --- a/source/libs/sync/src/syncRaftLog.c +++ b/source/libs/sync/src/syncRaftLog.c @@ -25,7 +25,7 @@ static SyncIndex raftLogEndIndex(struct SSyncLogStore* pLogStore); static SyncIndex raftLogWriteIndex(struct SSyncLogStore* pLogStore); static bool raftLogIsEmpty(struct SSyncLogStore* pLogStore); static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore); -static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index); + static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore); static SyncTerm raftLogLastTerm(struct SSyncLogStore* pLogStore); static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry); @@ -58,8 +58,6 @@ static int32_t raftLogSetBeginIndex(struct SSyncLogStore* pLogStore, SyncIndex b return 0; } -int32_t raftLogResetBeginIndex(struct SSyncLogStore* pLogStore) { return 0; } - static SyncIndex raftLogBeginIndex(struct SSyncLogStore* pLogStore) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -81,6 +79,7 @@ static int32_t raftLogEntryCount(struct SSyncLogStore* pLogStore) { return count > 0 ? count : 0; } +#if 0 static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) { SyncIndex beginIndex = raftLogBeginIndex(pLogStore); SyncIndex endIndex = raftLogEndIndex(pLogStore); @@ -90,6 +89,7 @@ static bool raftLogInRange(struct SSyncLogStore* pLogStore, SyncIndex index) { return false; } } +#endif static SyncIndex raftLogLastIndex(struct SSyncLogStore* pLogStore) { SyncIndex lastIndex; @@ -143,7 +143,11 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr SWal* pWal = pData->pWal; SyncIndex writeIndex = raftLogWriteIndex(pLogStore); - ASSERT(pEntry->index == writeIndex); + if (pEntry->index != writeIndex) { + sError("vgId:%d wal write index error, entry-index:%ld update to %ld", pData->pSyncNode->vgId, pEntry->index, + writeIndex); + pEntry->index = writeIndex; + } int code = 0; SSyncLogMeta syncMeta; @@ -154,10 +158,10 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("raftLogAppendEntry error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, - linuxErrMsg); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, + pEntry->index, err, err, errStr, sysErr, sysErrStr); ASSERT(0); } @@ -171,6 +175,7 @@ static int32_t raftLogAppendEntry(struct SSyncLogStore* pLogStore, SSyncRaftEntr return code; } +#if 0 static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; @@ -215,6 +220,53 @@ static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, return code; } +#endif + +static int32_t raftLogGetEntry(struct SSyncLogStore* pLogStore, SyncIndex index, SSyncRaftEntry** ppEntry) { + SSyncLogStoreData* pData = pLogStore->data; + SWal* pWal = pData->pWal; + int32_t code; + + *ppEntry = NULL; + + SWalReadHandle* pWalHandle = walOpenReadHandle(pWal); + if (pWalHandle == NULL) { + return -1; + } + + code = walReadWithHandle(pWalHandle, index); + if (code != 0) { + int32_t err = terrno; + const char* errStr = tstrerror(err); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, index, + err, err, errStr, sysErr, sysErrStr); + + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + + return code; + } + + *ppEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); + ASSERT(*ppEntry != NULL); + (*ppEntry)->msgType = TDMT_SYNC_CLIENT_REQUEST; + (*ppEntry)->originalRpcType = pWalHandle->pHead->head.msgType; + (*ppEntry)->seqNum = pWalHandle->pHead->head.syncMeta.seqNum; + (*ppEntry)->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; + (*ppEntry)->term = pWalHandle->pHead->head.syncMeta.term; + (*ppEntry)->index = index; + ASSERT((*ppEntry)->dataLen == pWalHandle->pHead->head.bodyLen); + memcpy((*ppEntry)->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); + + int32_t saveErr = terrno; + walCloseReadHandle(pWalHandle); + terrno = saveErr; + + return code; +} static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; @@ -223,10 +275,11 @@ static int32_t raftLogTruncate(struct SSyncLogStore* pLogStore, SyncIndex fromIn if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("raftLogTruncate error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, - linuxErrMsg); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal truncate error, from-index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); + ASSERT(0); } return code; @@ -245,10 +298,10 @@ static int32_t raftLogGetLastEntry(SSyncLogStore* pLogStore, SSyncRaftEntry** pp //------------------------------- SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { SSyncLogStore* pLogStore = taosMemoryMalloc(sizeof(SSyncLogStore)); - assert(pLogStore != NULL); + ASSERT(pLogStore != NULL); pLogStore->data = taosMemoryMalloc(sizeof(SSyncLogStoreData)); - assert(pLogStore->data != NULL); + ASSERT(pLogStore->data != NULL); SSyncLogStoreData* pData = pLogStore->data; pData->pSyncNode = pSyncNode; @@ -277,7 +330,6 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->syncLogEndIndex = raftLogEndIndex; pLogStore->syncLogIsEmpty = raftLogIsEmpty; pLogStore->syncLogEntryCount = raftLogEntryCount; - pLogStore->syncLogInRange = raftLogInRange; pLogStore->syncLogLastIndex = raftLogLastIndex; pLogStore->syncLogLastTerm = raftLogLastTerm; pLogStore->syncLogAppendEntry = raftLogAppendEntry; @@ -285,6 +337,8 @@ SSyncLogStore* logStoreCreate(SSyncNode* pSyncNode) { pLogStore->syncLogTruncate = raftLogTruncate; pLogStore->syncLogWriteIndex = raftLogWriteIndex; + // pLogStore->syncLogInRange = raftLogInRange; + return pLogStore; } @@ -301,7 +355,7 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { SWal* pWal = pData->pWal; SyncIndex lastIndex = logStoreLastIndex(pLogStore); - assert(pEntry->index == lastIndex + 1); + ASSERT(pEntry->index == lastIndex + 1); int code = 0; SSyncLogMeta syncMeta; @@ -312,10 +366,11 @@ int32_t logStoreAppendEntry(SSyncLogStore* pLogStore, SSyncRaftEntry* pEntry) { if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("walWriteWithSyncInfo error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, - linuxErrMsg); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal write error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, + pEntry->index, err, err, errStr, sysErr, sysErrStr); + ASSERT(0); } @@ -341,16 +396,17 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("walReadWithHandle error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, - linuxErrMsg); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal read error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", pData->pSyncNode->vgId, + index, err, err, errStr, sysErr, sysErrStr); + ASSERT(0); } - // assert(walReadWithHandle(pWalHandle, index) == 0); + // ASSERT(walReadWithHandle(pWalHandle, index) == 0); SSyncRaftEntry* pEntry = syncEntryBuild(pWalHandle->pHead->head.bodyLen); - assert(pEntry != NULL); + ASSERT(pEntry != NULL); pEntry->msgType = TDMT_SYNC_CLIENT_REQUEST; pEntry->originalRpcType = pWalHandle->pHead->head.msgType; @@ -358,11 +414,13 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { pEntry->isWeak = pWalHandle->pHead->head.syncMeta.isWeek; pEntry->term = pWalHandle->pHead->head.syncMeta.term; pEntry->index = index; - assert(pEntry->dataLen == pWalHandle->pHead->head.bodyLen); + ASSERT(pEntry->dataLen == pWalHandle->pHead->head.bodyLen); memcpy(pEntry->data, pWalHandle->pHead->head.body, pWalHandle->pHead->head.bodyLen); - // need to hold, do not new every time!! + int32_t saveErr = terrno; walCloseReadHandle(pWalHandle); + terrno = saveErr; + return pEntry; } else { @@ -373,15 +431,16 @@ SSyncRaftEntry* logStoreGetEntry(SSyncLogStore* pLogStore, SyncIndex index) { int32_t logStoreTruncate(SSyncLogStore* pLogStore, SyncIndex fromIndex) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - // assert(walRollback(pWal, fromIndex) == 0); + // ASSERT(walRollback(pWal, fromIndex) == 0); int32_t code = walRollback(pWal, fromIndex); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("walRollback error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, - linuxErrMsg); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal truncate error, from-index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, fromIndex, err, err, errStr, sysErr, sysErrStr); + ASSERT(0); } return 0; @@ -407,14 +466,16 @@ SyncTerm logStoreLastTerm(SSyncLogStore* pLogStore) { int32_t logStoreUpdateCommitIndex(SSyncLogStore* pLogStore, SyncIndex index) { SSyncLogStoreData* pData = pLogStore->data; SWal* pWal = pData->pWal; - // assert(walCommit(pWal, index) == 0); + // ASSERT(walCommit(pWal, index) == 0); int32_t code = walCommit(pWal, index); if (code != 0) { int32_t err = terrno; const char* errStr = tstrerror(err); - int32_t linuxErr = errno; - const char* linuxErrMsg = strerror(errno); - sError("walCommit error, err:%d %X, msg:%s, linuxErr:%d, linuxErrMsg:%s", err, err, errStr, linuxErr, linuxErrMsg); + int32_t sysErr = errno; + const char* sysErrStr = strerror(errno); + sError("vgId:%d wal update commit index error, index:%ld, err:%d %X, msg:%s, syserr:%d, sysmsg:%s", + pData->pSyncNode->vgId, index, err, err, errStr, sysErr, sysErrStr); + ASSERT(0); } return 0; diff --git a/source/libs/sync/src/syncRaftStore.c b/source/libs/sync/src/syncRaftStore.c index 52e815292607d69e7d364f6a11c31c184f07914a..a1ab95c00fc3d07b57b86fca79b19c8a1048aef3 100644 --- a/source/libs/sync/src/syncRaftStore.c +++ b/source/libs/sync/src/syncRaftStore.c @@ -39,40 +39,40 @@ SRaftStore *raftStoreOpen(const char *path) { if (!raftStoreFileExist(pRaftStore->path)) { ret = raftStoreInit(pRaftStore); - assert(ret == 0); + ASSERT(ret == 0); } pRaftStore->pFile = taosOpenFile(path, TD_FILE_READ | TD_FILE_WRITE); - assert(pRaftStore->pFile != NULL); + ASSERT(pRaftStore->pFile != NULL); int len = taosReadFile(pRaftStore->pFile, storeBuf, RAFT_STORE_BLOCK_SIZE); - assert(len == RAFT_STORE_BLOCK_SIZE); + ASSERT(len == RAFT_STORE_BLOCK_SIZE); ret = raftStoreDeserialize(pRaftStore, storeBuf, len); - assert(ret == 0); + ASSERT(ret == 0); return pRaftStore; } static int32_t raftStoreInit(SRaftStore *pRaftStore) { - assert(pRaftStore != NULL); + ASSERT(pRaftStore != NULL); pRaftStore->pFile = taosOpenFile(pRaftStore->path, TD_FILE_CREATE | TD_FILE_WRITE); - assert(pRaftStore->pFile != NULL); + ASSERT(pRaftStore->pFile != NULL); pRaftStore->currentTerm = 0; pRaftStore->voteFor.addr = 0; pRaftStore->voteFor.vgId = 0; int32_t ret = raftStorePersist(pRaftStore); - assert(ret == 0); + ASSERT(ret == 0); taosCloseFile(&pRaftStore->pFile); return 0; } int32_t raftStoreClose(SRaftStore *pRaftStore) { - assert(pRaftStore != NULL); + ASSERT(pRaftStore != NULL); taosCloseFile(&pRaftStore->pFile); taosMemoryFree(pRaftStore); @@ -81,17 +81,17 @@ int32_t raftStoreClose(SRaftStore *pRaftStore) { } int32_t raftStorePersist(SRaftStore *pRaftStore) { - assert(pRaftStore != NULL); + ASSERT(pRaftStore != NULL); int32_t ret; char storeBuf[RAFT_STORE_BLOCK_SIZE] = {0}; ret = raftStoreSerialize(pRaftStore, storeBuf, sizeof(storeBuf)); - assert(ret == 0); + ASSERT(ret == 0); taosLSeekFile(pRaftStore->pFile, 0, SEEK_SET); ret = taosWriteFile(pRaftStore->pFile, storeBuf, sizeof(storeBuf)); - assert(ret == RAFT_STORE_BLOCK_SIZE); + ASSERT(ret == RAFT_STORE_BLOCK_SIZE); taosFsyncFile(pRaftStore->pFile); return 0; @@ -103,7 +103,7 @@ static bool raftStoreFileExist(char *path) { } int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { - assert(pRaftStore != NULL); + ASSERT(pRaftStore != NULL); cJSON *pRoot = cJSON_CreateObject(); @@ -125,7 +125,7 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { char *serialized = cJSON_Print(pRoot); int len2 = strlen(serialized); - assert(len2 < len); + ASSERT(len2 < len); memset(buf, 0, len); snprintf(buf, len, "%s", serialized); taosMemoryFree(serialized); @@ -135,17 +135,17 @@ int32_t raftStoreSerialize(SRaftStore *pRaftStore, char *buf, size_t len) { } int32_t raftStoreDeserialize(SRaftStore *pRaftStore, char *buf, size_t len) { - assert(pRaftStore != NULL); + ASSERT(pRaftStore != NULL); - assert(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); + ASSERT(len > 0 && len <= RAFT_STORE_BLOCK_SIZE); cJSON *pRoot = cJSON_Parse(buf); cJSON *pCurrentTerm = cJSON_GetObjectItem(pRoot, "current_term"); - assert(cJSON_IsString(pCurrentTerm)); + ASSERT(cJSON_IsString(pCurrentTerm)); sscanf(pCurrentTerm->valuestring, "%lu", &(pRaftStore->currentTerm)); cJSON *pVoteForAddr = cJSON_GetObjectItem(pRoot, "vote_for_addr"); - assert(cJSON_IsString(pVoteForAddr)); + ASSERT(cJSON_IsString(pVoteForAddr)); sscanf(pVoteForAddr->valuestring, "%lu", &(pRaftStore->voteFor.addr)); cJSON *pVoteForVgid = cJSON_GetObjectItem(pRoot, "vote_for_vgid"); @@ -161,7 +161,7 @@ bool raftStoreHasVoted(SRaftStore *pRaftStore) { } void raftStoreVote(SRaftStore *pRaftStore, SRaftId *pRaftId) { - assert(!syncUtilEmptyId(pRaftId)); + ASSERT(!syncUtilEmptyId(pRaftId)); pRaftStore->voteFor = *pRaftId; raftStorePersist(pRaftStore); } @@ -216,7 +216,7 @@ cJSON *raftStore2Json(SRaftStore *pRaftStore) { char *raftStore2Str(SRaftStore *pRaftStore) { cJSON *pJson = raftStore2Json(pRaftStore); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index f044ae5733aa7714af208fe9c4208859206ac35e..c378926e286761ef17508840d198e13eca601792 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -49,7 +49,7 @@ // /\ UNCHANGED <> // int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { - assert(pSyncNode->state == TAOS_SYNC_STATE_LEADER); + ASSERT(pSyncNode->state == TAOS_SYNC_STATE_LEADER); syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pNextIndex", pSyncNode->pNextIndex); syncIndexMgrLog2("==syncNodeAppendEntriesPeers== pMatchIndex", pSyncNode->pMatchIndex); @@ -68,7 +68,7 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { SyncTerm preLogTerm = 0; if (preLogIndex >= SYNC_INDEX_BEGIN) { SSyncRaftEntry* pPreEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, preLogIndex); - assert(pPreEntry != NULL); + ASSERT(pPreEntry != NULL); preLogTerm = pPreEntry->term; syncEntryDestory(pPreEntry); @@ -81,12 +81,12 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { SSyncRaftEntry* pEntry = pSyncNode->pLogStore->getEntry(pSyncNode->pLogStore, nextIndex); if (pEntry != NULL) { pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); // add pEntry into msg uint32_t len; char* serialized = syncEntrySerialize(pEntry, &len); - assert(len == pEntry->bytes); + ASSERT(len == pEntry->bytes); memcpy(pMsg->data, serialized, len); taosMemoryFree(serialized); @@ -95,10 +95,10 @@ int32_t syncNodeAppendEntriesPeers(SSyncNode* pSyncNode) { } else { // maybe overflow, send empty record pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); - assert(pMsg != NULL); + ASSERT(pMsg != NULL); } - assert(pMsg != NULL); + ASSERT(pMsg != NULL); pMsg->srcId = pSyncNode->myRaftId; pMsg->destId = *pDestId; pMsg->term = pSyncNode->pRaftStore->currentTerm; @@ -139,6 +139,15 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { // pre index, pre term SyncIndex preLogIndex = syncNodeGetPreIndex(pSyncNode, nextIndex); SyncTerm preLogTerm = syncNodeGetPreTerm(pSyncNode, nextIndex); + if (preLogTerm == SYNC_TERM_INVALID) { + SyncIndex newNextIndex = syncNodeGetLastIndex(pSyncNode) + 1; + syncIndexMgrSetIndex(pSyncNode->pNextIndex, pDestId, newNextIndex); + syncIndexMgrSetIndex(pSyncNode->pMatchIndex, pDestId, SYNC_INDEX_INVALID); + sError("vgId:%d sync get pre term error, nextIndex:%ld, update next-index:%ld, match-index:%d, raftid:%ld", + pSyncNode->vgId, nextIndex, newNextIndex, SYNC_INDEX_INVALID, pDestId->addr); + + return -1; + } // batch optimized // SyncIndex lastIndex = syncUtilMinIndex(pSyncNode->pLogStore->getLastIndex(pSyncNode->pLogStore), nextIndex); @@ -148,25 +157,32 @@ int32_t syncNodeAppendEntriesPeersSnapshot(SSyncNode* pSyncNode) { SSyncRaftEntry* pEntry; int32_t code = pSyncNode->pLogStore->syncLogGetEntry(pSyncNode->pLogStore, nextIndex, &pEntry); - ASSERT(code == 0); - if (pEntry != NULL) { + if (code == 0) { + ASSERT(pEntry != NULL); + pMsg = syncAppendEntriesBuild(pEntry->bytes, pSyncNode->vgId); ASSERT(pMsg != NULL); // add pEntry into msg uint32_t len; char* serialized = syncEntrySerialize(pEntry, &len); - assert(len == pEntry->bytes); + ASSERT(len == pEntry->bytes); memcpy(pMsg->data, serialized, len); taosMemoryFree(serialized); syncEntryDestory(pEntry); } else { - // no entry in log - pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); - ASSERT(pMsg != NULL); + if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { + // no entry in log + pMsg = syncAppendEntriesBuild(0, pSyncNode->vgId); + ASSERT(pMsg != NULL); + + } else { + syncNodeLog3("", pSyncNode); + ASSERT(0); + } } // prepare msg diff --git a/source/libs/sync/src/syncRequestVote.c b/source/libs/sync/src/syncRequestVote.c index 9ed7f00982720e4cf4720645e3393bb6d8c84af4..def5c321ad4f29b316ef1ed6597e9cd2ca29adc1 100644 --- a/source/libs/sync/src/syncRequestVote.c +++ b/source/libs/sync/src/syncRequestVote.c @@ -52,7 +52,7 @@ int32_t syncNodeOnRequestVoteCb(SSyncNode* ths, SyncRequestVote* pMsg) { if (pMsg->term > ths->pRaftStore->currentTerm) { syncNodeUpdateTerm(ths, pMsg->term); } - assert(pMsg->term <= ths->pRaftStore->currentTerm); + ASSERT(pMsg->term <= ths->pRaftStore->currentTerm); bool logOK = (pMsg->lastLogTerm > ths->pLogStore->getLastTerm(ths->pLogStore)) || ((pMsg->lastLogTerm == ths->pLogStore->getLastTerm(ths->pLogStore)) && diff --git a/source/libs/sync/src/syncRequestVoteReply.c b/source/libs/sync/src/syncRequestVoteReply.c index 5d041cefcd62eba0593f1bc19ec17da805a43cb2..60963d0dd3b3bd4f7ce18de1a6746fd446306851 100644 --- a/source/libs/sync/src/syncRequestVoteReply.c +++ b/source/libs/sync/src/syncRequestVoteReply.c @@ -50,7 +50,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } - // assert(!(pMsg->term > ths->pRaftStore->currentTerm)); + // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); @@ -65,7 +65,7 @@ int32_t syncNodeOnRequestVoteReplyCb(SSyncNode* ths, SyncRequestVoteReply* pMsg) return ret; } - assert(pMsg->term == ths->pRaftStore->currentTerm); + ASSERT(pMsg->term == ths->pRaftStore->currentTerm); // This tallies votes even when the current state is not Candidate, // but they won't be looked at, so it doesn't matter. @@ -115,7 +115,7 @@ int32_t syncNodeOnRequestVoteReplySnapshotCb(SSyncNode* ths, SyncRequestVoteRepl return ret; } - // assert(!(pMsg->term > ths->pRaftStore->currentTerm)); + // ASSERT(!(pMsg->term > ths->pRaftStore->currentTerm)); // no need this code, because if I receive reply.term, then I must have sent for that term. // if (pMsg->term > ths->pRaftStore->currentTerm) { // syncNodeUpdateTerm(ths, pMsg->term); diff --git a/source/libs/sync/src/syncRespMgr.c b/source/libs/sync/src/syncRespMgr.c index 354575d29e6be2b4abf5a88743572c869d3344f7..48c1b70a04e965145027912c5cbebf6e339cd670 100644 --- a/source/libs/sync/src/syncRespMgr.c +++ b/source/libs/sync/src/syncRespMgr.c @@ -22,7 +22,7 @@ SSyncRespMgr *syncRespMgrCreate(void *data, int64_t ttl) { pObj->pRespHash = taosHashInit(sizeof(uint64_t), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_NO_LOCK); - assert(pObj->pRespHash != NULL); + ASSERT(pObj->pRespHash != NULL); pObj->ttl = ttl; pObj->data = data; pObj->seqNum = 0; diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index c694a0b71539c410dd20c06c3085b2c10aebcbc0..ed9d62fd149d94c89f12c6f99518359ba2ac77ed 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -67,51 +67,63 @@ void snapshotSenderDestroy(SSyncSnapshotSender *pSender) { bool snapshotSenderIsStart(SSyncSnapshotSender *pSender) { return pSender->start; } // begin send snapshot (current term, seq begin) -void snapshotSenderStart(SSyncSnapshotSender *pSender) { +void snapshotSenderStart(SSyncSnapshotSender *pSender, SSnapshot snapshot, void *pReader) { ASSERT(!snapshotSenderIsStart(pSender)); pSender->seq = SYNC_SNAPSHOT_SEQ_BEGIN; pSender->ack = SYNC_SNAPSHOT_SEQ_INVALID; - // open snapshot reader + // init snapshot and reader ASSERT(pSender->pReader == NULL); - int32_t ret = pSender->pSyncNode->pFsm->FpSnapshotStartRead(pSender->pSyncNode->pFsm, &(pSender->pReader)); - ASSERT(ret == 0); + pSender->pReader = pReader; + pSender->snapshot = snapshot; if (pSender->pCurrentBlock != NULL) { taosMemoryFree(pSender->pCurrentBlock); } - pSender->blockLen = 0; - // get current snapshot info - pSender->pSyncNode->pFsm->FpGetSnapshotInfo(pSender->pSyncNode->pFsm, &(pSender->snapshot)); - - sTrace("snapshotSenderStart lastApplyIndex:%ld, lastApplyTerm:%lu, lastConfigIndex:%ld", - pSender->snapshot.lastApplyIndex, pSender->snapshot.lastApplyTerm, pSender->snapshot.lastConfigIndex); - if (pSender->snapshot.lastConfigIndex != SYNC_INDEX_INVALID) { - /* + int32_t code = 0; SSyncRaftEntry *pEntry = NULL; - int32_t code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, - pSender->snapshot.lastConfigIndex, &pEntry); - ASSERT(code == 0); - ASSERT(pEntry != NULL); - */ + code = pSender->pSyncNode->pLogStore->syncLogGetEntry(pSender->pSyncNode->pLogStore, + pSender->snapshot.lastConfigIndex, &pEntry); + + bool getLastConfig = false; + if (code == 0) { + ASSERT(pEntry != NULL); + + SRpcMsg rpcMsg; + syncEntry2OriginalRpc(pEntry, &rpcMsg); + + SSyncCfg lastConfig; + int32_t ret = syncCfgFromStr(rpcMsg.pCont, &lastConfig); + ASSERT(ret == 0); + pSender->lastConfig = lastConfig; + getLastConfig = true; + + rpcFreeCont(rpcMsg.pCont); + syncEntryDestory(pEntry); + } else { + if (pSender->snapshot.lastConfigIndex == pSender->pSyncNode->pRaftCfg->lastConfigIndex) { + sTrace("vgId:%d sync sender get cfg from local", pSender->pSyncNode->vgId); + pSender->lastConfig = pSender->pSyncNode->pRaftCfg->cfg; + getLastConfig = true; + } + } - SSyncRaftEntry *pEntry = - pSender->pSyncNode->pLogStore->getEntry(pSender->pSyncNode->pLogStore, pSender->snapshot.lastConfigIndex); - ASSERT(pEntry != NULL); + if (!getLastConfig) { + char logBuf[128]; + snprintf(logBuf, sizeof(logBuf), "snapshot sender update lcindex from %ld to -1", + pSender->snapshot.lastConfigIndex); + pSender->snapshot.lastConfigIndex = -1; - SRpcMsg rpcMsg; - syncEntry2OriginalRpc(pEntry, &rpcMsg); - SSyncCfg lastConfig; - int32_t ret = syncCfgFromStr(rpcMsg.pCont, &lastConfig); - ASSERT(ret == 0); - pSender->lastConfig = lastConfig; + char *eventLog = snapshotSender2SimpleStr(pSender, logBuf); + syncNodeEventLog(pSender->pSyncNode, eventLog); + taosMemoryFree(eventLog); - rpcFreeCont(rpcMsg.pCont); - syncEntryDestory(pEntry); + memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); + } } else { memset(&(pSender->lastConfig), 0, sizeof(SSyncCfg)); @@ -529,6 +541,10 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { ASSERT(writeCode == 0); pSyncNode->pFsm->FpSnapshotStopWrite(pSyncNode->pFsm, pReceiver->pWriter, true); + if (pReceiver->snapshot.lastApplyIndex > pReceiver->pSyncNode->commitIndex) { + pReceiver->pSyncNode->commitIndex = pReceiver->snapshot.lastApplyIndex; + } + pSyncNode->pLogStore->syncLogSetBeginIndex(pSyncNode->pLogStore, pMsg->lastIndex + 1); // maybe update lastconfig @@ -548,7 +564,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { pSyncNode->pFsm->FpGetSnapshotInfo(pSyncNode->pFsm, &snapshot); do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver finish"); + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver finish, apply snapshot"); syncNodeEventLog(pSyncNode, eventLog); taosMemoryFree(eventLog); } while (0); @@ -559,7 +575,7 @@ int32_t syncNodeOnSnapshotSendCb(SSyncNode *pSyncNode, SyncSnapshotSend *pMsg) { needRsp = true; do { - char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver end"); + char *eventLog = snapshotReceiver2SimpleStr(pReceiver, "snapshot receiver stop"); syncNodeEventLog(pSyncNode, eventLog); taosMemoryFree(eventLog); } while (0); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index 1d1ff7ae53682b3b47badc7e04084de89463b2f3..5b73d980c4088966cc56ef0d58e2a097920bbee5 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -86,7 +86,7 @@ void syncUtilraftId2EpSet(const SRaftId* raftId, SEpSet* pEpSet) { void syncUtilnodeInfo2raftId(const SNodeInfo* pNodeInfo, SyncGroupId vgId, SRaftId* raftId) { uint32_t ipv4 = taosGetIpv4FromFqdn(pNodeInfo->nodeFqdn); - assert(ipv4 != 0xFFFFFFFF); + ASSERT(ipv4 != 0xFFFFFFFF); char ipbuf[128] = {0}; tinet_ntoa(ipbuf, ipv4); raftId->addr = syncUtilAddr2U64(ipbuf, pNodeInfo->nodePort); @@ -124,7 +124,7 @@ void syncUtilbufCopyDeep(const SSyncBuffer* src, SSyncBuffer* dest) { int32_t syncUtilRand(int32_t max) { return taosRand() % max; } int32_t syncUtilElectRandomMS(int32_t min, int32_t max) { - assert(min > 0 && max > 0 && max >= min); + ASSERT(min > 0 && max > 0 && max >= min); return min + syncUtilRand(max - min); } @@ -201,7 +201,7 @@ bool syncUtilCanPrint(char c) { char* syncUtilprintBin(char* ptr, uint32_t len) { char* s = taosMemoryMalloc(len + 1); - assert(s != NULL); + ASSERT(s != NULL); memset(s, 0, len + 1); memcpy(s, ptr, len); @@ -216,7 +216,7 @@ char* syncUtilprintBin(char* ptr, uint32_t len) { char* syncUtilprintBin2(char* ptr, uint32_t len) { uint32_t len2 = len * 4 + 1; char* s = taosMemoryMalloc(len2); - assert(s != NULL); + ASSERT(s != NULL); memset(s, 0, len2); char* p = s; diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 528c2f26c85c17f33f0a783def69ef9f26798b1b..d6c2cbd34e3d90935bc651fac54a697b2e9bbc4e 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -24,7 +24,7 @@ static void voteGrantedClearVotes(SVotesGranted *pVotesGranted) { SVotesGranted *voteGrantedCreate(SSyncNode *pSyncNode) { SVotesGranted *pVotesGranted = taosMemoryMalloc(sizeof(SVotesGranted)); - assert(pVotesGranted != NULL); + ASSERT(pVotesGranted != NULL); memset(pVotesGranted, 0, sizeof(SVotesGranted)); pVotesGranted->replicas = &(pSyncNode->replicasId); @@ -62,9 +62,9 @@ bool voteGrantedMajority(SVotesGranted *pVotesGranted) { } void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { - assert(pMsg->voteGranted == true); - assert(pMsg->term == pVotesGranted->term); - assert(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); + ASSERT(pMsg->voteGranted == true); + ASSERT(pMsg->term == pVotesGranted->term); + ASSERT(syncUtilSameId(&pVotesGranted->pSyncNode->myRaftId, &pMsg->destId)); int j = -1; for (int i = 0; i < pVotesGranted->replicaNum; ++i) { @@ -73,14 +73,14 @@ void voteGrantedVote(SVotesGranted *pVotesGranted, SyncRequestVoteReply *pMsg) { break; } } - assert(j != -1); - assert(j >= 0 && j < pVotesGranted->replicaNum); + ASSERT(j != -1); + ASSERT(j >= 0 && j < pVotesGranted->replicaNum); if (pVotesGranted->isGranted[j] != true) { ++(pVotesGranted->votes); pVotesGranted->isGranted[j] = true; } - assert(pVotesGranted->votes <= pVotesGranted->replicaNum); + ASSERT(pVotesGranted->votes <= pVotesGranted->replicaNum); } void voteGrantedReset(SVotesGranted *pVotesGranted, SyncTerm term) { @@ -127,7 +127,7 @@ cJSON *voteGranted2Json(SVotesGranted *pVotesGranted) { char *voteGranted2Str(SVotesGranted *pVotesGranted) { cJSON *pJson = voteGranted2Json(pVotesGranted); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } @@ -162,7 +162,7 @@ void voteGrantedLog2(char *s, SVotesGranted *pObj) { // SVotesRespond ----------------------------- SVotesRespond *votesRespondCreate(SSyncNode *pSyncNode) { SVotesRespond *pVotesRespond = taosMemoryMalloc(sizeof(SVotesRespond)); - assert(pVotesRespond != NULL); + ASSERT(pVotesRespond != NULL); memset(pVotesRespond, 0, sizeof(SVotesRespond)); pVotesRespond->replicas = &(pSyncNode->replicasId); @@ -198,15 +198,15 @@ bool votesResponded(SVotesRespond *pVotesRespond, const SRaftId *pRaftId) { } void votesRespondAdd(SVotesRespond *pVotesRespond, const SyncRequestVoteReply *pMsg) { - assert(pVotesRespond->term == pMsg->term); + ASSERT(pVotesRespond->term == pMsg->term); for (int i = 0; i < pVotesRespond->replicaNum; ++i) { if (syncUtilSameId(&((*(pVotesRespond->replicas))[i]), &pMsg->srcId)) { - // assert(pVotesRespond->isRespond[i] == false); + // ASSERT(pVotesRespond->isRespond[i] == false); pVotesRespond->isRespond[i] = true; return; } } - assert(0); + ASSERT(0); } void votesRespondReset(SVotesRespond *pVotesRespond, SyncTerm term) { @@ -256,7 +256,7 @@ cJSON *votesRespond2Json(SVotesRespond *pVotesRespond) { char *votesRespond2Str(SVotesRespond *pVotesRespond) { cJSON *pJson = votesRespond2Json(pVotesRespond); - char * serialized = cJSON_Print(pJson); + char *serialized = cJSON_Print(pJson); cJSON_Delete(pJson); return serialized; } diff --git a/source/libs/sync/test/CMakeLists.txt b/source/libs/sync/test/CMakeLists.txt index 2057aa23a4d07e492a70164017b5111b434218ff..27084286da53dc68c9346af9b5c5c00fa0dc28f2 100644 --- a/source/libs/sync/test/CMakeLists.txt +++ b/source/libs/sync/test/CMakeLists.txt @@ -34,6 +34,7 @@ add_executable(syncWriteTest "") add_executable(syncReplicateTest "") add_executable(syncRefTest "") add_executable(syncLogStoreCheck "") +add_executable(syncLogStoreCheck2 "") add_executable(syncRaftCfgTest "") add_executable(syncRespMgrTest "") add_executable(syncSnapshotTest "") @@ -196,6 +197,10 @@ target_sources(syncLogStoreCheck PRIVATE "syncLogStoreCheck.cpp" ) +target_sources(syncLogStoreCheck2 + PRIVATE + "syncLogStoreCheck2.cpp" +) target_sources(syncRaftCfgTest PRIVATE "syncRaftCfgTest.cpp" @@ -442,6 +447,11 @@ target_include_directories(syncLogStoreCheck "${TD_SOURCE_DIR}/include/libs/sync" "${CMAKE_CURRENT_SOURCE_DIR}/../inc" ) +target_include_directories(syncLogStoreCheck2 + PUBLIC + "${TD_SOURCE_DIR}/include/libs/sync" + "${CMAKE_CURRENT_SOURCE_DIR}/../inc" +) target_include_directories(syncRaftCfgTest PUBLIC "${TD_SOURCE_DIR}/include/libs/sync" @@ -668,6 +678,10 @@ target_link_libraries(syncLogStoreCheck sync gtest_main ) +target_link_libraries(syncLogStoreCheck2 + sync + gtest_main +) target_link_libraries(syncRaftCfgTest sync gtest_main diff --git a/source/libs/sync/test/syncLogStoreCheck2.cpp b/source/libs/sync/test/syncLogStoreCheck2.cpp new file mode 100644 index 0000000000000000000000000000000000000000..80679bc85c4067fef291dcc583b727f1f3e909c3 --- /dev/null +++ b/source/libs/sync/test/syncLogStoreCheck2.cpp @@ -0,0 +1,76 @@ +#include +#include +#include "syncEnv.h" +#include "syncIO.h" +#include "syncInt.h" +#include "syncRaftLog.h" +#include "syncRaftStore.h" +#include "syncUtil.h" +#include "wal.h" + +void logTest() { + sTrace("--- sync log test: trace"); + sDebug("--- sync log test: debug"); + sInfo("--- sync log test: info"); + sWarn("--- sync log test: warn"); + sError("--- sync log test: error"); + sFatal("--- sync log test: fatal"); +} + +void init() { + int code = walInit(); + assert(code == 0); +} + +void cleanup() { walCleanUp(); } + +SWal* createWal(char* path, int32_t vgId) { + SWalCfg walCfg; + memset(&walCfg, 0, sizeof(SWalCfg)); + walCfg.vgId = vgId; + walCfg.fsyncPeriod = 1000; + walCfg.retentionPeriod = 1000; + walCfg.rollPeriod = 1000; + walCfg.retentionSize = 1000; + walCfg.segSize = 1000; + walCfg.level = TAOS_WAL_FSYNC; + SWal* pWal = walOpen(path, &walCfg); + assert(pWal != NULL); + return pWal; +} + +SSyncNode* createSyncNode(SWal* pWal) { + SSyncNode* pSyncNode = (SSyncNode*)taosMemoryMalloc(sizeof(SSyncNode)); + memset(pSyncNode, 0, sizeof(SSyncNode)); + pSyncNode->pWal = pWal; + return pSyncNode; +} + +void usage(char* exe) { printf("usage: %s path vgId \n", exe); } + +int main(int argc, char** argv) { + if (argc != 3) { + usage(argv[0]); + exit(-1); + } + char* path = argv[1]; + int32_t vgId = atoi(argv[2]); + + init(); + SWal* pWal = createWal(path, vgId); + assert(pWal != NULL); + SSyncNode* pSyncNode = createSyncNode(pWal); + assert(pSyncNode != NULL); + + SSyncLogStore* pLog = logStoreCreate(pSyncNode); + assert(pLog != NULL); + + logStoreSimplePrint2((char*)"==syncLogStoreCheck2==", pLog); + + walClose(pWal); + logStoreDestory(pLog); + taosMemoryFree(pSyncNode); + + cleanup(); + return 0; +} diff --git a/source/libs/sync/test/syncRaftLogTest2.cpp b/source/libs/sync/test/syncRaftLogTest2.cpp index 32ff441a6f43fdaf8cce260473ebb89d635348c6..bd6fa7c2c3c5616b206540e63e0b52821608691f 100644 --- a/source/libs/sync/test/syncRaftLogTest2.cpp +++ b/source/libs/sync/test/syncRaftLogTest2.cpp @@ -69,6 +69,7 @@ void test1() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest1 ----- ", pLogStore); if (gAssert) { @@ -88,6 +89,7 @@ void test1() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest1 restart ----- ", pLogStore); if (gAssert) { @@ -110,6 +112,7 @@ void test2() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; pLogStore->syncLogSetBeginIndex(pLogStore, 5); logStoreLog2((char*)"\n\n\ntest2 ----- ", pLogStore); @@ -130,6 +133,7 @@ void test2() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest2 restart ----- ", pLogStore); if (gAssert) { @@ -152,6 +156,7 @@ void test3() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest3 ----- ", pLogStore); if (gAssert) { @@ -198,6 +203,7 @@ void test3() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest3 restart ----- ", pLogStore); if (gAssert) { @@ -220,6 +226,7 @@ void test4() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest4 ----- ", pLogStore); pLogStore->syncLogSetBeginIndex(pLogStore, 5); @@ -257,6 +264,7 @@ void test4() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest4 restart ----- ", pLogStore); if (gAssert) { @@ -279,6 +287,7 @@ void test5() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 ----- ", pLogStore); pLogStore->syncLogSetBeginIndex(pLogStore, 5); @@ -329,6 +338,7 @@ void test5() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest5 restart ----- ", pLogStore); if (gAssert) { @@ -351,6 +361,7 @@ void test6() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest6 ----- ", pLogStore); pLogStore->syncLogSetBeginIndex(pLogStore, 5); @@ -401,6 +412,7 @@ void test6() { init(); pLogStore = logStoreCreate(pSyncNode); assert(pLogStore); + pSyncNode->pLogStore = pLogStore; logStoreLog2((char*)"\n\n\ntest6 restart ----- ", pLogStore); if (gAssert) { diff --git a/source/libs/sync/test/syncReconfigFinishTest.cpp b/source/libs/sync/test/syncReconfigFinishTest.cpp index 22e22bb5621d5942f78d49d320483667a13bbec1..2aac38bbd1c71effe2662fa90b5cdfff3fa2db19 100644 --- a/source/libs/sync/test/syncReconfigFinishTest.cpp +++ b/source/libs/sync/test/syncReconfigFinishTest.cpp @@ -14,8 +14,8 @@ void logTest() { sFatal("--- sync log test: fatal"); } -SSyncCfg* createSyncOldCfg() { - SSyncCfg* pCfg = (SSyncCfg*)taosMemoryMalloc(sizeof(SSyncCfg)); +SSyncCfg *createSyncOldCfg() { + SSyncCfg *pCfg = (SSyncCfg *)taosMemoryMalloc(sizeof(SSyncCfg)); memset(pCfg, 0, sizeof(SSyncCfg)); pCfg->replicaNum = 3; @@ -28,8 +28,8 @@ SSyncCfg* createSyncOldCfg() { return pCfg; } -SSyncCfg* createSyncNewCfg() { - SSyncCfg* pCfg = (SSyncCfg*)taosMemoryMalloc(sizeof(SSyncCfg)); +SSyncCfg *createSyncNewCfg() { + SSyncCfg *pCfg = (SSyncCfg *)taosMemoryMalloc(sizeof(SSyncCfg)); memset(pCfg, 0, sizeof(SSyncCfg)); pCfg->replicaNum = 3; @@ -44,9 +44,9 @@ SSyncCfg* createSyncNewCfg() { SyncReconfigFinish *createMsg() { SyncReconfigFinish *pMsg = syncReconfigFinishBuild(1234); - - SSyncCfg* pOld = createSyncOldCfg(); - SSyncCfg* pNew = createSyncNewCfg(); + + SSyncCfg *pOld = createSyncOldCfg(); + SSyncCfg *pNew = createSyncNewCfg(); pMsg->oldCfg = *pOld; pMsg->newCfg = *pNew; @@ -60,18 +60,16 @@ SyncReconfigFinish *createMsg() { return pMsg; } - void test1() { SyncReconfigFinish *pMsg = createMsg(); syncReconfigFinishLog2((char *)"test1:", pMsg); syncReconfigFinishDestroy(pMsg); } - void test2() { SyncReconfigFinish *pMsg = createMsg(); - uint32_t len = pMsg->bytes; - char * serialized = (char *)taosMemoryMalloc(len); + uint32_t len = pMsg->bytes; + char * serialized = (char *)taosMemoryMalloc(len); syncReconfigFinishSerialize(pMsg, serialized, len); SyncReconfigFinish *pMsg2 = syncReconfigFinishBuild(1000); syncReconfigFinishDeserialize(serialized, len, pMsg2); @@ -84,8 +82,8 @@ void test2() { void test3() { SyncReconfigFinish *pMsg = createMsg(); - uint32_t len; - char * serialized = syncReconfigFinishSerialize2(pMsg, &len); + uint32_t len; + char * serialized = syncReconfigFinishSerialize2(pMsg, &len); SyncReconfigFinish *pMsg2 = syncReconfigFinishDeserialize2(serialized, len); syncReconfigFinishLog2((char *)"test3: SyncReconfigFinishSerialize2 -> syncReconfigFinishDeserialize2 ", pMsg2); @@ -96,7 +94,7 @@ void test3() { void test4() { SyncReconfigFinish *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncReconfigFinish2RpcMsg(pMsg, &rpcMsg); SyncReconfigFinish *pMsg2 = (SyncReconfigFinish *)taosMemoryMalloc(rpcMsg.contLen); syncReconfigFinishFromRpcMsg(&rpcMsg, pMsg2); @@ -109,7 +107,7 @@ void test4() { void test5() { SyncReconfigFinish *pMsg = createMsg(); - SRpcMsg rpcMsg; + SRpcMsg rpcMsg; syncReconfigFinish2RpcMsg(pMsg, &rpcMsg); SyncReconfigFinish *pMsg2 = syncReconfigFinishFromRpcMsg2(&rpcMsg); syncReconfigFinishLog2((char *)"test5: syncReconfigFinish2RpcMsg -> syncReconfigFinishFromRpcMsg2 ", pMsg2); diff --git a/source/libs/tdb/src/db/tdbBtree.c b/source/libs/tdb/src/db/tdbBtree.c index 45e71f6c0dd706117549b9d0a54aa2a9be675be4..05fe62762a7402872ef7a5af43452f1ba3752a56 100644 --- a/source/libs/tdb/src/db/tdbBtree.c +++ b/source/libs/tdb/src/db/tdbBtree.c @@ -56,14 +56,9 @@ typedef struct { } SIntHdr; #pragma pack(pop) -typedef struct { - u8 flags; - SBTree *pBt; -} SBtreeInitPageArg; - static int tdbDefaultKeyCmprFn(const void *pKey1, int keyLen1, const void *pKey2, int keyLen2); static int tdbBtreeOpenImpl(SBTree *pBt); -static int tdbBtreeInitPage(SPage *pPage, void *arg, int init); +//static int tdbBtreeInitPage(SPage *pPage, void *arg, int init); static int tdbBtreeEncodeCell(SPage *pPage, const void *pKey, int kLen, const void *pVal, int vLen, SCell *pCell, int *szCell, TXN *pTxn, SBTree *pBt); static int tdbBtreeDecodeCell(SPage *pPage, const SCell *pCell, SCellDecoder *pDecoder, TXN *pTxn, SBTree *pBt); @@ -348,7 +343,7 @@ static int tdbBtreeOpenImpl(SBTree *pBt) { return 0; } -static int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { +int tdbBtreeInitPage(SPage *pPage, void *arg, int init) { SBTree *pBt; u8 flags; u8 leaf; @@ -862,20 +857,20 @@ static int tdbBtreeBalance(SBTC *pBtc) { } // TDB_BTREE_BALANCE -static int tdbFetchOvflPage(SPager *pPager, SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) { +static int tdbFetchOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) { int ret = 0; *pPgno = 0; SBtreeInitPageArg iArg; iArg.pBt = pBt; iArg.flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); - ret = tdbPagerFetchPage(pPager, pPgno, ppOfp, tdbBtreeInitPage, &iArg, pTxn); + ret = tdbPagerFetchPage(pBt->pPager, pPgno, ppOfp, tdbBtreeInitPage, &iArg, pTxn); if (ret < 0) { return -1; } // mark dirty - ret = tdbPagerWrite(pPager, *ppOfp); + ret = tdbPagerWrite(pBt->pPager, *ppOfp); if (ret < 0) { ASSERT(0); return -1; @@ -884,13 +879,13 @@ static int tdbFetchOvflPage(SPager *pPager, SPgno *pPgno, SPage **ppOfp, TXN *pT return ret; } -static int tdbLoadOvflPage(SPager *pPager, SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) { +static int tdbLoadOvflPage(SPgno *pPgno, SPage **ppOfp, TXN *pTxn, SBTree *pBt) { int ret = 0; SBtreeInitPageArg iArg; iArg.pBt = pBt; iArg.flags = TDB_FLAG_ADD(0, TDB_BTREE_OVFL); - ret = tdbPagerFetchPage(pPager, pPgno, ppOfp, tdbBtreeInitPage, &iArg, pTxn); + ret = tdbPagerFetchPage(pBt->pPager, pPgno, ppOfp, tdbBtreeInitPage, &iArg, pTxn); if (ret < 0) { return -1; } @@ -927,7 +922,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const SPgno pgno = 0; SPage *ofp, *nextOfp; - ret = tdbFetchOvflPage(pPage->pPager, &pgno, &ofp, pTxn, pBt); + ret = tdbFetchOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } @@ -967,7 +962,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch next ofp if not last page if (!lastPage) { // fetch a new ofp and make it dirty - ret = tdbFetchOvflPage(pPage->pPager, &pgno, &nextOfp, pTxn, pBt); + ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt); if (ret < 0) { tdbFree(pBuf); return -1; @@ -1024,14 +1019,14 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const nLeft -= lastKeyPageSpace; // fetch next ofp, a new ofp and make it dirty - ret = tdbFetchOvflPage(pPage->pPager, &pgno, &nextOfp, pTxn, pBt); + ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt); if (ret < 0) { return -1; } } } else { // fetch next ofp, a new ofp and make it dirty - ret = tdbFetchOvflPage(pPage->pPager, &pgno, &nextOfp, pTxn, pBt); + ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt); if (ret < 0) { return -1; } @@ -1062,7 +1057,7 @@ static int tdbBtreeEncodePayload(SPage *pPage, SCell *pCell, int nHeader, const // fetch next ofp if not last page if (!lastPage) { // fetch a new ofp and make it dirty - ret = tdbFetchOvflPage(pPage->pPager, &pgno, &nextOfp, pTxn, pBt); + ret = tdbFetchOvflPage(&pgno, &nextOfp, pTxn, pBt); if (ret < 0) { tdbFree(pBuf); return -1; @@ -1203,7 +1198,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, // unpack left val data from ovpages while (pgno != 0) { - ret = tdbLoadOvflPage(pPage->pPager, &pgno, &ofp, pTxn, pBt); + ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } @@ -1240,7 +1235,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, int lastKeyPageSpace = 0; // load left key & val to ovpages while (pgno != 0) { - ret = tdbLoadOvflPage(pPage->pPager, &pgno, &ofp, pTxn, pBt); + ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } @@ -1285,7 +1280,7 @@ static int tdbBtreeDecodePayload(SPage *pPage, const SCell *pCell, int nHeader, } while (nLeft > 0) { - ret = tdbLoadOvflPage(pPage->pPager, &pgno, &ofp, pTxn, pBt); + ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } @@ -1416,7 +1411,7 @@ static int tdbBtreeCellSize(const SPage *pPage, SCell *pCell, int dropOfp, TXN * int bytes; while (pgno != 0) { - ret = tdbLoadOvflPage(pPage->pPager, &pgno, &ofp, pTxn, pBt); + ret = tdbLoadOvflPage(&pgno, &ofp, pTxn, pBt); if (ret < 0) { return -1; } @@ -2028,7 +2023,7 @@ int tdbBtcMoveTo(SBTC *pBtc, const void *pKey, int kLen, int *pCRst) { // check if key <= current position if (idx < nCells) { pCell = tdbPageGetCell(pPage, idx); - tdbBtreeDecodeCell(pPage, pCell, &cd, pBtc->pTxn, pBtc->pBt); + tdbBtreeDecodeCell(pPage, pCell, &cd); c = pBt->kcmpr(pKey, kLen, cd.pKey, cd.kLen); if (c > 0) break; } diff --git a/source/libs/tdb/src/db/tdbPager.c b/source/libs/tdb/src/db/tdbPager.c index a74bb54883fe3e8d4ab344e80d3cd4b01f2cb525..dd3f09d5d2de800773b2f444c1876c9679aa6982 100644 --- a/source/libs/tdb/src/db/tdbPager.c +++ b/source/libs/tdb/src/db/tdbPager.c @@ -424,4 +424,71 @@ static int tdbPagerWritePageToDB(SPager *pPager, SPage *pPage) { } return 0; -} \ No newline at end of file +} + +int tdbPagerRestore(SPager *pPager, SBTree *pBt) { + int ret = 0; + SPgno journalSize = 0; + u8 *pageBuf = NULL; + + tdb_fd_t jfd = tdbOsOpen(pPager->jFileName, TDB_O_RDWR, 0755); + if (jfd == NULL) { + return 0; + } + + ret = tdbGetFileSize(jfd, pPager->pageSize, &journalSize); + if (ret < 0) { + return -1; + } + + pageBuf = tdbOsCalloc(1, pPager->pageSize); + if (pageBuf == NULL) { + return -1; + } + + TXN txn; + tdbTxnOpen(&txn, 0, tdbDefaultMalloc, tdbDefaultFree, NULL, 0); + SBtreeInitPageArg iArg; + iArg.pBt = pBt; + iArg.flags = 0; + + for (int pgIndex = 0; pgIndex < journalSize; ++pgIndex) { + // read pgno & the page from journal + SPgno pgno; + SPage *pPage; + + int ret = tdbOsRead(jfd, &pgno, sizeof(pgno)); + if (ret < 0) { + return -1; + } + + ret = tdbOsRead(jfd, pageBuf, pPager->pageSize); + if (ret < 0) { + return -1; + } + + ret = tdbPagerFetchPage(pPager, &pgno, &pPage, tdbBtreeInitPage, &iArg, &txn); + if (ret < 0) { + return -1; + } + + // write the page to db + ret = tdbPagerWritePageToDB(pPager, pPage); + if (ret < 0) { + return -1; + } + + tdbPCacheRelease(pPager->pCache, pPage, &txn); + } + + tdbOsFSync(pPager->fd); + + tdbTxnClose(&txn); + + tdbOsFree(pageBuf); + + tdbOsClose(jfd); + tdbOsRemove(pPager->jFileName); + + return 0; +} diff --git a/source/libs/tdb/src/db/tdbTable.c b/source/libs/tdb/src/db/tdbTable.c index 1575f9f206d3dd6b77eec6e61a5840e47efedc4b..f1bfb4eabcc2f899c00ae4627177f154de6c140e 100644 --- a/source/libs/tdb/src/db/tdbTable.c +++ b/source/libs/tdb/src/db/tdbTable.c @@ -61,6 +61,11 @@ int tdbTbOpen(const char *tbname, int keyLen, int valLen, tdb_cmpr_fn_t keyCmprF return -1; } + ret = tdbPagerRestore(pPager, pTb->pBt); + if (ret < 0) { + return -1; + } + *ppTb = pTb; return 0; } diff --git a/source/libs/tdb/src/db/tdbUtil.c b/source/libs/tdb/src/db/tdbUtil.c index 4acb83c8e4aa514dfce03f6d5cd8697837025591..9021e08ffe590a215404f846a70cba6ce19183ef 100644 --- a/source/libs/tdb/src/db/tdbUtil.c +++ b/source/libs/tdb/src/db/tdbUtil.c @@ -64,4 +64,4 @@ int tdbGetFileSize(tdb_fd_t fd, int szPage, SPgno *size) { *size = szBytes / szPage; return 0; -} \ No newline at end of file +} diff --git a/source/libs/tdb/src/inc/tdbInt.h b/source/libs/tdb/src/inc/tdbInt.h index 39b5584fab86f1b2ce1c15478c79eb71f924562e..c914e098f7afd55ea2cfda2c5c7c5465fe97ff50 100644 --- a/source/libs/tdb/src/inc/tdbInt.h +++ b/source/libs/tdb/src/inc/tdbInt.h @@ -158,6 +158,13 @@ int tdbBtreeUpsert(SBTree *pBt, const void *pKey, int nKey, const void *pData, i int tdbBtreeGet(SBTree *pBt, const void *pKey, int kLen, void **ppVal, int *vLen); int tdbBtreePGet(SBTree *pBt, const void *pKey, int kLen, void **ppKey, int *pkLen, void **ppVal, int *vLen); +typedef struct { + u8 flags; + SBTree *pBt; +} SBtreeInitPageArg; + +int tdbBtreeInitPage(SPage *pPage, void *arg, int init); + // SBTC int tdbBtcOpen(SBTC *pBtc, SBTree *pBt, TXN *pTxn); int tdbBtcClose(SBTC *pBtc); @@ -185,6 +192,7 @@ int tdbPagerFetchPage(SPager *pPager, SPgno *ppgno, SPage **ppPage, int (*initP TXN *pTxn); void tdbPagerReturnPage(SPager *pPager, SPage *pPage, TXN *pTxn); int tdbPagerAllocPage(SPager *pPager, SPgno *ppgno); +int tdbPagerRestore(SPager *pPager, SBTree *pBt); // tdbPCache.c ==================================== #define TDB_PCACHE_PAGE \ diff --git a/source/libs/tdb/src/inc/tdbUtil.h b/source/libs/tdb/src/inc/tdbUtil.h index c518e8efcc2e372af9a3e4cce09c5a320035fc88..fe97f9c98622acea963a3204066f80217f06c4c9 100644 --- a/source/libs/tdb/src/inc/tdbUtil.h +++ b/source/libs/tdb/src/inc/tdbUtil.h @@ -20,7 +20,7 @@ extern "C" { #endif -#if __STDC_VERSION__ >= 201112L +#if __STDC_VERSION__ >= 201112LL #define TDB_STATIC_ASSERT(op, info) static_assert(op, info) #else #define TDB_STATIC_ASSERT(op, info) diff --git a/source/libs/transport/inc/transLog.h b/source/libs/transport/inc/transLog.h index 9d5c80d8e6210a5ca1302d3ebb6417f9d2eb423d..121939787cc0e9c35adcc357c376e710f6f156bd 100644 --- a/source/libs/transport/inc/transLog.h +++ b/source/libs/transport/inc/transLog.h @@ -24,13 +24,13 @@ extern "C" { #include "tlog.h" #include "ttrace.h" -#define tFatal(...) do {if (rpcDebugFlag & DEBUG_FATAL){ taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); }} while (0) -#define tError(...)do { if (rpcDebugFlag & DEBUG_ERROR){ taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); } } while(0) -#define tWarn(...) do { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); }} while(0) -#define tInfo(...) do { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); }} while(0) -#define tDebug(...) do {if (rpcDebugFlag & DEBUG_DEBUG){ taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); }} while(0) -#define tTrace(...) do {if (rpcDebugFlag & DEBUG_TRACE){ taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} while(0) -#define tDump(x, y) do {if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } while(0) +#define tFatal(...) { if (rpcDebugFlag & DEBUG_FATAL) { taosPrintLog("RPC FATAL ", DEBUG_FATAL, rpcDebugFlag, __VA_ARGS__); }} +#define tError(...) { if (rpcDebugFlag & DEBUG_ERROR ){ taosPrintLog("RPC ERROR ", DEBUG_ERROR, rpcDebugFlag, __VA_ARGS__); }} +#define tWarn(...) { if (rpcDebugFlag & DEBUG_WARN) { taosPrintLog("RPC WARN ", DEBUG_WARN, rpcDebugFlag, __VA_ARGS__); }} +#define tInfo(...) { if (rpcDebugFlag & DEBUG_INFO) { taosPrintLog("RPC ", DEBUG_INFO, rpcDebugFlag, __VA_ARGS__); }} +#define tDebug(...) { if (rpcDebugFlag & DEBUG_DEBUG) { taosPrintLog("RPC ", DEBUG_DEBUG, rpcDebugFlag, __VA_ARGS__); }} +#define tTrace(...) { if (rpcDebugFlag & DEBUG_TRACE) { taosPrintLog("RPC ", DEBUG_TRACE, rpcDebugFlag, __VA_ARGS__); }} +#define tDump(x, y) { if (rpcDebugFlag & DEBUG_DUMP) { taosDumpData((unsigned char *)x, y); } } #define tGTrace(param, ...) do { if (rpcDebugFlag & DEBUG_TRACE){char buf[40] = {0}; TRACE_TO_STR(trace, buf); tTrace(param ", gtid:%s", __VA_ARGS__, buf);}} while(0) #define tGFatal(param, ...) do {if (rpcDebugFlag & DEBUG_FATAL){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tFatal(param ", gtid:%s", __VA_ARGS__, buf); }} while (0) @@ -39,6 +39,7 @@ extern "C" { #define tGInfo(param, ...) do { if (rpcDebugFlag & DEBUG_INFO) { char buf[40] = {0}; TRACE_TO_STR(trace, buf); tInfo(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) #define tGDebug(param,...) do {if (rpcDebugFlag & DEBUG_DEBUG){ char buf[40] = {0}; TRACE_TO_STR(trace, buf); tDebug(param ", gtid:%s", __VA_ARGS__, buf); }} while(0) + // clang-format on #ifdef __cplusplus } diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 6abd230f31b4356f0987b98c7f231e5b49a8b350..5b100e3d56d1977c7073fe0539edd231f504076d 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -457,7 +457,7 @@ void* destroyConnPool(void* pool) { while (connList != NULL) { while (!QUEUE_IS_EMPTY(&connList->conn)) { queue* h = QUEUE_HEAD(&connList->conn); - QUEUE_REMOVE(h); + // QUEUE_REMOVE(h); SCliConn* c = QUEUE_DATA(h, SCliConn, conn); cliDestroyConn(c, true); } @@ -488,6 +488,7 @@ static SCliConn* getConnFromPool(void* pool, char* ip, uint32_t port) { conn->status = ConnNormal; QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); + assert(h == &conn->conn); return conn; } static void allocConnRef(SCliConn* conn, bool update) { @@ -585,7 +586,6 @@ static SCliConn* cliCreateConn(SCliThrd* pThrd) { } static void cliDestroyConn(SCliConn* conn, bool clear) { tTrace("%s conn %p remove from conn pool", CONN_GET_INST_LABEL(conn), conn); - QUEUE_REMOVE(&conn->conn); QUEUE_INIT(&conn->conn); transRemoveExHandle(refMgt, conn->refId); @@ -991,7 +991,7 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { /* * upper layer handle retry if code equal TSDB_CODE_RPC_NETWORK_UNAVAIL */ - if (CONN_NO_PERSIST_BY_APP(pConn)) { + if (CONN_NO_PERSIST_BY_APP(pConn) && pThrd->quit == false) { tmsg_t msgType = pCtx->msgType; if ((pTransInst->retry != NULL && pEpSet->numOfEps > 1 && (pTransInst->retry(pResp->code))) || (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL || pResp->code == TSDB_CODE_APP_NOT_READY || @@ -1004,7 +1004,6 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { if (pResp->code == TSDB_CODE_RPC_NETWORK_UNAVAIL) { if (pCtx->retryCount < pEpSet->numOfEps * 3) { pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); arg->param1 = pMsg; arg->param2 = pThrd; @@ -1015,35 +1014,34 @@ int cliAppCb(SCliConn* pConn, STransMsg* pResp, SCliMsg* pMsg) { transUnrefCliHandle(pConn); return -1; - } - } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { - if (pResp->contLen == 0) { - pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; - transPrintEpSet(&pCtx->epSet); - tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } else { - SEpSet epSet = {0}; - tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); - pCtx->epSet = epSet; - - transPrintEpSet(&pCtx->epSet); - tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, - pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); - } - if (pConn->status != ConnInPool) { - addConnToPool(pThrd->pool, pConn); - } + } else if (pCtx->retryCount < TRANS_RETRY_COUNT_LIMIT) { + if (pResp->contLen == 0) { + pEpSet->inUse = (++pEpSet->inUse) % pEpSet->numOfEps; + transPrintEpSet(&pCtx->epSet); + tTrace("%s use local epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } else { + SEpSet epSet = {0}; + tDeserializeSEpSet(pResp->pCont, pResp->contLen, &epSet); + pCtx->epSet = epSet; + + transPrintEpSet(&pCtx->epSet); + tTrace("%s use remote epset, inUse: %d, retry count:%d, limit: %d", pTransInst->label, pEpSet->inUse, + pCtx->retryCount + 1, TRANS_RETRY_COUNT_LIMIT); + } + if (pConn->status != ConnInPool) { + addConnToPool(pThrd->pool, pConn); + } - STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); - arg->param1 = pMsg; - arg->param2 = pThrd; - transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); - return -1; + STaskArg* arg = taosMemoryMalloc(sizeof(STaskArg)); + arg->param1 = pMsg; + arg->param2 = pThrd; + transDQSched(pThrd->delayQueue, doDelayTask, arg, TRANS_RETRY_INTERVAL); + return -1; + } } } } - STraceId* trace = &pResp->info.traceId; if (pCtx->pSem != NULL) { tGTrace("%s conn %p(sync) handle resp", CONN_GET_INST_LABEL(pConn), pConn); diff --git a/source/libs/wal/src/walMeta.c b/source/libs/wal/src/walMeta.c index 8e9cb3a84b87233901cfdead73498486f5a70687..4150fe6d1b2322f891765db30ff75320883ef334 100644 --- a/source/libs/wal/src/walMeta.c +++ b/source/libs/wal/src/walMeta.c @@ -19,6 +19,8 @@ #include "tref.h" #include "walInt.h" +bool FORCE_INLINE walIsEmpty(SWal* pWal) { return pWal->vers.firstVer == -1; } + int64_t FORCE_INLINE walGetFirstVer(SWal* pWal) { return pWal->vers.firstVer; } int64_t FORCE_INLINE walGetSnaphostVer(SWal* pWal) { return pWal->vers.snapshotVer; } diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index e940191ceafe7688f606df47114920360e16dad0..9bed6d230bcd227317075c1894462b1d805d270b 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -100,7 +100,6 @@ static int32_t walReadChangeFile(SWalReadHandle *pRead, int64_t fileFirstVer) { TdFilePtr pLogTFile = taosOpenFile(fnameStr, TD_FILE_READ); if (pLogTFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - terrno = TSDB_CODE_WAL_INVALID_VER; wError("cannot open file %s, since %s", fnameStr, terrstr()); return -1; } @@ -259,6 +258,12 @@ int32_t walReadWithHandle_s(SWalReadHandle *pRead, int64_t ver, SWalReadHead **p int32_t walReadWithHandle(SWalReadHandle *pRead, int64_t ver) { int64_t code; + + if (pRead->pWal->vers.firstVer == -1) { + terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; + return -1; + } + // TODO: check wal life if (pRead->curVersion != ver) { if (walReadSeekVer(pRead, ver) < 0) { diff --git a/source/libs/wal/src/walWrite.c b/source/libs/wal/src/walWrite.c index 9cbc9a3b020580c5c859a3731a0ce5f15fc5d748..031289f93ea8ab58dadd55be7490b4ec4a8caeda 100644 --- a/source/libs/wal/src/walWrite.c +++ b/source/libs/wal/src/walWrite.c @@ -141,7 +141,7 @@ int32_t walRollback(SWal *pWal, int64_t ver) { // validate offset SWalHead head; ASSERT(taosValidFile(pLogTFile)); - int size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); + int64_t size = taosReadFile(pLogTFile, &head, sizeof(SWalHead)); if (size != sizeof(SWalHead)) { return -1; } @@ -149,22 +149,33 @@ int32_t walRollback(SWal *pWal, int64_t ver) { ASSERT(code == 0); if (code != 0) { + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } if (head.head.version != ver) { - // TODO + ASSERT(0); + terrno = TSDB_CODE_WAL_FILE_CORRUPTED; return -1; } + // truncate old files code = taosFtruncateFile(pLogTFile, entry.offset); if (code < 0) { + ASSERT(0); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } code = taosFtruncateFile(pIdxTFile, idxOff); if (code < 0) { + ASSERT(0); + terrno = TAOS_SYSTEM_ERROR(errno); return -1; } pWal->vers.lastVer = ver - 1; + if (pWal->vers.lastVer < pWal->vers.firstVer) { + ASSERT(pWal->vers.lastVer == pWal->vers.firstVer - 1); + pWal->vers.firstVer = -1; + } ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->lastVer = ver - 1; ((SWalFileInfo *)taosArrayGetLast(pWal->fileInfoSet))->fileSize = entry.offset; taosCloseFile(&pIdxTFile); diff --git a/source/os/src/osLocale.c b/source/os/src/osLocale.c index d2369ea2a29e2e45979b041e1f2b586c6374c4e3..c2d10f3668ba2856380eccbdadde74f5d6ed23ef 100644 --- a/source/os/src/osLocale.c +++ b/source/os/src/osLocale.c @@ -88,11 +88,11 @@ void taosSetSystemLocale(const char *inLocale, const char *inCharSet) { void taosGetSystemLocale(char *outLocale, char *outCharset) { #ifdef WINDOWS - char *locale = setlocale(LC_CTYPE, "chs"); + char *locale = setlocale(LC_CTYPE, "en_US.UTF-8"); if (locale != NULL) { tstrncpy(outLocale, locale, TD_LOCALE_LEN); } - strcpy(outCharset, "cp936"); + strcpy(outCharset, "UTF-8"); #elif defined(_TD_DARWIN_64) /* diff --git a/source/util/src/tbuffer.c b/source/util/src/tbuffer.c index 8552ccac2c66600ab851fb9770887f6e8a431c78..d2fac72c77dc55a1c9e7c087d1af8f78dc466409 100644 --- a/source/util/src/tbuffer.c +++ b/source/util/src/tbuffer.c @@ -21,7 +21,7 @@ typedef union Un4B { uint32_t ui; float f; } Un4B; -#if __STDC_VERSION__ >= 201112L +#if __STDC_VERSION__ >= 201112LL static_assert(sizeof(Un4B) == sizeof(uint32_t), "sizeof(Un4B) must equal to sizeof(uint32_t)"); static_assert(sizeof(Un4B) == sizeof(float), "sizeof(Un4B) must equal to sizeof(float)"); #endif @@ -30,7 +30,7 @@ typedef union Un8B { uint64_t ull; double d; } Un8B; -#if __STDC_VERSION__ >= 201112L +#if __STDC_VERSION__ >= 201112LL static_assert(sizeof(Un8B) == sizeof(uint64_t), "sizeof(Un8B) must equal to sizeof(uint64_t)"); static_assert(sizeof(Un8B) == sizeof(double), "sizeof(Un8B) must equal to sizeof(double)"); #endif diff --git a/source/util/src/tcompression.c b/source/util/src/tcompression.c index 041aec4054761b52e926f874401f9acca5bfb27f..38613a77e6c4bf7729d523fd5acbab93e8e366d9 100644 --- a/source/util/src/tcompression.c +++ b/source/util/src/tcompression.c @@ -58,7 +58,7 @@ static const int32_t TEST_NUMBER = 1; #define is_bigendian() ((*(char *)&TEST_NUMBER) == 0) -#define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951L) +#define SIMPLE8B_MAX_INT64 ((uint64_t)2305843009213693951LL) #define safeInt64Add(a, b) (((a >= 0) && (b <= INT64_MAX - a)) || ((a < 0) && (b >= INT64_MIN - a))) #define ZIGZAG_ENCODE(T, v) ((u##T)((v) >> (sizeof(T) * 8 - 1))) ^ (((u##T)(v)) << 1) // zigzag encode diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index 11d7d9831aadf9f419e618d745c70066147bfa45..db1207f05755da5d9e48f74df9719f2bb96106ec 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -503,6 +503,38 @@ const char *cfgDtypeStr(ECfgDataType type) { } } +void cfgDumpItemValue(SConfigItem *pItem, char* buf, int32_t bufSize, int32_t* pLen) { + int32_t len = 0; + switch (pItem->dtype) { + case CFG_DTYPE_BOOL: + len = snprintf(buf, bufSize, "%u", pItem->bval); + break; + case CFG_DTYPE_INT32: + len = snprintf(buf, bufSize, "%d", pItem->i32); + break; + case CFG_DTYPE_INT64: + len = snprintf(buf, bufSize, "%" PRId64, pItem->i64); + break; + case CFG_DTYPE_FLOAT: + len = snprintf(buf, bufSize, "%f", pItem->fval); + break; + case CFG_DTYPE_STRING: + case CFG_DTYPE_DIR: + case CFG_DTYPE_LOCALE: + case CFG_DTYPE_CHARSET: + case CFG_DTYPE_TIMEZONE: + case CFG_DTYPE_NONE: + len = snprintf(buf, bufSize, "%s", pItem->str); + break; + } + + if (len > bufSize) { + len = bufSize; + } + + *pLen = len; +} + void cfgDumpCfg(SConfig *pCfg, bool tsc, bool dump) { if (dump) { printf(" global config"); @@ -996,4 +1028,4 @@ int32_t cfgGetApollUrl(const char **envCmd, const char *envFile, char* apolloUrl uInfo("fail get apollo url from cmd env file"); return -1; -} \ No newline at end of file +} diff --git a/source/util/src/tencode.c b/source/util/src/tencode.c index 185daf9e45e70c86729b467563625677c32a5e07..2267c907833a7826c322996dd61cb3dad8cff518 100644 --- a/source/util/src/tencode.c +++ b/source/util/src/tencode.c @@ -16,7 +16,7 @@ #define _DEFAULT_SOURCE #include "tencode.h" -#if __STDC_VERSION__ >= 201112L +#if __STDC_VERSION__ >= 201112LL static_assert(sizeof(float) == sizeof(uint32_t), "sizeof(float) must equal to sizeof(uint32_t)"); static_assert(sizeof(double) == sizeof(uint64_t), "sizeof(double) must equal to sizeof(uint64_t)"); #endif diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 6afb240ab91b52c603cd3009c57d8b3c708f7ad0..fa10fc26dd7cf3817e70cb00fb76ceb4522dadf3 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -76,6 +76,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NEED_RETRY, "Retry needed") TAOS_DEFINE_ERROR(TSDB_CODE_OUT_OF_RPC_MEMORY_QUEUE, "Out of memory in rpc queue") TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_TIMESTAMP, "Invalid timestamp format") TAOS_DEFINE_ERROR(TSDB_CODE_MSG_DECODE_ERROR, "Msg decode error") +TAOS_DEFINE_ERROR(TSDB_CODE_NO_AVAIL_DISK, "No available disk") TAOS_DEFINE_ERROR(TSDB_CODE_REF_NO_MEMORY, "Ref out of memory") TAOS_DEFINE_ERROR(TSDB_CODE_REF_FULL, "too many Ref Objs") @@ -134,6 +135,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_TBNAME_ERROR, "Stmt table name not s TAOS_DEFINE_ERROR(TSDB_CODE_TSC_STMT_CLAUSE_ERROR, "not supported stmt clause") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_QUERY_KILLED, "Query killed") TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NO_EXEC_NODE, "No available execution node") +TAOS_DEFINE_ERROR(TSDB_CODE_TSC_NOT_STABLE_ERROR, "Table is not a super table") // mnode-common TAOS_DEFINE_ERROR(TSDB_CODE_MND_APP_ERROR, "Mnode internal error") @@ -355,7 +357,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_TABLE_DATA_IN_MEM, "No table data in memo TAOS_DEFINE_ERROR(TSDB_CODE_TDB_FILE_ALREADY_EXISTS, "File already exists") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_TABLE_RECONFIGURE, "Need to reconfigure table") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVD_CREATE_TABLE_INFO, "Invalid information to create table") -TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "No available disk") +TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_AVAIL_DISK, "TSDB no available disk") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_MESSED_MSG, "TSDB messed message") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_IVLD_TAG_VAL, "TSDB invalid tag value") TAOS_DEFINE_ERROR(TSDB_CODE_TDB_NO_CACHE_LAST_ROW, "TSDB no cache last row data") @@ -426,6 +428,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_IN_NEW_CONFIG, "Sync not in new confi TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEW_CONFIG_ERROR, "Sync new config error") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RECONFIG_NOT_READY, "Sync not ready for reconfig") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") // wal diff --git a/source/util/src/thash.c b/source/util/src/thash.c index 98762d8fb0e4b48907910e6a7e154ed524f57262..8275072748123a4f4fa28b0e8401b967b0d2b099 100644 --- a/source/util/src/thash.c +++ b/source/util/src/thash.c @@ -31,12 +31,12 @@ #define GET_HASH_NODE_DATA(_n) ((char *)(_n) + sizeof(SHashNode)) #define GET_HASH_PNODE(_n) ((SHashNode *)((char *)(_n) - sizeof(SHashNode))) -#define FREE_HASH_NODE(_fp, _n) \ - do { \ - if (_fp != NULL) { \ - (_fp)(_n); \ - } \ - taosMemoryFreeClear(_n); \ +#define FREE_HASH_NODE(_fp, _n) \ + do { \ + if (_fp != NULL) { \ + (_fp)(GET_HASH_NODE_DATA(_n)); \ + } \ + taosMemoryFreeClear(_n); \ } while (0); struct SHashNode { @@ -56,7 +56,7 @@ typedef struct SHashEntry { } SHashEntry; struct SHashObj { - SHashEntry ** hashList; + SHashEntry **hashList; size_t capacity; // number of slots int64_t size; // number of elements in hash table _hash_fn_t hashFp; // hash function @@ -65,7 +65,7 @@ struct SHashObj { SRWLatch lock; // read-write spin lock SHashLockTypeE type; // lock type bool enableUpdate; // enable update - SArray * pMemBlock; // memory block allocated for SHashEntry + SArray *pMemBlock; // memory block allocated for SHashEntry _hash_before_fn_t callbackFp; // function invoked before return the value to caller }; @@ -633,7 +633,7 @@ void taosHashTableResize(SHashObj *pHashObj) { } int64_t st = taosGetTimestampUs(); - void * pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); + void *pNewEntryList = taosMemoryRealloc(pHashObj->hashList, sizeof(void *) * newCapacity); if (pNewEntryList == NULL) { // uDebug("cache resize failed due to out of memory, capacity remain:%zu", pHashObj->capacity); return; @@ -642,7 +642,7 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->hashList = pNewEntryList; size_t inc = newCapacity - pHashObj->capacity; - void * p = taosMemoryCalloc(inc, sizeof(SHashEntry)); + void *p = taosMemoryCalloc(inc, sizeof(SHashEntry)); for (int32_t i = 0; i < inc; ++i) { pHashObj->hashList[i + pHashObj->capacity] = (void *)((char *)p + i * sizeof(SHashEntry)); @@ -653,9 +653,9 @@ void taosHashTableResize(SHashObj *pHashObj) { pHashObj->capacity = newCapacity; for (int32_t idx = 0; idx < pHashObj->capacity; ++idx) { SHashEntry *pe = pHashObj->hashList[idx]; - SHashNode * pNode; - SHashNode * pNext; - SHashNode * pPrev = NULL; + SHashNode *pNode; + SHashNode *pNext; + SHashNode *pPrev = NULL; if (pe->num == 0) { assert(pe->next == NULL); diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index 353e94a49096822fe581d7faa0df8a29a6494c12..0439ed148bdaee8d5fee012eb961d57283e62aaa 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -743,7 +743,7 @@ cmp_end: } void taosSetAllDebugFlag(int32_t flag) { - if (!(flag & DEBUG_TRACE || flag & DEBUG_DEBUG || flag & DEBUG_DUMP)) return; + if (flag <= 0) return; dDebugFlag = flag; vDebugFlag = flag; diff --git a/tests/pytest/util/boundary.py b/tests/pytest/util/boundary.py new file mode 100644 index 0000000000000000000000000000000000000000..4cef926f2ef74e72ed371bea85e2e385db6aa2a7 --- /dev/null +++ b/tests/pytest/util/boundary.py @@ -0,0 +1,44 @@ +class DataBoundary: + def __init__(self): + self.TINYINT_BOUNDARY = [-128, 127] + self.SMALLINT_BOUNDARY = [-32768, 32767] + self.INT_BOUNDARY = [-2147483648, 2147483647] + self.BIGINT_BOUNDARY = [-9223372036854775808, 9223372036854775807] + self.UTINYINT_BOUNDARY = [0, 255] + self.USMALLINT_BOUNDARY = [0, 65535] + self.UINT_BOUNDARY = [0, 4294967295] + self.UBIGINT_BOUNDARY = [0, 18446744073709551615] + self.FLOAT_BOUNDARY = [-3.40E+38, 3.40E+38] + self.DOUBLE_BOUNDARY = [-1.7e+308, 1.7e+308] + self.BOOL_BOUNDARY = [True, False] + self.BINARY_MAX_LENGTH = 16374 + self.NCHAR_MAX_LENGTH = 4093 + self.DBNAME_MAX_LENGTH = 64 + self.STBNAME_MAX_LENGTH = 192 + self.TBNAME_MAX_LENGTH = 192 + self.CHILD_TBNAME_MAX_LENGTH = 192 + self.TAG_KEY_MAX_LENGTH = 64 + self.COL_KEY_MAX_LENGTH = 64 + self.MAX_TAG_COUNT = 128 + self.MAX_TAG_COL_COUNT = 4096 + self.mnodeShmSize = [6292480, 2147483647] + self.mnodeShmSize_default = 6292480 + self.vnodeShmSize = [6292480, 2147483647] + self.vnodeShmSize_default = 31458304 + self.DB_PARAM_BUFFER_CONFIG = {"create_name": "buffer", "query_name": "buffer", "vnode_json_key": "szBuf", "boundary": [3, 16384], "default": 96} + self.DB_PARAM_CACHELAST_CONFIG = {"create_name": "cachelast", "query_name": "cache_model", "vnode_json_key": "", "boundary": [0, 1, 2, 3], "default": 0} + self.DB_PARAM_COMP_CONFIG = {"create_name": "comp", "query_name": "compression", "vnode_json_key": "", "boundary": [0, 1, 2], "default": 2} + self.DB_PARAM_DURATION_CONFIG = {"create_name": "duration", "query_name": "duration", "vnode_json_key": "daysPerFile", "boundary": [1, 3650, '60m', '5256000m', '1h', '87600h', '1d', '3650d'], "default": "14400m"} + self.DB_PARAM_FSYNC_CONFIG = {"create_name": "fsync", "query_name": "fsync", "vnode_json_key": "", "boundary": [0, 180000], "default": 3000} + self.DB_PARAM_KEEP_CONFIG = {"create_name": "keep", "query_name": "fsync", "vnode_json_key": "", "boundary": [1, 365000,'1440m','525600000m','24h','8760000h','1d','365000d'], "default": "5256000m,5256000m,5256000m"} + self.DB_PARAM_MAXROWS_CONFIG = {"create_name": "maxrows", "query_name": "maxrows", "vnode_json_key": "maxRows", "boundary": [200, 10000], "default": 4096} + self.DB_PARAM_MINROWS_CONFIG = {"create_name": "minrows", "query_name": "minrows", "vnode_json_key": "minRows", "boundary": [10, 1000], "default": 100} + self.DB_PARAM_NTABLES_CONFIG = {"create_name": "ntables", "query_name": "ntables", "vnode_json_key": "", "boundary": 0, "default": 0} + self.DB_PARAM_PAGES_CONFIG = {"create_name": "pages", "query_name": "pages", "vnode_json_key": "szCache", "boundary": [64], "default": 256} + self.DB_PARAM_PAGESIZE_CONFIG = {"create_name": "pagesize", "query_name": "pagesize", "vnode_json_key": "szPage", "boundary": [1, 16384], "default": 4} + self.DB_PARAM_PRECISION_CONFIG = {"create_name": "precision", "query_name": "precision", "vnode_json_key": "", "boundary": ['ms', 'us', 'ns'], "default": "ms"} + self.DB_PARAM_REPLICA_CONFIG = {"create_name": "replica", "query_name": "replica", "vnode_json_key": "", "boundary": [1], "default": 1} + self.DB_PARAM_SINGLE_STABLE_CONFIG = {"create_name": "single_stable", "query_name": "single_stable_model", "vnode_json_key": "", "boundary": [0, 1], "default": 0} + self.DB_PARAM_STRICT_CONFIG = {"create_name": "strict", "query_name": "strict", "vnode_json_key": "", "boundary": {"no_strict": 0, "strict": 1}, "default": "no_strict"} + self.DB_PARAM_VGROUPS_CONFIG = {"create_name": "vgroups", "query_name": "vgroups", "vnode_json_key": "", "boundary": [1, 32], "default": 2} + self.DB_PARAM_WAL_CONFIG = {"create_name": "wal", "query_name": "wal", "vnode_json_key": "", "boundary": [1, 2], "default": 1} \ No newline at end of file diff --git a/tests/pytest/util/cluster.py b/tests/pytest/util/cluster.py new file mode 100644 index 0000000000000000000000000000000000000000..5debfab2c4266954577fcabf5e78c8b000f5ed04 --- /dev/null +++ b/tests/pytest/util/cluster.py @@ -0,0 +1,302 @@ +from ssl import ALERT_DESCRIPTION_CERTIFICATE_UNOBTAINABLE +import taos +import sys +import time +import os +import socket + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * + +class ClusterDnodes(TDDnodes): + """rewrite TDDnodes and make MyDdnodes as TDDnodes child class""" + def __init__(self ,dnodes_lists): + + super(ClusterDnodes,self).__init__() + self.dnodes = dnodes_lists # dnode must be TDDnode instance + self.simDeployed = False + self.testCluster = False + self.valgrind = 0 + self.killValgrind = 1 + self.independent = True + self.dnodeNums = 5 + + # def getTDDnodes(dnodeNums): + + # return + + +class ConfigureyCluster: + """configure dnodes and return TDDnodes list, it can """ + + def __init__(self): + self.dnodes = None + self.dnodes_nums = 5 + self.independent = True + self.start_port = 6030 + self.portStep = 100 + hostname1= socket.gethostname() + + def configure_cluster(self ,dnodes_nums=5,independent=True,start_port=6030,portStep=100,hostname="%s"%hostname1): + self.start_port=int(start_port) + self.portStep=int(portStep) + self.hostname=hostname + self.dnodes_nums = int(dnodes_nums) + self.independent = independent + self.dnodes = [] + start_port_sec = 6130 + for num in range(1, (self.dnodes_nums+1)): + dnode = TDDnode(num) + dnode.addExtraCfg("firstEp", f"{hostname}:{self.start_port}") + dnode.addExtraCfg("fqdn", f"{hostname}") + dnode.addExtraCfg("serverPort", f"{self.start_port + (num-1)*self.portStep}") + # dnode.addExtraCfg("monitorFqdn", hostname) + # dnode.addExtraCfg("monitorPort", 7043) + dnode.addExtraCfg("secondEp", f"{hostname}:{start_port_sec}") + # configure three dnoe don't support vnodes + if self.dnodes_nums > 4 : + if self.independent and (num < 4): + dnode.addExtraCfg("supportVnodes", 0) + # print(dnode) + self.dnodes.append(dnode) + return self.dnodes + + def create_dnode(self,conn): + tdSql.init(conn.cursor()) + for dnode in self.dnodes[1:]: + # print(dnode.cfgDict) + dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] + print(dnode_id) + tdSql.execute(" create dnode '%s';"%dnode_id) + # count=0 + # while count < 10: + # time.sleep(1) + # tdSql.query("show dnodes;") + # if tdSql.checkRows(self.dnodes_nums) : + # print("mnode is three nodes") + # if tdSql.queryResult[0][4]=='leader' : + # if tdSql.queryResult[2][4]=='offline': + # if tdSql.queryResult[1][2]=='follower': + # print("stop mnodes on dnode 3 successfully in 10s") + # break + # count+=1 + # else: + # print("stop mnodes on dnode 3 failed in 10s") + # return -1 + checkstatus=False + + + + def check_dnode(self,conn): + tdSql.init(conn.cursor()) + count=0 + while count < 5: + tdSql.query("show dnodes") + # tdLog.debug(tdSql.queryResult) + status=0 + for i in range(self.dnodes_nums): + if tdSql.queryResult[i][4] == "ready": + status+=1 + tdLog.debug(status) + + if status == self.dnodes_nums: + tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %self.dnodes_nums) + break + count+=1 + time.sleep(1) + else: + tdLog.debug("create cluster with %d dnode but check dnode not ready within 5s ! "%self.dnodes_nums) + return -1 + + +cluster = ConfigureyCluster() + + # def start(self ,dnodes_nums): + + # self.TDDnodes = ClusterDnodes(dnodes) + # self.TDDnodes.init("") + # self.TDDnodes.setTestCluster(testCluster) + # self.TDDnodes.setValgrind(valgrind) + # self.TDDnodes.stopAll() + # for dnode in self.TDDnodes.dnodes: + # self.TDDnodes.deploy(dnode.index,{}) + + # for dnode in self.TDDnodes.dnodes: + # self.TDDnodes.starttaosd(dnode.index) + + # # create cluster + # for dnode in self.TDDnodes.dnodes[1:]: + # # print(dnode.cfgDict) + # dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] + # dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] + # dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] + # cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" + # print(cmd) + # os.system(cmd) + + # time.sleep(2) + # tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) + + # def buildcluster(self,dnodenumber): + # self.depoly_cluster(dnodenumber) + # self.master_dnode = self.TDDnodes.dnodes[0] + # self.host=self.master_dnode.cfgDict["fqdn"] + # conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) + # tdSql.init(conn1.cursor()) + + # def checkdnodes(self,dnodenumber): + # count=0 + # while count < 10: + # time.sleep(1) + # statusReadyBumber=0 + # tdSql.query("show dnodes;") + # if tdSql.checkRows(dnodenumber) : + # print("dnode is %d nodes"%dnodenumber) + # for i in range(dnodenumber): + # if tdSql.queryResult[i][4] !='ready' : + # status=tdSql.queryResult[i][4] + # print("dnode:%d status is %s "%(i,status)) + # break + # else: + # statusReadyBumber+=1 + # print(statusReadyBumber) + # if statusReadyBumber == dnodenumber : + # print("all of %d mnodes is ready in 10s "%dnodenumber) + # return True + # break + # count+=1 + # else: + # print("%d mnodes is not ready in 10s "%dnodenumber) + # return False + + + # def check3mnode(self): + # count=0 + # while count < 10: + # time.sleep(1) + # tdSql.query("show mnodes;") + # if tdSql.checkRows(3) : + # print("mnode is three nodes") + # if tdSql.queryResult[0][2]=='leader' : + # if tdSql.queryResult[1][2]=='follower': + # if tdSql.queryResult[2][2]=='follower': + # print("three mnodes is ready in 10s") + # break + # elif tdSql.queryResult[0][2]=='follower' : + # if tdSql.queryResult[1][2]=='leader': + # if tdSql.queryResult[2][2]=='follower': + # print("three mnodes is ready in 10s") + # break + # elif tdSql.queryResult[0][2]=='follower' : + # if tdSql.queryResult[1][2]=='follower': + # if tdSql.queryResult[2][2]=='leader': + # print("three mnodes is ready in 10s") + # break + # count+=1 + # else: + # print("three mnodes is not ready in 10s ") + # return -1 + + # tdSql.query("show mnodes;") + # tdSql.checkRows(3) + # tdSql.checkData(0,1,'%s:6030'%self.host) + # tdSql.checkData(0,3,'ready') + # tdSql.checkData(1,1,'%s:6130'%self.host) + # tdSql.checkData(1,3,'ready') + # tdSql.checkData(2,1,'%s:6230'%self.host) + # tdSql.checkData(2,3,'ready') + + # def check3mnode1off(self): + # count=0 + # while count < 10: + # time.sleep(1) + # tdSql.query("show mnodes;") + # if tdSql.checkRows(3) : + # print("mnode is three nodes") + # if tdSql.queryResult[0][2]=='offline' : + # if tdSql.queryResult[1][2]=='leader': + # if tdSql.queryResult[2][2]=='follower': + # print("stop mnodes on dnode 2 successfully in 10s") + # break + # elif tdSql.queryResult[1][2]=='follower': + # if tdSql.queryResult[2][2]=='leader': + # print("stop mnodes on dnode 2 successfully in 10s") + # break + # count+=1 + # else: + # print("stop mnodes on dnode 2 failed in 10s ") + # return -1 + # tdSql.error("drop mnode on dnode 1;") + + # tdSql.query("show mnodes;") + # tdSql.checkRows(3) + # tdSql.checkData(0,1,'%s:6030'%self.host) + # tdSql.checkData(0,2,'offline') + # tdSql.checkData(0,3,'ready') + # tdSql.checkData(1,1,'%s:6130'%self.host) + # tdSql.checkData(1,3,'ready') + # tdSql.checkData(2,1,'%s:6230'%self.host) + # tdSql.checkData(2,3,'ready') + + # def check3mnode2off(self): + # count=0 + # while count < 40: + # time.sleep(1) + # tdSql.query("show mnodes;") + # if tdSql.checkRows(3) : + # print("mnode is three nodes") + # if tdSql.queryResult[0][2]=='leader' : + # if tdSql.queryResult[1][2]=='offline': + # if tdSql.queryResult[2][2]=='follower': + # print("stop mnodes on dnode 2 successfully in 10s") + # break + # count+=1 + # else: + # print("stop mnodes on dnode 2 failed in 10s ") + # return -1 + # tdSql.error("drop mnode on dnode 2;") + + # tdSql.query("show mnodes;") + # tdSql.checkRows(3) + # tdSql.checkData(0,1,'%s:6030'%self.host) + # tdSql.checkData(0,2,'leader') + # tdSql.checkData(0,3,'ready') + # tdSql.checkData(1,1,'%s:6130'%self.host) + # tdSql.checkData(1,2,'offline') + # tdSql.checkData(1,3,'ready') + # tdSql.checkData(2,1,'%s:6230'%self.host) + # tdSql.checkData(2,2,'follower') + # tdSql.checkData(2,3,'ready') + + # def check3mnode3off(self): + # count=0 + # while count < 10: + # time.sleep(1) + # tdSql.query("show mnodes;") + # if tdSql.checkRows(3) : + # print("mnode is three nodes") + # if tdSql.queryResult[0][2]=='leader' : + # if tdSql.queryResult[2][2]=='offline': + # if tdSql.queryResult[1][2]=='follower': + # print("stop mnodes on dnode 3 successfully in 10s") + # break + # count+=1 + # else: + # print("stop mnodes on dnode 3 failed in 10s") + # return -1 + # tdSql.error("drop mnode on dnode 3;") + # tdSql.query("show mnodes;") + # tdSql.checkRows(3) + # tdSql.checkData(0,1,'%s:6030'%self.host) + # tdSql.checkData(0,2,'leader') + # tdSql.checkData(0,3,'ready') + # tdSql.checkData(1,1,'%s:6130'%self.host) + # tdSql.checkData(1,2,'follower') + # tdSql.checkData(1,3,'ready') + # tdSql.checkData(2,1,'%s:6230'%self.host) + # tdSql.checkData(2,2,'offline') + # tdSql.checkData(2,3,'ready') + diff --git a/tests/pytest/util/common.py b/tests/pytest/util/common.py index e158f8edd2e30386afad73f688a25c0f7c56fc46..b43f87be29d86c06ac422cf86b05c7e086f8cff4 100644 --- a/tests/pytest/util/common.py +++ b/tests/pytest/util/common.py @@ -17,7 +17,7 @@ import string import requests import time import socket - +from .boundary import DataBoundary import taos from util.log import * from util.sql import * @@ -25,9 +25,45 @@ from util.cases import * from util.dnodes import * from util.common import * -class TDCom: - def init(self, conn, logSql): - tdSql.init(conn.cursor(), logSql) +class TDCom: + def __init__(self): + self.sml_type = None + self.env_setting = None + self.smlChildTableName_value = None + self.defaultJSONStrType_value = None + self.smlTagNullName_value = None + self.default_varchar_length = 256 + self.default_nchar_length = 256 + self.default_varchar_datatype = "letters" + self.default_nchar_datatype = "letters" + self.default_tagname_prefix = "t" + self.default_colname_prefix = "c" + self.default_stbname_prefix = "stb" + self.default_ctbname_prefix = "ctb" + self.default_tbname_prefix = "tb" + self.default_tag_index_start_num = 1 + self.default_column_index_start_num = 1 + self.default_stbname_index_start_num = 1 + self.default_ctbname_index_start_num = 1 + self.default_tbname_index_start_num = 1 + self.default_tagts_name = "ts" + self.default_colts_name = "ts" + self.dbname = "test" + self.stb_name = "stb" + self.ctb_name = "ctb" + self.tb_name = "tb" + self.need_tagts = False + self.tag_type_str = "" + self.column_type_str = "" + self.columns_str = None + self.ts_value = None + self.tag_value_list = list() + self.column_value_list = list() + self.full_type_list = ["tinyint", "smallint", "int", "bigint", "tinyint unsigned", "smallint unsigned", "int unsigned", "bigint unsigned", "float", "double", "binary", "nchar", "bool"] + self.white_list = ["statsd", "node_exporter", "collectd", "icinga2", "tcollector", "information_schema", "performance_schema"] + self.Boundary = DataBoundary() + # def init(self, conn, logSql): + # # tdSql.init(conn.cursor(), logSql) def preDefine(self): header = {'Authorization': 'Basic cm9vdDp0YW9zZGF0YQ=='} @@ -113,6 +149,59 @@ class TDCom: def dateToTs(self, datetime_input): return int(time.mktime(time.strptime(datetime_input, "%Y-%m-%d %H:%M:%S.%f"))) + def genTs(self, precision="ms", ts="", protype="taosc", ns_tag=None): + """ + protype = "taosc" or "restful" + gen ts and datetime + """ + if precision == "ns": + if ts == "" or ts is None: + ts = time.time_ns() + else: + ts = ts + if ns_tag is None: + dt = ts + else: + dt = datetime.fromtimestamp(ts // 1000000000) + dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000000000)).zfill(9) + if protype == "restful": + dt = datetime.fromtimestamp(ts // 1000000000) + dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000000000)).zfill(9) + else: + if ts == "" or ts is None: + ts = time.time() + else: + ts = ts + if precision == "ms" or precision is None: + ts = int(round(ts * 1000)) + dt = datetime.fromtimestamp(ts // 1000) + if protype == "taosc": + dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000)).zfill(3) + '000' + elif protype == "restful": + dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000)).zfill(3) + else: + pass + elif precision == "us": + ts = int(round(ts * 1000000)) + dt = datetime.fromtimestamp(ts // 1000000) + dt = dt.strftime('%Y-%m-%d %H:%M:%S') + '.' + str(int(ts % 1000000)).zfill(6) + return ts, dt + + def get_long_name(self, length=10, mode="letters"): + """ + generate long name + mode could be numbers/letters/letters_mixed/mixed + """ + if mode == "numbers": + population = string.digits + elif mode == "letters": + population = string.ascii_letters.lower() + elif mode == "letters_mixed": + population = string.ascii_letters.upper() + string.ascii_letters.lower() + else: + population = string.ascii_letters.lower() + string.digits + return "".join(random.choices(population, k=length)) + def getLongName(self, len, mode = "mixed"): """ generate long name @@ -177,84 +266,92 @@ class TDCom: def close(self): self.cursor.close() - def create_database(self,tsql, dbName='test',dropFlag=1,precision="ms", **kwargs): + ######################################################################################################################################## + # new common API + ######################################################################################################################################## + def create_database(self,tsql, dbName='test',dropFlag=1,**kwargs): if dropFlag == 1: tsql.execute("drop database if exists %s"%(dbName)) ''' vgroups replica precision strict wal fsync comp cachelast single_stable buffer pagesize pages minrows maxrows duration keep retentions ''' - sqlString = f'create database if not exists {dbName} precision "{precision}" vgroups 4' + sqlString = f'create database if not exists {dbName} ' + + dbParams = "" if len(kwargs) > 0: - dbParams = "" for param, value in kwargs.items(): - dbParams += f'{param} {value} ' + if param == "precision": + dbParams += f'{param} "{value}" ' + else: + dbParams += f'{param} {value} ' sqlString += f'{dbParams}' + tdLog.debug("create db sql: %s"%sqlString) tsql.execute(sqlString) tdLog.debug("complete to create database %s"%(dbName)) return - def create_stable(self,tsql, dbName,stbName,columnDict,tagDict): - colSchema = '' - for i in range(columnDict['int']): - colSchema += ', c%d int'%i - tagSchema = '' - for i in range(tagDict['int']): - if i > 0: - tagSchema += ',' - tagSchema += 't%d int'%i - - tsql.execute("create table if not exists %s.%s (ts timestamp %s) tags(%s)"%(dbName, stbName, colSchema, tagSchema)) - tdLog.debug("complete to create %s.%s" %(dbName, stbName)) - return - - def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict): - tsql.execute("use %s" %dbName) - tagsValues = '' - for i in range(tagDict['int']): - if i > 0: - tagsValues += ',' - tagsValues += '%d'%i - - pre_create = "create table" - sql = pre_create - #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) - for i in range(ctbNum): - sql += " %s_%d using %s tags(%s)"%(stbName,i,stbName,tagsValues) - if (i > 0) and (i%100 == 0): - tsql.execute(sql) - sql = pre_create - if sql != pre_create: - tsql.execute(sql) + # def create_stable(self,tsql, dbName,stbName,column_elm_list=None, tag_elm_list=None): + # colSchema = '' + # for i in range(columnDict['int']): + # colSchema += ', c%d int'%i + # tagSchema = '' + # for i in range(tagDict['int']): + # if i > 0: + # tagSchema += ',' + # tagSchema += 't%d int'%i + + # tsql.execute("create table if not exists %s.%s (ts timestamp %s) tags(%s)"%(dbName, stbName, colSchema, tagSchema)) + # tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + # return + + # def create_ctables(self,tsql, dbName,stbName,ctbNum,tagDict): + # tsql.execute("use %s" %dbName) + # tagsValues = '' + # for i in range(tagDict['int']): + # if i > 0: + # tagsValues += ',' + # tagsValues += '%d'%i + + # pre_create = "create table" + # sql = pre_create + # #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + # for i in range(ctbNum): + # sql += " %s_%d using %s tags(%s)"%(stbName,i,stbName,tagsValues) + # if (i > 0) and (i%100 == 0): + # tsql.execute(sql) + # sql = pre_create + # if sql != pre_create: + # tsql.execute(sql) - tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) - return - - def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): - tdLog.debug("start to insert data ............") - tsql.execute("use %s" %dbName) - pre_insert = "insert into " - sql = pre_insert - if startTs == 0: - t = time.time() - startTs = int(round(t * 1000)) - #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) - for i in range(ctbNum): - sql += " %s_%d values "%(stbName,i) - for j in range(rowsPerTbl): - sql += "(%d, %d, %d)"%(startTs + j, j, j) - if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): - tsql.execute(sql) - if j < rowsPerTbl - 1: - sql = "insert into %s_%d values " %(stbName,i) - else: - sql = "insert into " - #end sql - if sql != pre_insert: - #print("insert sql:%s"%sql) - tsql.execute(sql) - tdLog.debug("insert data ............ [OK]") - return + # tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + # return + + # def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=0): + # tdLog.debug("start to insert data ............") + # tsql.execute("use %s" %dbName) + # pre_insert = "insert into " + # sql = pre_insert + # if startTs == 0: + # t = time.time() + # startTs = int(round(t * 1000)) + # #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + # for i in range(ctbNum): + # sql += " %s_%d values "%(stbName,i) + # for j in range(rowsPerTbl): + # sql += "(%d, %d, %d)"%(startTs + j, j, j) + # if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + # tsql.execute(sql) + # if j < rowsPerTbl - 1: + # sql = "insert into %s_%d values " %(stbName,i) + # else: + # sql = "insert into " + # #end sql + # if sql != pre_insert: + # #print("insert sql:%s"%sql) + # tsql.execute(sql) + # tdLog.debug("insert data ............ [OK]") + # return def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -295,4 +392,250 @@ class TDCom: newTdSql.init(cur, False) return newTdSql + ################################################################################################################ + # port from the common.py of new test frame + ################################################################################################################ + def gen_default_tag_str(self): + default_tag_str = "" + for tag_type in self.full_type_list: + if tag_type.lower() not in ["varchar", "binary", "nchar"]: + default_tag_str += f" {self.default_tagname_prefix}{self.default_tag_index_start_num} {tag_type}," + else: + if tag_type.lower() in ["varchar", "binary"]: + default_tag_str += f" {self.default_tagname_prefix}{self.default_tag_index_start_num} {tag_type}({self.default_varchar_length})," + else: + default_tag_str += f" {self.default_tagname_prefix}{self.default_tag_index_start_num} {tag_type}({self.default_nchar_length})," + self.default_tag_index_start_num += 1 + if self.need_tagts: + default_tag_str = self.default_tagts_name + " timestamp," + default_tag_str + return default_tag_str[:-1].lstrip() + + def gen_default_column_str(self): + self.default_column_index_start_num = 1 + default_column_str = "" + for col_type in self.full_type_list: + if col_type.lower() not in ["varchar", "binary", "nchar"]: + default_column_str += f" {self.default_colname_prefix}{self.default_column_index_start_num} {col_type}," + else: + if col_type.lower() in ["varchar", "binary"]: + default_column_str += f" {self.default_colname_prefix}{self.default_column_index_start_num} {col_type}({self.default_varchar_length})," + else: + default_column_str += f" {self.default_colname_prefix}{self.default_column_index_start_num} {col_type}({self.default_nchar_length})," + self.default_column_index_start_num += 1 + default_column_str = self.default_colts_name + " timestamp," + default_column_str + return default_column_str[:-1].lstrip() + + def gen_tag_type_str(self, tagname_prefix, tag_elm_list): + tag_index_start_num = 1 + tag_type_str = "" + if tag_elm_list is None: + tag_type_str = self.gen_default_tag_str() + else: + for tag_elm in tag_elm_list: + if "count" in tag_elm: + total_count = int(tag_elm["count"]) + else: + total_count = 1 + if total_count > 0: + for _ in range(total_count): + tag_type_str += f'{tagname_prefix}{tag_index_start_num} {tag_elm["type"]}, ' + if tag_elm["type"] in ["varchar", "binary", "nchar"]: + tag_type_str = tag_type_str.rstrip()[:-1] + f'({tag_elm["len"]}), ' + tag_index_start_num += 1 + else: + continue + tag_type_str = tag_type_str.rstrip()[:-1] + + return tag_type_str + + def gen_column_type_str(self, colname_prefix, column_elm_list): + column_index_start_num = 1 + column_type_str = "" + if column_elm_list is None: + column_type_str = self.gen_default_column_str() + else: + for column_elm in column_elm_list: + if "count" in column_elm: + total_count = int(column_elm["count"]) + else: + total_count = 1 + if total_count > 0: + for _ in range(total_count): + column_type_str += f'{colname_prefix}{column_index_start_num} {column_elm["type"]}, ' + if column_elm["type"] in ["varchar", "binary", "nchar"]: + column_type_str = column_type_str.rstrip()[:-1] + f'({column_elm["len"]}), ' + column_index_start_num += 1 + else: + continue + column_type_str = self.default_colts_name + " timestamp, " + column_type_str.rstrip()[:-1] + return column_type_str + + def gen_random_type_value(self, type_name, binary_length, binary_type, nchar_length, nchar_type): + if type_name.lower() == "tinyint": + return random.randint(self.Boundary.TINYINT_BOUNDARY[0], self.Boundary.TINYINT_BOUNDARY[1]) + elif type_name.lower() == "smallint": + return random.randint(self.Boundary.SMALLINT_BOUNDARY[0], self.Boundary.SMALLINT_BOUNDARY[1]) + elif type_name.lower() == "int": + return random.randint(self.Boundary.INT_BOUNDARY[0], self.Boundary.INT_BOUNDARY[1]) + elif type_name.lower() == "bigint": + return random.randint(self.Boundary.BIGINT_BOUNDARY[0], self.Boundary.BIGINT_BOUNDARY[1]) + elif type_name.lower() == "tinyint unsigned": + return random.randint(self.Boundary.UTINYINT_BOUNDARY[0], self.Boundary.UTINYINT_BOUNDARY[1]) + elif type_name.lower() == "smallint unsigned": + return random.randint(self.Boundary.USMALLINT_BOUNDARY[0], self.Boundary.USMALLINT_BOUNDARY[1]) + elif type_name.lower() == "int unsigned": + return random.randint(self.Boundary.UINT_BOUNDARY[0], self.Boundary.UINT_BOUNDARY[1]) + elif type_name.lower() == "bigint unsigned": + return random.randint(self.Boundary.UBIGINT_BOUNDARY[0], self.Boundary.UBIGINT_BOUNDARY[1]) + elif type_name.lower() == "float": + return random.uniform(self.Boundary.FLOAT_BOUNDARY[0], self.Boundary.FLOAT_BOUNDARY[1]) + elif type_name.lower() == "double": + return random.uniform(self.Boundary.FLOAT_BOUNDARY[0], self.Boundary.FLOAT_BOUNDARY[1]) + elif type_name.lower() == "binary": + return f'{self.get_long_name(binary_length, binary_type)}' + elif type_name.lower() == "varchar": + return self.get_long_name(binary_length, binary_type) + elif type_name.lower() == "nchar": + return self.get_long_name(nchar_length, nchar_type) + elif type_name.lower() == "bool": + return random.choice(self.Boundary.BOOL_BOUNDARY) + elif type_name.lower() == "timestamp": + return self.genTs()[0] + else: + pass + + def gen_tag_value_list(self, tag_elm_list): + tag_value_list = list() + if tag_elm_list is None: + tag_value_list = list(map(lambda i: self.gen_random_type_value(i, self.default_varchar_length, self.default_varchar_datatype, self.default_nchar_length, self.default_nchar_datatype), self.full_type_list)) + else: + for tag_elm in tag_elm_list: + if "count" in tag_elm: + total_count = int(tag_elm["count"]) + else: + total_count = 1 + if total_count > 0: + for _ in range(total_count): + if tag_elm["type"] in ["varchar", "binary", "nchar"]: + tag_value_list.append(self.gen_random_type_value(tag_elm["type"], tag_elm["len"], self.default_varchar_datatype, tag_elm["len"], self.default_nchar_datatype)) + else: + tag_value_list.append(self.gen_random_type_value(tag_elm["type"], "", "", "", "")) + else: + continue + return tag_value_list + + def gen_column_value_list(self, column_elm_list, ts_value=None): + if ts_value is None: + ts_value = self.genTs()[0] + + column_value_list = list() + if column_elm_list is None: + column_value_list = list(map(lambda i: self.gen_random_type_value(i, self.default_varchar_length, self.default_varchar_datatype, self.default_nchar_length, self.default_nchar_datatype), self.full_type_list)) + else: + for column_elm in column_elm_list: + if "count" in column_elm: + total_count = int(column_elm["count"]) + else: + total_count = 1 + if total_count > 0: + for _ in range(total_count): + if column_elm["type"] in ["varchar", "binary", "nchar"]: + column_value_list.append(self.gen_random_type_value(column_elm["type"], column_elm["len"], self.default_varchar_datatype, column_elm["len"], self.default_nchar_datatype)) + else: + column_value_list.append(self.gen_random_type_value(column_elm["type"], "", "", "", "")) + else: + continue + column_value_list = [self.ts_value] + self.column_value_list + return column_value_list + + def create_stable(self, tsql, dbname=None, stbname="stb", column_elm_list=None, tag_elm_list=None, + count=1, default_stbname_prefix="stb", **kwargs): + colname_prefix = 'c' + tagname_prefix = 't' + stbname_index_start_num = 1 + stb_params = "" + if len(kwargs) > 0: + for param, value in kwargs.items(): + stb_params += f'{param} "{value}" ' + column_type_str = self.gen_column_type_str(colname_prefix, column_elm_list) + tag_type_str = self.gen_tag_type_str(tagname_prefix, tag_elm_list) + + if int(count) <= 1: + create_stable_sql = f'create table {dbname}.{stbname} ({column_type_str}) tags ({tag_type_str}) {stb_params};' + tdLog.info("create stb sql: %s"%create_stable_sql) + tsql.execute(create_stable_sql) + else: + for _ in range(count): + create_stable_sql = f'create table {dbname}.{default_stbname_prefix}{stbname_index_start_num} ({column_type_str}) tags ({tag_type_str}) {stb_params};' + stbname_index_start_num += 1 + tsql.execute(create_stable_sql) + + def create_ctable(self, tsql, dbname=None, stbname=None, tag_elm_list=None, count=1, default_ctbname_prefix="ctb", **kwargs): + ctbname_index_start_num = 0 + ctb_params = "" + if len(kwargs) > 0: + for param, value in kwargs.items(): + ctb_params += f'{param} "{value}" ' + tag_value_list = self.gen_tag_value_list(tag_elm_list) + tag_value_str = "" + # tag_value_str = ", ".join(str(v) for v in self.tag_value_list) + for tag_value in tag_value_list: + if isinstance(tag_value, str): + tag_value_str += f'"{tag_value}", ' + else: + tag_value_str += f'{tag_value}, ' + tag_value_str = tag_value_str.rstrip()[:-1] + + if int(count) <= 1: + create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};' + tsql.execute(create_ctable_sql) + else: + for _ in range(count): + create_ctable_sql = f'create table {dbname}.{default_ctbname_prefix}{ctbname_index_start_num} using {dbname}.{stbname} tags ({tag_value_str}) {ctb_params};' + ctbname_index_start_num += 1 + tdLog.info("create ctb sql: %s"%create_ctable_sql) + tsql.execute(create_ctable_sql) + + def create_table(self, tsql, dbname=None, tbname="ntb", column_elm_list=None, count=1, **kwargs): + tbname_index_start_num = 1 + tbname_prefix="ntb" + + tb_params = "" + if len(kwargs) > 0: + for param, value in kwargs.items(): + tb_params += f'{param} "{value}" ' + column_type_str = self.gen_column_type_str(tbname_prefix, column_elm_list) + + if int(count) <= 1: + create_table_sql = f'create table {dbname}.{tbname} ({column_type_str}) {tb_params};' + tsql.execute(create_table_sql) + else: + for _ in range(count): + create_table_sql = f'create table {dbname}.{tbname_prefix}{tbname_index_start_num} ({column_type_str}) {tb_params};' + tbname_index_start_num += 1 + tsql.execute(create_table_sql) + + def insert_rows(self, tsql, dbname=None, tbname=None, column_ele_list=None, start_ts_value=None, count=1): + if start_ts_value is None: + start_ts_value = self.genTs()[0] + + column_value_list = self.gen_column_value_list(column_ele_list, start_ts_value) + # column_value_str = ", ".join(str(v) for v in self.column_value_list) + column_value_str = "" + for column_value in column_value_list: + if isinstance(column_value, str): + column_value_str += f'"{column_value}", ' + else: + column_value_str += f'{column_value}, ' + column_value_str = column_value_str.rstrip()[:-1] + if int(count) <= 1: + insert_sql = f'insert into {self.tb_name} values ({column_value_str});' + tsql.execute(insert_sql) + else: + for num in range(count): + column_value_list = self.gen_column_value_list(column_ele_list, f'{start_ts_value}+{num}s') + column_value_str = ", ".join(str(v) for v in column_value_list) + insert_sql = f'insert into {dbname}.{tbname} values ({column_value_str});' + tsql.execute(insert_sql) + tdCom = TDCom() diff --git a/tests/pytest/util/constant.py b/tests/pytest/util/constant.py new file mode 100644 index 0000000000000000000000000000000000000000..e5095c74dca855fdfa46c26e27c418898abb4c31 --- /dev/null +++ b/tests/pytest/util/constant.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- + +# basic data type boundary +TINYINT_MAX = 127 +TINYINT_MIN = -128 + +TINYINT_UN_MAX = 255 +TINYINT_UN_MIN = 0 + +SMALLINT_MAX = 32767 +SMALLINT_MIN = -32768 + +SMALLINT_UN_MAX = 65535 +MALLINT_UN_MIN = 0 + +INT_MAX = 2147483647 +INT_MIN = -2147483648 + +INT_UN_MAX = 4294967295 +INT_UN_MIN = 0 + +BIGINT_MAX = 9223372036854775807 +BIGINT_MIN = -9223372036854775808 + +BIGINT_UN_MAX = 18446744073709551615 +BIGINT_UN_MIN = 0 + +FLOAT_MAX = 3.40E+38 +FLOAT_MIN = -3.40E+38 + +DOUBLE_MAX = 1.7E+308 +DOUBLE_MIN = -1.7E+308 + +# schema boundary +BINARY_LENGTH_MAX = 16374 +NCAHR_LENGTH_MAX_ = 4093 +DBNAME_LENGTH_MAX_ = 64 + +STBNAME_LENGTH_MAX = 192 +STBNAME_LENGTH_MIN = 1 + +TBNAME_LENGTH_MAX = 192 +TBNAME_LENGTH_MIN = 1 + +CHILD_TBNAME_LENGTH_MAX = 192 +CHILD_TBNAME_LENGTH_MIN = 1 + +TAG_NAME_LENGTH_MAX = 64 +TAG_NAME_LENGTH_MIN = 1 + +COL_NAME_LENGTH_MAX = 64 +COL_NAME_LENGTH_MIN = 1 + +TAG_COUNT_MAX = 128 +TAG_COUNT_MIN = 1 + +COL_COUNT_MAX = 4096 +COL_COUNT_MIN = 2 + +TAG_COL_COUNT_MAX = 4096 +TAG_COL_COUNT_MIN = 3 + +MNODE_SHM_SIZE_MAX = 2147483647 +MNODE_SHM_SIZE_MIN = 6292480 +MNODE_SHM_SIZE_DEFAULT = 6292480 + +VNODE_SHM_SIZE_MAX = 2147483647 +VNODE_SHM_SIZE_MIN = 6292480 +VNODE_SHM_SIZE_DEFAULT = 31458304 \ No newline at end of file diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index b7c6dce0186f15d6e4bf45942bf567a0c762d522..580fc8ee47b8c336c9cd15234f1169f83ca28727 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -61,7 +61,7 @@ class TDSql: self.cursor.execute(s) s = 'use db' self.cursor.execute(s) - time.sleep(2) + time.sleep(2) def error(self, sql): expectErrNotOccured = True @@ -249,7 +249,7 @@ class TDSql: raise Exception(repr(e)) return self.queryResult - + def executeTimes(self, sql, times): for i in range(times): try: diff --git a/tests/pytest/util/sqlset.py b/tests/pytest/util/sqlset.py new file mode 100644 index 0000000000000000000000000000000000000000..939f53154aebbfee22cf5d49cae13ae002fab879 --- /dev/null +++ b/tests/pytest/util/sqlset.py @@ -0,0 +1,46 @@ +################################################################### +# Copyright (c) 2016 by TAOS Technologies, Inc. +# All rights reserved. +# +# This file is proprietary and confidential to TAOS Technologies. +# No part of this file may be reproduced, stored, transmitted, +# disclosed or used in any form or by any means other than as +# expressly provided by the written permission from Jianhui Tao +# +################################################################### + +# -*- coding: utf-8 -*- + +from util.sql import tdSql + +class TDSetSql: + def init(self, conn, logSql): + tdSql.init(conn.cursor(), logSql) + self.stbname = 'stb' + + def set_create_normaltable_sql(self, ntbname='ntb', + column_dict={'ts':'timestamp','col1':'tinyint','col2':'smallint','col3':'int','col4':'bigint','col5': 'unsigned int','col6': 'unsigned tinyint','col7': 'unsigned smallint', + 'col8': 'unsigned int','col9': 'unsigned bigint','col10': 'float','col11': 'double','col12': 'bool','col13': 'binary(20)','col14': 'nchar(20)'}): + column_sql = '' + for k, v in column_dict.items(): + column_sql += f"{k} {v}," + create_ntb_sql = f'create table {ntbname} ({column_sql[:-1]})' + return create_ntb_sql + + def set_create_stable_sql(self,stbname='stb', + column_dict={'ts':'timestamp','col1':'tinyint','col2':'smallint','col3':'int','col4':'bigint','col5': 'unsigned int','col6': 'unsigned tinyint','col7': 'unsigned smallint', + 'col8': 'unsigned int','col9': 'unsigned bigint','col10': 'float','col11': 'double','col12': 'bool','col13': 'binary(20)','col14': 'nchar(20)'}, + tag_dict={'ts_tag':'timestamp','t1':'tinyint','t2':'smallint','t3':'int','t4':'bigint','t5': 'unsigned int','t6': 'unsigned tinyint','t7': 'unsigned smallint', + 't8': 'unsigned int','t9': 'unsigned bigint','t10': 'float','t11': 'double','t12': 'bool','t13': 'binary(20)','t14': 'nchar(20)'}): + column_sql = '' + tag_sql = '' + for k,v in column_dict.items(): + column_sql += f"{k} {v}," + for k,v in tag_dict.items(): + tag_sql += f"{k} {v}," + create_stb_sql = f'create table {stbname} ({column_sql[:-1]}) tags({tag_sql[:-1]})' + return create_stb_sql + + + + \ No newline at end of file diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 7dd7621d0b429caeb2e54c0215b29c4a0b396124..0e7030b230cc73ba2a79222ccdeda03ac02160f9 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -353,13 +353,13 @@ void taosMemoryFree(const void *ptr) { static int64_t taosGetTimestampMs() { struct timeval systemTime; taosGetTimeOfDay(&systemTime); - return (int64_t)systemTime.tv_sec * 1000L + (int64_t)systemTime.tv_usec/1000; + return (int64_t)systemTime.tv_sec * 1000LL + (int64_t)systemTime.tv_usec/1000; } static int64_t taosGetTimestampUs() { struct timeval systemTime; taosGetTimeOfDay(&systemTime); - return (int64_t)systemTime.tv_sec * 1000000L + (int64_t)systemTime.tv_usec; + return (int64_t)systemTime.tv_sec * 1000000LL + (int64_t)systemTime.tv_usec; } bool colExists(TAOS_MULTI_BIND* pBind, int32_t dataType) { diff --git a/tests/script/general/parser/udf.sim b/tests/script/general/parser/udf.sim index c048a7c37a82b8591f60c209cf49cc40bdfd10f1..4b767dbf60a413945f34ce4f17aa6b19efe0bb5e 100644 --- a/tests/script/general/parser/udf.sim +++ b/tests/script/general/parser/udf.sim @@ -44,6 +44,7 @@ endi sql create function n02 as '/tmp/normal' outputtype bool; sql show functions; if $rows != 1 then + print expect 1, actual $rows return -1 endi diff --git a/tests/script/jenkins/basic.txt b/tests/script/jenkins/basic.txt index 41bbfaa9388e810f2df6e70631aade4d06f8ee3f..cc28b19de9816b751d99e63d4b8b046b3be9814a 100644 --- a/tests/script/jenkins/basic.txt +++ b/tests/script/jenkins/basic.txt @@ -12,6 +12,8 @@ ## ---- db ./test.sh -f tsim/db/create_all_options.sim ./test.sh -f tsim/db/alter_option.sim +./test.sh -f tsim/db/alter_replica_13.sim +#./test.sh -f tsim/db/alter_replica_31.sim ./test.sh -f tsim/db/basic1.sim ./test.sh -f tsim/db/basic2.sim ./test.sh -f tsim/db/basic3.sim @@ -21,8 +23,16 @@ ./test.sh -f tsim/db/taosdlog.sim # ---- dnode +./test.sh -f tsim/dnode/balance_replica1.sim ./test.sh -f tsim/dnode/create_dnode.sim ./test.sh -f tsim/dnode/drop_dnode_has_mnode.sim +./test.sh -f tsim/dnode/drop_dnode_has_qnode_snode.sim +./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica1.sim +#./test.sh -f tsim/dnode/drop_dnode_has_vnode_replica3.sim +#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim +#./test.sh -f tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim +#./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_leader.sim +./test.sh -f tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim # ---- insert ./test.sh -f tsim/insert/basic0.sim @@ -75,12 +85,13 @@ ./test.sh -f tsim/stream/distributeInterval0.sim # ./test.sh -f tsim/stream/distributesession0.sim # ./test.sh -f tsim/stream/session0.sim -# ./test.sh -f tsim/stream/session1.sim +./test.sh -f tsim/stream/session1.sim # ./test.sh -f tsim/stream/state0.sim ./test.sh -f tsim/stream/triggerInterval0.sim # ./test.sh -f tsim/stream/triggerSession0.sim ./test.sh -f tsim/stream/partitionby.sim ./test.sh -f tsim/stream/schedSnode.sim +./test.sh -f tsim/stream/windowClose.sim # ---- transaction @@ -98,6 +109,7 @@ ./test.sh -f tsim/tmq/basic4Of2Cons.sim ./test.sh -f tsim/tmq/basic2Of2ConsOverlap.sim ./test.sh -f tsim/tmq/topic.sim +./test.sh -f tsim/tmq/snapshot.sim # --- stable ./test.sh -f tsim/stable/disk.sim diff --git a/tests/script/sh/abs_max.c b/tests/script/sh/abs_max.c index cd8ba0ff15c135bdf845af57e39d5085c0fbcb20..d623adacf941e26d0de74c0c582beb7ca83c9c13 100644 --- a/tests/script/sh/abs_max.c +++ b/tests/script/sh/abs_max.c @@ -11,8 +11,8 @@ typedef struct SUdfInit{ } SUdfInit; -#define TSDB_DATA_INT_NULL 0x80000000L -#define TSDB_DATA_BIGINT_NULL 0x8000000000000000L +#define TSDB_DATA_INT_NULL 0x80000000LL +#define TSDB_DATA_BIGINT_NULL 0x8000000000000000LL void abs_max(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, int* numOfOutput, short otype, short obytes, SUdfInit* buf) { diff --git a/tests/script/sh/demo.c b/tests/script/sh/demo.c index 23d217444892e5bd8dc1c83b569dc22616c42e78..2a862c0464497536ff1e004fb8db87d41c8d32c8 100644 --- a/tests/script/sh/demo.c +++ b/tests/script/sh/demo.c @@ -17,7 +17,7 @@ typedef struct SDemo{ }SDemo; #define FLOAT_NULL 0x7FF00000 // it is an NAN -#define DOUBLE_NULL 0x7FFFFF0000000000L // it is an NAN +#define DOUBLE_NULL 0x7FFFFF0000000000LL // it is an NAN void demo(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, diff --git a/tests/script/sh/deploy.bat b/tests/script/sh/deploy.bat index 921f1611fb5ea80bbeb746b693d7c529c421ef27..f0d8c10b7e32ca51342a4bb548fd8332e4d5c350 100644 --- a/tests/script/sh/deploy.bat +++ b/tests/script/sh/deploy.bat @@ -59,48 +59,29 @@ for /f "skip=1" %%A in ( 'wmic computersystem get caption' ) do if not defined fqdn set "fqdn=%%A" -echo firstEp %fqdn%:7100 > %TAOS_CFG% +echo firstEp %fqdn%:7100 >> %TAOS_CFG% +echo secondEp %fqdn%:7200 >> %TAOS_CFG% echo fqdn %fqdn% >> %TAOS_CFG% echo serverPort %NODE% >> %TAOS_CFG% +echo supportVnodes 128 >> %TAOS_CFG% echo dataDir %DATA_DIR% >> %TAOS_CFG% echo logDir %LOG_DIR% >> %TAOS_CFG% echo debugFlag 0 >> %TAOS_CFG% -echo mDebugFlag 135 >> %TAOS_CFG% -echo sdbDebugFlag 135 >> %TAOS_CFG% -echo dDebugFlag 135 >> %TAOS_CFG% -echo vDebugFlag 135 >> %TAOS_CFG% -echo tsdbDebugFlag 135 >> %TAOS_CFG% -echo cDebugFlag 135 >> %TAOS_CFG% -echo jnidebugFlag 135 >> %TAOS_CFG% -echo odbcdebugFlag 135 >> %TAOS_CFG% -echo httpDebugFlag 135 >> %TAOS_CFG% -echo monDebugFlag 135 >> %TAOS_CFG% -echo mqttDebugFlag 135 >> %TAOS_CFG% -echo qdebugFlag 135 >> %TAOS_CFG% -echo rpcDebugFlag 135 >> %TAOS_CFG% +echo mDebugFlag 143 >> %TAOS_CFG% +echo dDebugFlag 143 >> %TAOS_CFG% +echo vDebugFlag 143 >> %TAOS_CFG% +echo tqDebugFlag 143 >> %TAOS_CFG% +echo tsdbDebugFlag 143 >> %TAOS_CFG% +echo cDebugFlag 143 >> %TAOS_CFG% +echo jniDebugFlag 143 >> %TAOS_CFG% +echo qDebugFlag 143 >> %TAOS_CFG% +echo rpcDebugFlag 143 >> %TAOS_CFG% echo tmrDebugFlag 131 >> %TAOS_CFG% -echo udebugFlag 135 >> %TAOS_CFG% -echo sdebugFlag 135 >> %TAOS_CFG% -echo wdebugFlag 135 >> %TAOS_CFG% -echo cqdebugFlag 135 >> %TAOS_CFG% -echo monitor 0 >> %TAOS_CFG% -echo monitorInterval 1 >> %TAOS_CFG% -echo http 0 >> %TAOS_CFG% -echo slaveQuery 0 >> %TAOS_CFG% -echo numOfThreadsPerCore 2.0 >> %TAOS_CFG% -echo defaultPass taosdata >> %TAOS_CFG% +echo uDebugFlag 143 >> %TAOS_CFG% +echo sDebugFlag 143 >> %TAOS_CFG% +echo wDebugFlag 143 >> %TAOS_CFG% echo numOfLogLines 20000000 >> %TAOS_CFG% -echo mnodeEqualVnodeNum 0 >> %TAOS_CFG% -echo balanceInterval 1 >> %TAOS_CFG% -echo clog 2 >> %TAOS_CFG% -echo days 10 >> %TAOS_CFG% echo statusInterval 1 >> %TAOS_CFG% -echo maxVgroupsPerDb 4 >> %TAOS_CFG% -echo minTablesPerVnode 4 >> %TAOS_CFG% -echo maxTablesPerVnode 1000 >> %TAOS_CFG% -echo tableIncStepPerVnode 10000 >> %TAOS_CFG% echo asyncLog 0 >> %TAOS_CFG% -echo numOfMnodes 1 >> %TAOS_CFG% echo locale en_US.UTF-8 >> %TAOS_CFG% -echo fsync 0 >> %TAOS_CFG% echo telemetryReporting 0 >> %TAOS_CFG% diff --git a/tests/script/sh/exec.bat b/tests/script/sh/exec.bat index 6651c7aa8f726d5d42d52a4c8f3a9395e7a12151..7a2c58fc66e5b61ce3449f3dcfacdec22612b449 100644 --- a/tests/script/sh/exec.bat +++ b/tests/script/sh/exec.bat @@ -30,9 +30,28 @@ rem echo CFG_DIR: %CFG_DIR% set TAOS_CFG=%CFG_DIR%taos.cfg rem echo TAOS_CFG: %TAOS_CFG% +set LOG_DIR=%NODE_DIR%log\ +rem echo LOG_DIR: %LOG_DIR% + +set TAOS_LOG=%LOG_DIR%taosdlog.0 +rem echo TAOS_LOG: %TAOS_LOG% + if %EXEC_OPTON% == start ( + rm -rf %TAOS_LOG% echo start %TAOSD% -c %CFG_DIR% start %TAOSD% -c %CFG_DIR% + set /a check_num=0 +:check_online + sleep 1 + set /a check_num=check_num+1 + if "%check_num%" == "11" ( + echo check online out time. + goto :finish + ) + echo check taosd online + tail -n +0 %TAOS_LOG% | grep -q "TDengine initialized successfully" || goto :check_online + echo finish + goto :finish ) if %EXEC_OPTON% == stop ( @@ -44,5 +63,8 @@ if %EXEC_OPTON% == stop ( ) do ( rem echo taskkill /IM %%A taskkill /IM %%A > NUL 2>&1 + goto :finish ) ) + +:finish \ No newline at end of file diff --git a/tests/script/sh/sum_double.c b/tests/script/sh/sum_double.c index 02979203617cf9da2b8e5b18bf326b56a45734d7..d6eea5d291c4add6bd40ae36492a1366c67f2cfd 100644 --- a/tests/script/sh/sum_double.c +++ b/tests/script/sh/sum_double.c @@ -10,7 +10,7 @@ typedef struct SUdfInit{ int const_item; /* 0 if result is independent of arguments */ } SUdfInit; -#define TSDB_DATA_INT_NULL 0x80000000L +#define TSDB_DATA_INT_NULL 0x80000000LL void sum_double(char* data, short itype, short ibytes, int numOfRows, long long* ts, char* dataOutput, char* interBuf, char* tsOutput, diff --git a/tests/script/tsim/db/alter_option.sim b/tests/script/tsim/db/alter_option.sim index fede960a6ae1709f8fc267856b2a243f6dd349f4..f24e32279d9e9782af547891a2bc7376c723b276 100644 --- a/tests/script/tsim/db/alter_option.sim +++ b/tests/script/tsim/db/alter_option.sim @@ -43,8 +43,16 @@ print ===> $data20 $data21 $data22 $data23 $data24 $data25 if $data00 != 1 then return -1 endi -if $data01 != localhost:7100 then - return -1 +system_content printf %OS% +if $system_content == Windows_NT then + system_content printf %COMPUTERNAME%:7100 + if $data01 != $system_content then + return -1 + endi +else + if $data01 != localhost:7100 then + return -1 + endi endi if $data04 != ready then goto check_dnode_ready_1 diff --git a/tests/script/tsim/db/create_all_options.sim b/tests/script/tsim/db/create_all_options.sim index efe7ff99cfdd7bfb79c614dbaafd021b004cdfdc..3979ef1ec1a82ec822d6ac6724046c70037b98ba 100644 --- a/tests/script/tsim/db/create_all_options.sim +++ b/tests/script/tsim/db/create_all_options.sim @@ -43,8 +43,16 @@ print ===> $data20 $data21 $data22 $data23 $data24 $data25 if $data00 != 1 then return -1 endi -if $data01 != localhost:7100 then - return -1 +system_content printf %OS% +if $system_content == Windows_NT then + system_content printf %COMPUTERNAME%:7100 + if $data01 != $system_content then + return -1 + endi +else + if $data01 != localhost:7100 then + return -1 + endi endi if $data04 != ready then goto check_dnode_ready_1 diff --git a/tests/script/tsim/dnode/balance_replica1.sim b/tests/script/tsim/dnode/balance_replica1.sim new file mode 100644 index 0000000000000000000000000000000000000000..14f3f130fbd17083756aa29b9347d872efcaa426 --- /dev/null +++ b/tests/script/tsim/dnode/balance_replica1.sim @@ -0,0 +1,123 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +# no enough vnodes +sql balance vgroup + +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not online! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != offline then + goto step1 +endi + +print =============== step2 create database +sql create database d1 vgroups 2 +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql create table d1.c2 using st tags(1) +sql create table d1.c3 using st tags(1) +sql create table d1.c4 using st tags(1) +sql create table d1.c5 using st tags(1) +sql create table d1.c6 using st tags(1) +sql show d1.tables +if $rows != 6 then + return -1 +endi + +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 2 then + return -1 +endi +if $data(2)[3] != 2 then + return -1 +endi +if $data(3)[3] != 2 then + return -1 +endi + +print =============== step3: balance vgroup +# has offline dnode +sql_error balance vgroup + +system sh/exec.sh -n dnode3 -s start +$x = 0 +step3: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ---> dnode not online! + return -1 + endi +sql show dnodes +print ---> $data00 $data01 $data02 $data03 $data04 $data05 +print ---> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step3 +endi +if $data(2)[4] != ready then + goto step3 +endi +if $data(3)[4] != ready then + goto step3 +endi + +print =============== step4: balance +sql balance vgroup + +print show d1.vgroups +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 2 then + return -1 +endi +if $data(2)[3] != 3 then + return -1 +endi +if $data(3)[3] != 2 then + return -1 +endi + +print =============== step7: select data +sql show d1.tables +print rows $rows +if $rows != 6 then + return -1 +endi + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim index 1dbec2c04d0db3974a55cf7bccb757dc99f35016..16cd2bb3b0a6eb5537c9461611feefa3c2a51b00 100644 --- a/tests/script/tsim/dnode/drop_dnode_has_mnode.sim +++ b/tests/script/tsim/dnode/drop_dnode_has_mnode.sim @@ -70,6 +70,7 @@ if $data(2)[2] != follower then endi print =============== step4: drop dnode 2 +sql_error drop dnode 1 sql drop dnode 2 print show dnodes; @@ -83,7 +84,7 @@ if $data00 != 1 then return -1 endi -print show dnodes; +print show mnodes; sql show mnodes print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim new file mode 100644 index 0000000000000000000000000000000000000000..15ad13b6fa4208d4cac3eca5ad5427328d09ea6b --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica1.sim @@ -0,0 +1,135 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != offline then + goto step1 +endi + +print =============== step2 create database +sql create database d1 vgroups 4 +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 4 then + return -1 +endi +if $data(2)[3] != 2 then + return -1 +endi +if $data(3)[3] != 2 then + return -1 +endi +if $data(4)[3] != 2 then + return -1 +endi +if $data(5)[3] != 2 then + return -1 +endi + +print =============== step3: start dnode 3 +system sh/exec.sh -n dnode3 -s start +$x = 0 +step4: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != ready then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi + +print =============== step3: drop dnode2 +sql drop dnode 2 + +print show dnodes; +sql show dnodes; +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 2 then + return -1 +endi + +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 4 then + return -1 +endi +if $data(2)[3] != 3 then + return -1 +endi +if $data(3)[3] != 3 then + return -1 +endi +if $data(4)[3] != 3 then + return -1 +endi +if $data(5)[3] != 3 then + return -1 +endi + +print =============== step4: select data +sql show d1.tables +if $rows != 1 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim new file mode 100644 index 0000000000000000000000000000000000000000..35a246629789a629b86a75ce7702097c174fd832 --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_has_multi_vnode_replica3.sim @@ -0,0 +1,135 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1 create dnode2 dnode3 dnode4 dnode 5 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi +if $data(5)[4] != offline then + goto step1 +endi + +print =============== step3 create database +sql create database d1 vgroups 4 replica 3 +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 1 then + return -1 +endi +if $data(2)[3] != 2 then + return -1 +endi +if $data(2)[5] != 3 then + return -1 +endi +if $data(2)[7] != 4 then + return -1 +endi + +print =============== step4: drop dnode 2 +system sh/exec.sh -n dnode5 -s start +$x = 0 +step4: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != ready then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi +if $data(4)[4] != ready then + goto step4 +endi +if $data(5)[4] != ready then + goto step4 +endi + +print =============== step5: drop dnode2 +sql drop dnode 2 + +print show dnodes; +sql show dnodes; +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 4 then + return -1 +endi + +print show d1.vgroups +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 1 then + return -1 +endi + + +print =============== step6: select data +sql show d1.tables +if $rows != 1 then + return -1 +endi + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_qnode_snode.sim b/tests/script/tsim/dnode/drop_dnode_has_qnode_snode.sim new file mode 100644 index 0000000000000000000000000000000000000000..6641ce053ccbf5bdd10984793acd699050207fcd --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_has_qnode_snode.sim @@ -0,0 +1,71 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 2 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi + +print =============== step2: create qnode snode on dnode 2 +sql create qnode on dnode 2 +sql create snode on dnode 2 + +sql show qnodes +if $rows != 1 then + return -1 +endi + +sql show snodes +if $rows != 1 then + return -1 +endi + +print =============== step3: drop dnode 2 +sql drop dnode 2 + +print show dnodes; +sql show dnodes; +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 1 then + return -1 +endi +if $data00 != 1 then + return -1 +endi + +sql show qnodes +if $rows != 0 then + return -1 +endi + +sql show snodes +if $rows != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica1.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica1.sim new file mode 100644 index 0000000000000000000000000000000000000000..63639877ee6108f1a6f9a337a4eb3b560b70c479 --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica1.sim @@ -0,0 +1,151 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +sql connect + +print =============== step1 create dnode2 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != offline then + goto step1 +endi + +print =============== step2 drop dnode which not exist +sql_error drop dnode 1 +sql_error drop dnode 4 + +print =============== step3 create database +sql create database d1 vgroups 1 +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 1 then + return -1 +endi +if $data(2)[3] != 2 then + return -1 +endi + +print =============== step4: drop dnode 2 +print cant drop since no enough vnodes +sql_error drop dnode 2 + +system sh/exec.sh -n dnode2 -s stop +system sh/exec.sh -n dnode3 -s start +$x = 0 +step4: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != offline then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi + +# cant drop since offline +sql_error drop dnode 2 + +print =============== step5: start dnode2 +system sh/exec.sh -n dnode2 -s start + +$x = 0 +step5: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step5 +endi +if $data(2)[4] != ready then + goto step5 +endi +if $data(3)[4] != ready then + goto step5 +endi + +print =============== step6: drop dnode2 +sql drop dnode 2 + +print show dnodes; +sql show dnodes; +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 2 then + return -1 +endi + +print show d1.vgroups +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 1 then + return -1 +endi +if $data(2)[3] != 3 then + return -1 +endi + +print =============== step7: select data +sql show d1.tables +if $rows != 1 then + return -1 +endi + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim new file mode 100644 index 0000000000000000000000000000000000000000..3ea351f7a73eb93d61cae26892db64b0854875c5 --- /dev/null +++ b/tests/script/tsim/dnode/drop_dnode_has_vnode_replica3.sim @@ -0,0 +1,144 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/deploy.sh -n dnode2 -i 2 +system sh/deploy.sh -n dnode3 -i 3 +system sh/deploy.sh -n dnode4 -i 4 +system sh/deploy.sh -n dnode5 -i 5 +system sh/cfg.sh -n dnode1 -c supportVnodes -v 0 +system sh/exec.sh -n dnode1 -s start +system sh/exec.sh -n dnode2 -s start +system sh/exec.sh -n dnode3 -s start +system sh/exec.sh -n dnode4 -s start +sql connect + +print =============== step1 create dnode2 dnode3 dnode4 dnode 5 +sql create dnode $hostname port 7200 +sql create dnode $hostname port 7300 +sql create dnode $hostname port 7400 +sql create dnode $hostname port 7500 + +$x = 0 +step1: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 5 then + return -1 +endi +if $data(1)[4] != ready then + goto step1 +endi +if $data(2)[4] != ready then + goto step1 +endi +if $data(3)[4] != ready then + goto step1 +endi +if $data(4)[4] != ready then + goto step1 +endi +if $data(5)[4] != offline then + goto step1 +endi + +print =============== step3 create database +sql create database d1 vgroups 1 replica 3 +sql use d1 +sql create table d1.st (ts timestamp, i int) tags (j int) +sql create table d1.c1 using st tags(1) +sql show d1.tables +if $rows != 1 then + return -1 +endi + +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 1 then + return -1 +endi +if $data(2)[3] != 2 then + return -1 +endi +if $data(2)[5] != 3 then + return -1 +endi +if $data(2)[7] != 4 then + return -1 +endi + +system sh/exec.sh -n dnode4 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT + + +return + + +print =============== step4: drop dnode 2 +system sh/exec.sh -n dnode5 -s start +$x = 0 +step4: + $ = $x + 1 + sleep 1000 + if $x == 10 then + print ====> dnode not online! + return -1 + endi +sql show dnodes +print ===> $data00 $data01 $data02 $data03 $data04 $data05 +print ===> $data10 $data11 $data12 $data13 $data14 $data15 +if $rows != 3 then + return -1 +endi +if $data(1)[4] != ready then + goto step4 +endi +if $data(2)[4] != ready then + goto step4 +endi +if $data(3)[4] != ready then + goto step4 +endi +if $data(4)[4] != ready then + goto step4 +endi +if $data(5)[4] != ready then + goto step4 +endi + +print =============== step5: drop dnode2 +sql drop dnode 2 + +print show dnodes; +sql show dnodes; +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +print $data[1][0] $data[1][1] $data[1][2] $data[1][3] $data[1][4] +if $rows != 4 then + return -1 +endi + +print show d1.vgroups +sql show d1.vgroups +print $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] +if $rows != 1 then + return -1 +endi +#if $data(2)[3] != 3 then +# return -1 +#endi + +print =============== step6: select data +sql show d1.tables +if $rows != 1 then + return -1 +endi + +return +system sh/exec.sh -n dnode1 -s stop -x SIGINT +system sh/exec.sh -n dnode2 -s stop -x SIGINT +system sh/exec.sh -n dnode3 -s stop -x SIGINT diff --git a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim index 3532332174db83b2f6fecff6005d405f06a6a09b..788a23991a96a4111e95588714283358a73da1c6 100644 --- a/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim +++ b/tests/script/tsim/dnode/redistribute_vgroup_replica3_v1_follower.sim @@ -176,6 +176,8 @@ if $rows != 1 then return -1 endi +return + print =============== step33: move follower1 print redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 sql redistribute vgroup 2 dnode $leaderVnode dnode $follower1 dnode 5 diff --git a/tests/script/tsim/mnode/basic5.sim b/tests/script/tsim/mnode/basic5.sim index fc591aa25d3677ebfaf6e982555ab09512badad8..23f5f6d7826661c076f56edf874f1cbc0cc20d42 100644 --- a/tests/script/tsim/mnode/basic5.sim +++ b/tests/script/tsim/mnode/basic5.sim @@ -148,7 +148,7 @@ endi print =============== step6: stop mnode1 system sh/exec.sh -n dnode1 -s stop -x SIGKILL -sql_error drop mnode on dnode 1 +# sql_error drop mnode on dnode 1 $x = 0 step61: diff --git a/tests/script/tsim/query/udf.sim b/tests/script/tsim/query/udf.sim index d9821a04955e6d4f5f80db84afa8531762432672..4e95095172befd74914009cf17cdaffef54d48f2 100644 --- a/tests/script/tsim/query/udf.sim +++ b/tests/script/tsim/query/udf.sim @@ -44,6 +44,7 @@ endi sql select udf2(f) from t; if $rows != 1 then + print expect 1, actual $rows return -1 endi if $data00 != 2.236067977 then diff --git a/tests/script/tsim/show/basic.sim b/tests/script/tsim/show/basic.sim index f23d75d78b432d5f69391b18a5bb0318e93af08b..4d646f39e38dca6844def98eb0b960f8309a2965 100644 --- a/tests/script/tsim/show/basic.sim +++ b/tests/script/tsim/show/basic.sim @@ -36,6 +36,7 @@ sql create database db vgroups 3 sql use db sql create table stb (ts timestamp, c int) tags (t int) sql create table t0 using stb tags (0) +sql create table tba (ts timestamp, c1 binary(10), c2 nchar(10)); print =============== run show xxxx sql show dnodes @@ -62,7 +63,7 @@ if $rows != 1 then endi #sql show streams, sql show tables -if $rows != 1 then +if $rows != 2 then return -1 endi #sql show user_table_distributed @@ -98,7 +99,7 @@ if $rows != 1 then endi #sql select * from information_schema.`streams` sql select * from information_schema.user_tables -if $rows != 29 then +if $rows != 31 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -160,7 +161,7 @@ if $rows != 1 then endi #sql show streams, sql show tables -if $rows != 1 then +if $rows != 2 then return -1 endi #sql show user_table_distributed @@ -196,7 +197,7 @@ if $rows != 1 then endi #sql select * from performance_schema.`streams` sql select * from information_schema.user_tables -if $rows != 29 then +if $rows != 31 then return -1 endi #sql select * from information_schema.user_table_distributed @@ -209,5 +210,37 @@ if $rows != 3 then return -1 endi +sql show create stable stb; +if $rows != 1 then + return -1 +endi + +sql show create table t0; +if $rows != 1 then + return -1 +endi + +sql show create table tba; +if $rows != 1 then + return -1 +endi + +sql_error show create stable t0; + +sql show variables; +if $rows != 4 then + return -1 +endi + +sql show dnode 1 variables; +if $rows <= 0 then + return -1 +endi + +sql show local variables; +if $rows <= 0 then + return -1 +endi + system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode2 -s stop -x SIGINT diff --git a/tests/script/tsim/stream/windowClose.sim b/tests/script/tsim/stream/windowClose.sim index 07d7fb794e75647f147b189dfffff193b4e80d6d..695d5749faeddc08fa6f8376600b6cbcb188edd5 100644 --- a/tests/script/tsim/stream/windowClose.sim +++ b/tests/script/tsim/stream/windowClose.sim @@ -22,7 +22,7 @@ sql create stream stream1 trigger window_close into streamt as select _wstartts sql insert into tu1 values(now, 1); -sleep 300 +sleep 500 sql select * from streamt; if $rows != 0 then print ======$rows diff --git a/tests/script/tsim/tmq/consume.sh b/tests/script/tsim/tmq/consume.sh index 3fa71d6edd96b6c664d5c680fb2615c730e14f8e..001ce6ae496cc7e5f7b4c5a3bf5b92b0c7cf12b3 100755 --- a/tests/script/tsim/tmq/consume.sh +++ b/tests/script/tsim/tmq/consume.sh @@ -17,8 +17,9 @@ VALGRIND=0 SIGNAL=SIGINT SHOW_MSG=0 SHOW_ROW=0 +EXP_USE_SNAPSHOT=0 -while getopts "d:s:v:y:x:g:r:w:" arg +while getopts "d:s:v:y:x:g:r:w:e:" arg do case $arg in d) @@ -45,6 +46,9 @@ do w) CDB_NAME=$OPTARG ;; + e) + EXP_USE_SNAPSHOT=$OPTARG + ;; ?) echo "unkown argument" ;; @@ -91,8 +95,8 @@ if [ "$EXEC_OPTON" = "start" ]; then echo nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 & nohup valgrind --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all -v --workaround-gcc296-bugs=yes --log-file=${LOG_DIR}/valgrind-tmq_sim.log $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW > /dev/null 2>&1 & else - echo "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 &" - nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME > /dev/null 2>&1 & + echo "nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME -e $EXP_USE_SNAPSHOT > /dev/null 2>&1 &" + nohup $PROGRAM -c $CFG_DIR -y $POLL_DELAY -d $DB_NAME -g $SHOW_MSG -r $SHOW_ROW -w $CDB_NAME -e $EXP_USE_SNAPSHOT > /dev/null 2>&1 & fi else PID=`ps -ef|grep tmq_sim | grep -v grep | awk '{print $2}'` diff --git a/tests/script/tsim/tmq/snapshot.sim b/tests/script/tsim/tmq/snapshot.sim new file mode 100644 index 0000000000000000000000000000000000000000..5683aaa55987a07ba09a3fc88b9da613792c3321 --- /dev/null +++ b/tests/script/tsim/tmq/snapshot.sim @@ -0,0 +1,289 @@ +#### test scenario, please refer to https://jira.taosdata.com:18090/pages/viewpage.action?pageId=135120406 +#basic1.sim: vgroups=1, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic2.sim: vgroups=1, multi topics for one consumer, firstly insert data, then start consume. Include six topics +#basic3.sim: vgroups=4, one topic for one consumer, firstly insert data, then start consume. Include six topics +#basic4.sim: vgroups=4, multi topics for one consumer, firstly insert data, then start consume. Include six topics + +# notes1: Scalar function: ABS/ACOS/ASIN/ATAN/CEIL/COS/FLOOR/LOG/POW/ROUND/SIN/SQRT/TAN +# The above use cases are combined with where filter conditions, such as: where ts > "2017-08-12 18:25:58.128Z" and sin(a) > 0.5; +# +# notes2: not support aggregate functions(such as sum/count/min/max) and time-windows(interval). +# + +run tsim/tmq/prepareBasicEnv-1vgrp.sim + +#---- global parameters start ----# +$dbName = db +$vgroups = 1 +$stbPrefix = stb +$ctbPrefix = ctb +$ntbPrefix = ntb +$stbNum = 1 +$ctbNum = 10 +$ntbNum = 10 +$rowsPerCtb = 10 +$tstart = 1640966400000 # 2022-01-01 00:00:00.000 +#---- global parameters end ----# + +$pullDelay = 3 +$ifcheckdata = 1 +$ifmanualcommit = 1 +$showMsg = 1 +$showRow = 0 + +sql connect +sql use $dbName + +print == create topics from super table +sql create topic topic_stb_column as select ts, c3 from stb +sql create topic topic_stb_all as select ts, c1, c2, c3 from stb +sql create topic topic_stb_function as select ts, abs(c1), sin(c2) from stb + +print == create topics from child table +sql create topic topic_ctb_column as select ts, c3 from ctb0 +sql create topic topic_ctb_all as select * from ctb0 +sql create topic topic_ctb_function as select ts, abs(c1), sin(c2) from ctb0 + +print == create topics from normal table +sql create topic topic_ntb_column as select ts, c3 from ntb0 +sql create topic topic_ntb_all as select * from ntb0 +sql create topic topic_ntb_function as select ts, abs(c1), sin(c2) from ntb0 + +#sql show topics +#if $rows != 9 then +# return -1 +#endi + +#'group.id:cgrp1,enable.auto.commit:false,auto.commit.interval.ms:6000,auto.offset.reset:earliest' +$keyList = ' . group.id:cgrp1 +$keyList = $keyList . , +$keyList = $keyList . enable.auto.commit:false +#$keyList = $keyList . , +#$keyList = $keyList . auto.commit.interval.ms:6000 +#$keyList = $keyList . , +#$keyList = $keyList . auto.offset.reset:earliest +$keyList = $keyList . ' +print ========== key list: $keyList + + +$cdb_index = 0 +#=============================== start consume =============================# + +print ================ test consume from stb +$loop_cnt = 0 +loop_consume_diff_topic_from_stb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_stb_column + $topicList = ' . topic_stb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_stb_all + $topicList = ' . topic_stb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_stb_function + $topicList = ' . topic_stb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_stb_end +endi + +$consumerId = 0 +$totalMsgOfStb = $ctbNum * $rowsPerCtb +$expectmsgcnt = 1 +$expectrowcnt = 100 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from stb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -e 1 -s start +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -e 1 -s start + +print == check consume result +wait_consumer_end_from_stb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_stb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != $expectmsgcnt then + return -1 +endi +if $data[0][3] != $expectrowcnt then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_stb +loop_consume_diff_topic_from_stb_end: + +print ================ test consume from ctb +$loop_cnt = 0 +loop_consume_diff_topic_from_ctb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ctb_column + $topicList = ' . topic_ctb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ctb_all + $topicList = ' . topic_ctb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ctb_function + $topicList = ' . topic_ctb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ctb_end +endi + +$consumerId = 0 +$totalMsgOfCtb = $rowsPerCtb +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ctb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -e 1 +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -e 1 + +print == check consume result +wait_consumer_end_from_ctb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ctb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != 1 then + return -1 +endi +if $data[0][3] != 10 then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ctb +loop_consume_diff_topic_from_ctb_end: + +print ================ test consume from ntb +$loop_cnt = 0 +loop_consume_diff_topic_from_ntb: + +####################################################################################### +# clear consume info and consume result +#run tsim/tmq/clearConsume.sim +# because drop table function no stable, so by create new db for consume info and result. Modify it later +$cdb_index = $cdb_index + 1 +$cdbName = cdb . $cdb_index +sql create database $cdbName vgroups 1 +sleep 500 +sql use $cdbName + +print == create consume info table and consume result table +sql create table consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int) +sql create table consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int) + +sql show tables +if $rows != 2 then + return -1 +endi +####################################################################################### + +if $loop_cnt == 0 then + print == scenario 1: topic_ntb_column + $topicList = ' . topic_ntb_column + $topicList = $topicList . ' +elif $loop_cnt == 1 then + print == scenario 2: topic_ntb_all + $topicList = ' . topic_ntb_all + $topicList = $topicList . ' +elif $loop_cnt == 2 then + print == scenario 3: topic_ntb_function + $topicList = ' . topic_ntb_function + $topicList = $topicList . ' +else + goto loop_consume_diff_topic_from_ntb_end +endi + +$consumerId = 0 +$totalMsgOfNtb = $rowsPerCtb +$expectmsgcnt = $totalMsgOfNtb +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) + +print == start consumer to pull msgs from ntb +print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start -e 1 +system tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start -e 1 + +print == check consume result from ntb +wait_consumer_end_from_ntb: +sql select * from consumeresult +print ==> rows: $rows +print ==> rows[0]: $data[0][0] $data[0][1] $data[0][2] $data[0][3] $data[0][4] $data[0][5] $data[0][6] +if $rows != 1 then + sleep 1000 + goto wait_consumer_end_from_ntb +endi +if $data[0][1] != $consumerId then + return -1 +endi +if $data[0][2] != 1 then + return -1 +endi +if $data[0][3] != $totalMsgOfNtb then + return -1 +endi +$loop_cnt = $loop_cnt + 1 +goto loop_consume_diff_topic_from_ntb +loop_consume_diff_topic_from_ntb_end: + +#------ not need stop consumer, because it exit after pull msg overthan expect msg +#system tsim/tmq/consume.sh -s stop -x SIGINT + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/trans/create_db.sim b/tests/script/tsim/trans/create_db.sim index f730f2fb676e32fa0652a9ef4f6dfe2e250b5430..d4d158ede0b9d0023246095f7797e89a6863b87f 100644 --- a/tests/script/tsim/trans/create_db.sim +++ b/tests/script/tsim/trans/create_db.sim @@ -96,50 +96,53 @@ sql_error create database d1 vgroups 2; print =============== kill dnode2 system sh/exec.sh -n dnode2 -s stop -x SIGINT -print =============== create database -sql show transactions -if $rows != 0 then - return -1 -endi - -sql_error create database d2 vgroups 2; - -print =============== show transactions -sql show transactions -if $rows != 1 then - return -1 -endi - -if $data[0][0] != 8 then - return -1 -endi - -if $data[0][2] != redoAction then - return -1 -endi - -if $data[0][3] != d2 then - return -1 -endi - -sql show databases ; -if $rows != 4 then - return -1 -endi -print d2 ==> $data(d2)[19] -if $data(d2)[19] != creating then - return -1 -endi - -sql_error create database d2 vgroups 2; - -print =============== kill transaction -sql kill transaction 8; -sleep 2000 - -sql show transactions -if $rows != 0 then - return -1 +system_content printf %OS% +if $system_content != Windows_NT then + print =============== create database + sql show transactions + if $rows != 0 then + return -1 + endi + + sql_error create database d2 vgroups 2; + + print =============== show transactions + sql show transactions + if $rows != 1 then + return -1 + endi + + if $data[0][0] != 8 then + return -1 + endi + + if $data[0][2] != redoAction then + return -1 + endi + + if $data[0][3] != d2 then + return -1 + endi + + sql show databases ; + if $rows != 4 then + return -1 + endi + print d2 ==> $data(d2)[19] + if $data(d2)[19] != creating then + return -1 + endi + + sql_error create database d2 vgroups 2; + + print =============== kill transaction + sql kill transaction 8; + sleep 2000 + + sql show transactions + if $rows != 0 then + return -1 + endi endi print =============== start dnode2 @@ -153,25 +156,25 @@ step3: print ====> dnode not ready! return -1 endi -sql show dnodes -print ===> $data00 $data01 $data02 $data03 $data04 $data05 -print ===> $data10 $data11 $data12 $data13 $data14 $data15 -if $rows != 2 then - return -1 -endi -if $data(1)[4] != ready then - goto step3 -endi -if $data(2)[4] != ready then - goto step3 -endi + sql show dnodes + print ===> $data00 $data01 $data02 $data03 $data04 $data05 + print ===> $data10 $data11 $data12 $data13 $data14 $data15 + if $rows != 2 then + return -1 + endi + if $data(1)[4] != ready then + goto step3 + endi + if $data(2)[4] != ready then + goto step3 + endi sql show transactions if $rows != 0 then return -1 endi -sql drop database d2; +sql drop database if exists d2; sql show transactions if $rows != 0 then diff --git a/tests/script/wtest.bat b/tests/script/wtest.bat index 9ed58a90d1ba6bd781d306070e03f56b75984093..1f089c9d869b8add918291e0807aed45063cf797 100644 --- a/tests/script/wtest.bat +++ b/tests/script/wtest.bat @@ -52,6 +52,7 @@ echo wal 0 >> %TAOS_CFG% echo asyncLog 0 >> %TAOS_CFG% echo locale en_US.UTF-8 >> %TAOS_CFG% echo enableCoreFile 1 >> %TAOS_CFG% +echo charset UTF-8 >> %TAOS_CFG% set "FILE_NAME=testSuite.sim" if "%1" == "-f" set "FILE_NAME=%2" diff --git a/tests/system-test/1-insert/create_retentions.py b/tests/system-test/1-insert/create_retentions.py index 313c643822537d19f3f54570e829a711927ec798..5a0684e2ee661eace0d8d9c747b240f6782a1ee4 100644 --- a/tests/system-test/1-insert/create_retentions.py +++ b/tests/system-test/1-insert/create_retentions.py @@ -1,5 +1,7 @@ import datetime +from dataclasses import dataclass +from typing import List from util.log import * from util.sql import * from util.cases import * @@ -28,6 +30,28 @@ CHAR_COL = [ BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [ BOOL_COL, ] TS_TYPE_COL = [ TS_COL, ] +## insert data args： +TIME_STEP = 10000 +NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + +@dataclass +class DataSet: + ts_data : List[int] = None + int_data : List[int] = None + bint_data : List[int] = None + sint_data : List[int] = None + tint_data : List[int] = None + int_un_data : List[int] = None + bint_un_data : List[int] = None + sint_un_data : List[int] = None + tint_un_data : List[int] = None + float_data : List[float] = None + double_data : List[float] = None + bool_data : List[int] = None + binary_data : List[str] = None + nchar_data : List[str] = None + + class TDTestCase: def init(self, conn, logSql): @@ -37,10 +61,13 @@ class TDTestCase: @property def create_databases_sql_err(self): return [ - "create database if not exists db1 retentions 0s:1d", - "create database if not exists db1 retentions 1s:1y", - "create database if not exists db1 retentions 1s:1n", - "create database if not exists db1 retentions 1s:1n,2s:2d,3s:3d,4s:4d", + "create database db1 retentions 0s:1d", + "create database db3 retentions 1s:0d", + "create database db1 retentions 1s:1y", + "create database db1 retentions 1s:1n", + "create database db2 retentions 1w:1d ;", + "create database db5 retentions 1s:1d,3s:3d,2s:2d", + "create database db1 retentions 1s:1n,2s:2d,3s:3d,4s:4d", ] @property @@ -60,16 +87,41 @@ class TDTestCase: @property def create_stable_sql_err(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) delay 1", - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) delay 1", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) delay 1", - f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) delay 1", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(ceil) watermark 1s maxdelay 1m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(count) watermark 1min", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay -1s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark -1m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) watermark 1m ", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) maxdelay 1m ", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} binary(16)) tags (tag1 int) rollup(avg) watermark 1s", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) rollup(avg) maxdelay 1m", + # f"create table ntb_1 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) rollup(avg) watermark 1s maxdelay 1s", + # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) tags (tag1 int) " , + # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) " , + # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) " , + # f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int, {BINARY_COL} nchar(16)) " , + + # watermark, maxdelay: [0, 900000], [ms, s, m, ?] + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1u", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 1b", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 900001ms", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 16m", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 901s", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 1h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) maxdelay 0.2h", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 0.002d", + ] @property def create_stable_sql_current(self): return [ - f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg) delay 5", + f"create stable stb1 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(avg)", + f"create stable stb2 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(min) watermark 5s maxdelay 1m", + f"create stable stb3 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(max) watermark 5s maxdelay 1m", + f"create stable stb4 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(sum) watermark 5s maxdelay 1m", + # f"create stable stb5 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(last) watermark 5s maxdelay 1m", + # f"create stable stb6 ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) rollup(first) watermark 5s maxdelay 1m", ] def test_create_stb(self): @@ -79,14 +131,19 @@ class TDTestCase: for cur_sql in self.create_stable_sql_current: tdSql.execute(cur_sql) tdSql.query("show stables") + # assert "rollup" in tdSql.description tdSql.checkRows(len(self.create_stable_sql_current)) + # tdSql.execute("use db") # because db is a noraml database, not a rollup database, should not be able to create a rollup database + # tdSql.error(f"create stable nor_db_rollup_stb ({PRIMARY_COL} timestamp, {INT_COL} int) tags (tag1 int) file_factor 5.0") + + def test_create_databases(self): for err_sql in self.create_databases_sql_err: tdSql.error(err_sql) for cur_sql in self.create_databases_sql_current: tdSql.execute(cur_sql) - tdSql.query("show databases") + # tdSql.query("show databases") for alter_sql in self.alter_database_sql: tdSql.error(alter_sql) @@ -95,9 +152,7 @@ class TDTestCase: self.test_create_stb() def __create_tb(self): - tdSql.prepare() - - tdLog.printNoPrefix("==========step1:create table") + tdLog.printNoPrefix("==========step: create table") create_stb_sql = f'''create table stb1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, @@ -120,129 +175,118 @@ class TDTestCase: for i in range(4): tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - def __create_data_set(self, rows): - now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) - pos_data = [] - neg_data = [] - spec_data = [] - for i in range(rows): - pos_data.append( - ( - now_time - i * 1000, i, 11111 * i, 111 * i % 32767 , 11 * i % 127, 1.11 * i, 1100.0011 * i, - i % 2, f'binary{i}', f'nchar_测试_{i}', now_time + 1 * i, 11 * i % 127, 111 * i % 32767, i, 11111 * i - ) - ) - neg_data.append( - ( - now_time - i * 7776000000, -i, -11111 * i, -111 * i % 32767, -11 * i % 127, -1.11 * i, -1100.0011 * i, - i % 2, f'binary{i}', f'nchar_测试_{i}', now_time + 1 * i, 11 * i % 127, 111 * i % 32767, i, 11111 * i - ) - ) + def __data_set(self, rows): + data_set = DataSet() + # neg_data_set = DataSet() + data_set.ts_data = [] + data_set.int_data = [] + data_set.bint_data = [] + data_set.sint_data = [] + data_set.tint_data = [] + data_set.int_un_data = [] + data_set.bint_un_data = [] + data_set.sint_un_data = [] + data_set.tint_un_data = [] + data_set.float_data = [] + data_set.double_data = [] + data_set.bool_data = [] + data_set.binary_data = [] + data_set.nchar_data = [] - def __insert_data(self, rows): - now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) for i in range(rows): - tdSql.execute( - f'''insert into ct1 values ( - { now_time - i * 1000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, - {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {11 * i % 127}, {111 * i % 32767}, {i}, {11111 * i} )''' - ) - tdSql.execute( - f'''insert into ct4 values ( - { now_time - i * 7776000000 }, {i}, {11111 * i}, {111 * i % 32767 }, {11 * i % 127}, {1.11*i}, {1100.0011*i}, - {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {11 * i % 127}, {111 * i % 32767}, {i}, {11111 * i} )''' - ) - tdSql.execute( - f'''insert into ct2 values ( - { now_time - i * 7776000000 }, {-i}, {-11111 * i}, {-111 * i % 32767 }, {-11 * i % 127}, {-1.11*i}, {-1100.0011*i}, - {i%2}, 'binary{i}', 'nchar_测试_{i}', { now_time + 1 * i }, {11 * i % 127}, {111 * i % 32767}, {i}, {11111 * i} )''' - ) - tdSql.execute( - f'''insert into ct1 values - ( { now_time - rows * 5 }, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar_测试_0', { now_time + 8 }, 0, 0, 0, 0) - ( { now_time + 10000 }, { rows }, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar_测试_9', { now_time + 9 }, 0, 0, 0, 0 ) - ''' - ) - - tdSql.execute( - f'''insert into ct4 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( - { now_time + 5184000000}, {pow(2,31)-pow(2,15)}, {pow(2,63)-pow(2,30)}, 32767, 127, - { 3.3 * pow(10,38) }, { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000}, - 254, 65534, {pow(2,32)-pow(2,16)}, {pow(2,64)-pow(2,31)} - ) - ( - { now_time + 2592000000 }, {pow(2,31)-pow(2,16)}, {pow(2,63)-pow(2,31)}, 32766, 126, - { 3.2 * pow(10,38) }, { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000}, - 255, 65535, {pow(2,32)-pow(2,15)}, {pow(2,64)-pow(2,30)} - ) + data_set.ts_data.append(NOW + 1 * (rows - i)) + data_set.int_data.append(rows - i) + data_set.bint_data.append(11111 * (rows - i)) + data_set.sint_data.append(111 * (rows - i) % 32767) + data_set.tint_data.append(11 * (rows - i) % 127) + data_set.int_un_data.append(rows - i) + data_set.bint_un_data.append(11111 * (rows - i)) + data_set.sint_un_data.append(111 * (rows - i) % 32767) + data_set.tint_un_data.append(11 * (rows - i) % 127) + data_set.float_data.append(1.11 * (rows - i)) + data_set.double_data.append(1100.0011 * (rows - i)) + data_set.bool_data.append((rows - i) % 2) + data_set.binary_data.append(f'binary{(rows - i)}') + data_set.nchar_data.append(f'nchar_测试_{(rows - i)}') + + # neg_data_set.ts_data.append(-1 * i) + # neg_data_set.int_data.append(-i) + # neg_data_set.bint_data.append(-11111 * i) + # neg_data_set.sint_data.append(-111 * i % 32767) + # neg_data_set.tint_data.append(-11 * i % 127) + # neg_data_set.int_un_data.append(-i) + # neg_data_set.bint_un_data.append(-11111 * i) + # neg_data_set.sint_un_data.append(-111 * i % 32767) + # neg_data_set.tint_un_data.append(-11 * i % 127) + # neg_data_set.float_data.append(-1.11 * i) + # neg_data_set.double_data.append(-1100.0011 * i) + # neg_data_set.binary_data.append(f'binary{i}') + # neg_data_set.nchar_data.append(f'nchar_测试_{i}') + + return data_set + + def __insert_data(self): + data = self.__data_set(rows=self.rows) + + # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + null_data = '''null, null, null, null, null, null, null, null, null, null, null, null, null, null''' + zero_data = "0, 0, 0, 0, 0, 0, 0, 'binary_0', 'nchar_0', 0, 0, 0, 0, 0" + + for i in range(self.rows): + row_data = f''' + {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, + {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, + {data.sint_un_data[i]}, {data.int_un_data[i]}, {data.bint_un_data[i]} ''' - ) - - tdSql.execute( - f'''insert into ct2 values - ( { now_time - rows * 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - rows * 3888000000 + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time + 7776000000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( - { now_time + 5184000000 }, { -1 * pow(2,31) + pow(2,15) }, { -1 * pow(2,63) + pow(2,30) }, -32766, -126, { -1 * 3.2 * pow(10,38) }, - { -1.2 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }, 1, 1, 1, 1 - ) - ( - { now_time + 2592000000 }, { -1 * pow(2,31) + pow(2,16) }, { -1 * pow(2,63) + pow(2,31) }, -32767, -127, { - 3.3 * pow(10,38) }, - { -1.3 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }, 1, 1, 1, 1 - ) + neg_row_data = f''' + {-1 * data.int_data[i]}, {-1 * data.bint_data[i]}, {-1 * data.sint_data[i]}, {-1 * data.tint_data[i]}, {-1 * data.float_data[i]}, {-1 * data.double_data[i]}, + {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, + {1 * data.sint_un_data[i]}, {1 * data.int_un_data[i]}, {1 * data.bint_un_data[i]} ''' - ) - for i in range(rows): - insert_data = f'''insert into t1 values - ( { now_time - i * 3600000 }, {i}, {i * 11111}, { i % 32767 }, { i % 127}, { i * 1.11111 }, { i * 1000.1111 }, { i % 2}, - "binary_{i}", "nchar_测试_{i}", { now_time - 1000 * i }, {i % 127}, {i % 32767}, {i}, {i * 11111}) - ''' - tdSql.execute(insert_data) - tdSql.execute( - f'''insert into t1 values - ( { now_time + 10800000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( { now_time - (( rows // 2 ) * 60 + 30) * 60000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL) - ( { now_time - rows * 3600000 }, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) - ( - { now_time + 7200000 }, { pow(2,31) - pow(2,15) }, { pow(2,63) - pow(2,30) }, 32767, 127, { 3.3 * pow(10,38) }, - { 1.3 * pow(10,308) }, { rows % 2 }, "binary_limit-1", "nchar_测试_limit-1", { now_time - 86400000 }, - 254, 65534, {pow(2,32)-pow(2,16)}, {pow(2,64)-pow(2,31)} - ) - ( - { now_time + 3600000 } , { pow(2,31) - pow(2,16) }, { pow(2,63) - pow(2,31) }, 32766, 126, { 3.2 * pow(10,38) }, - { 1.2 * pow(10,308) }, { (rows-1) % 2 }, "binary_limit-2", "nchar_测试_limit-2", { now_time - 172800000 }, - 255, 65535, {pow(2,32)-pow(2,15)}, {pow(2,64)-pow(2,30)} - ) - ''' - ) + tdSql.execute( f"insert into ct1 values ( {NOW - i * TIME_STEP}, {row_data} )" ) + tdSql.execute( f"insert into ct2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )" ) + tdSql.execute( f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )" ) + tdSql.execute( f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )" ) + + tdSql.execute( f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )" ) + tdSql.execute( f"insert into ct2 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 0.6)}, {null_data} )" ) + tdSql.execute( f"insert into ct2 values ( {NOW - self.rows * int(TIME_STEP * 0.29) }, {null_data} )" ) + + tdSql.execute( f"insert into ct4 values ( {NOW + int(TIME_STEP * 0.8)}, {null_data} )" ) + tdSql.execute( f"insert into ct4 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 0.8)}, {null_data} )" ) + tdSql.execute( f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )" ) + + tdSql.execute( f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )" ) + tdSql.execute( f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )" ) + tdSql.execute( f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )" ) def run(self): - tdSql.prepare() + self.rows = 10 - tdLog.printNoPrefix("==========step1:create table") + + tdLog.printNoPrefix("==========step0:all check") + # self.all_test() + + tdLog.printNoPrefix("==========step1:create table in normal database") + tdSql.prepare() self.__create_tb() + self.__insert_data() + # return - tdLog.printNoPrefix("==========step2:insert data") - self.rows = 10 - self.__insert_data(self.rows) + tdLog.printNoPrefix("==========step2:create table in rollup database") + tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m") + tdSql.execute("use db3") + self.__create_tb() + self.__insert_data() - tdLog.printNoPrefix("==========step3:all check") - self.all_test() tdSql.execute("drop database if exists db1 ") tdSql.execute("drop database if exists db2 ") tdDnodes.stop(1) tdDnodes.start(1) - tdSql.execute("use db") - tdLog.printNoPrefix("==========step4:after wal, all check again ") self.all_test() diff --git a/tests/system-test/1-insert/time_range_wise.py b/tests/system-test/1-insert/time_range_wise.py new file mode 100644 index 0000000000000000000000000000000000000000..5387970d568bbbc5ac9e0c1f83d653c7e50e3f04 --- /dev/null +++ b/tests/system-test/1-insert/time_range_wise.py @@ -0,0 +1,343 @@ +import datetime + +from dataclasses import dataclass +from typing import List, Any, Tuple +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.constant import * + +PRIMARY_COL = "ts" + +INT_COL = "c_int" +BINT_COL = "c_bint" +SINT_COL = "c_sint" +TINT_COL = "c_tint" +FLOAT_COL = "c_float" +DOUBLE_COL = "c_double" +BOOL_COL = "c_bool" +TINT_UN_COL = "c_tint_un" +SINT_UN_COL = "c_sint_un" +BINT_UN_COL = "c_bint_un" +INT_UN_COL = "c_int_un" +BINARY_COL = "c_binary" +NCHAR_COL = "c_nchar" +TS_COL = "c_ts" + + + +NUM_COL = [INT_COL, BINT_COL, SINT_COL, TINT_COL, FLOAT_COL, DOUBLE_COL, ] +CHAR_COL = [BINARY_COL, NCHAR_COL, ] +BOOLEAN_COL = [BOOL_COL, ] +TS_TYPE_COL = [TS_COL, ] + +# insert data args： +TIME_STEP = 10000 +NOW = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + + +@dataclass +class DataSet: + ts_data : List[int] = None + int_data : List[int] = None + bint_data : List[int] = None + sint_data : List[int] = None + tint_data : List[int] = None + int_un_data : List[int] = None + bint_un_data: List[int] = None + sint_un_data: List[int] = None + tint_un_data: List[int] = None + float_data : List[float] = None + double_data : List[float] = None + bool_data : List[int] = None + binary_data : List[str] = None + nchar_data : List[str] = None + + def __post_init__(self): + self.ts_data = [] + self.int_data = [] + self.bint_data = [] + self.sint_data = [] + self.tint_data = [] + self.int_un_data = [] + self.bint_un_data = [] + self.sint_un_data = [] + self.tint_un_data = [] + self.float_data = [] + self.double_data = [] + self.bool_data = [] + self.binary_data = [] + self.nchar_data = [] + + +@dataclass +class SMAschema: + creation : str = "CREATE" + index_name : str = "sma_index_1" + index_flag : str = "SMA INDEX" + operator : str = "ON" + tbname : str = None + watermark : str = None + maxdelay : str = None + func : Tuple[str] = None + interval : Tuple[str] = None + sliding : str = None + other : Any = None + drop : str = "DROP" + drop_flag : str = "INDEX" + + def __post_init__(self): + if isinstance(self.other, dict): + for k,v in self.other.items(): + + if k.lower() == "index_name" and isinstance(v, str) and not self.index_name: + self.index_name = v + del self.other[k] + + if k.lower() == "index_flag" and isinstance(v, str) and not self.index_flag: + self.index_flag = v + del self.other[k] + + if k.lower() == "operator" and isinstance(v, str) and not self.operator: + self.operator = v + del self.other[k] + + if k.lower() == "tbname" and isinstance(v, str) and not self.tbname: + self.tbname = v + del self.other[k] + + if k.lower() == "watermark" and isinstance(v, str) and not self.watermark: + self.watermark = v + del self.other[k] + + if k.lower() == "maxdelay" and isinstance(v, str) and not self.maxdelay: + self.maxdelay = v + del self.other[k] + + if k.lower() == "functions" and isinstance(v, tuple) and not self.func: + self.func = v + del self.other[k] + + if k.lower() == "interval" and isinstance(v, tuple) and not self.interval: + self.interval = v + del self.other[k] + + if k.lower() == "sliding" and isinstance(v, str) and not self.sliding: + self.sliding = v + del self.other[k] + + if k.lower() == "drop_flag" and isinstance(v, str) and not self.drop_flag: + self.drop_flag = v + del self.other[k] + + +class TDTestCase: + + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor(), False) + + """ + create sma index : + 1. only create on stable, err_type: [child-table, normal-table] + 2. one taosd, one sma index , err_type: [ + one stb --> multi sma index, + multi stb in one db--> multi sma index, + multi stb in multi db --> multi sma index + ] + 3. arg of (interval/sliding) in query sql is equal to this arg in sma index + 4. client timezone is equal to timezone of sma index + 5. does not take effect unless querySmaOptimize flag is turned on, + """ + def __create_sma_index(self, sma:SMAschema): + sql = f"{sma.creation} {sma.index_flag} {sma.index_name} {sma.operator} {sma.tbname}" + if sma.func: + sql += f" function({', '.join(sma.func)})" + if sma.interval: + sql += f" interval({', '.join(sma.interval)})" + if sma.sliding: + sql += f" sliding({sma.sliding})" + if sma.watermark: + sql += f" watermark {sma.watermark}" + if sma.maxdelay: + sql += f" maxdelay {sma.maxdelay}" + if isinstance(sma.other, dict): + for k,v in sma.other.items(): + if isinstance(v,tuple) or isinstance(v, list): + sql += f" {k} ({' '.join(v)})" + else: + sql += f" {k} {v}" + if isinstance(sma.other, tuple) or isinstance(sma.other, list): + sql += " ".join(sma.other) + if isinstance(sma.other, int) or isinstance(sma.other, float) or isinstance(sma.other, str): + sql += sma.other + + return sql + + def sma_create_check(self, sma:SMAschema): + tdSql.query("show stables") + stb_in_list = False + for row in tdSql.queryResult: + if sma.tbname == row[0]: + stb_in_list = True + break + if not stb_in_list: + tdSql.error(self.__create_sma_index(sma)) + if not sma.creation: + tdSql.error(self.__create_sma_index(sma)) + if not sma.index_flag: + tdSql.error(self.__create_sma_index(sma)) + if not sma.index_name: + tdSql.error(self.__create_sma_index(sma)) + if not sma.operator: + tdSql.error(self.__create_sma_index(sma)) + if not sma.tbname: + tdSql.error(self.__create_sma_index(sma)) + if not sma.func: + tdSql.error(self.__create_sma_index(sma)) + if not sma.interval: + tdSql.error(self.__create_sma_index(sma)) + if not sma.sliding: + tdSql.error(self.__create_sma_index(sma)) + if sma.other: + tdSql.error(self.__create_sma_index(sma)) + + + def all_test(self): + pass + + def __create_tb(self): + tdLog.printNoPrefix("==========step: create table") + create_stb_sql = f'''create table stb1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned + ) tags (tag1 int) + ''' + create_ntb_sql = f'''create table t1( + ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, + {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, + {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp, + {TINT_UN_COL} tinyint unsigned, {SINT_UN_COL} smallint unsigned, + {INT_UN_COL} int unsigned, {BINT_UN_COL} bigint unsigned + ) + ''' + tdSql.execute(create_stb_sql) + tdSql.execute(create_ntb_sql) + + for i in range(4): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') + + def __data_set(self, rows): + data_set = DataSet() + + for i in range(rows): + data_set.ts_data.append(NOW + 1 * (rows - i)) + data_set.int_data.append(rows - i) + data_set.bint_data.append(11111 * (rows - i)) + data_set.sint_data.append(111 * (rows - i) % 32767) + data_set.tint_data.append(11 * (rows - i) % 127) + data_set.int_un_data.append(rows - i) + data_set.bint_un_data.append(11111 * (rows - i)) + data_set.sint_un_data.append(111 * (rows - i) % 32767) + data_set.tint_un_data.append(11 * (rows - i) % 127) + data_set.float_data.append(1.11 * (rows - i)) + data_set.double_data.append(1100.0011 * (rows - i)) + data_set.bool_data.append((rows - i) % 2) + data_set.binary_data.append(f'binary{(rows - i)}') + data_set.nchar_data.append(f'nchar_测试_{(rows - i)}') + + return data_set + + def __insert_data(self): + data = self.__data_set(rows=self.rows) + + # now_time = int(datetime.datetime.timestamp(datetime.datetime.now()) * 1000) + null_data = '''null, null, null, null, null, null, null, null, null, null, null, null, null, null''' + zero_data = "0, 0, 0, 0, 0, 0, 0, 'binary_0', 'nchar_0', 0, 0, 0, 0, 0" + + for i in range(self.rows): + row_data = f''' + {data.int_data[i]}, {data.bint_data[i]}, {data.sint_data[i]}, {data.tint_data[i]}, {data.float_data[i]}, {data.double_data[i]}, + {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {data.tint_un_data[i]}, + {data.sint_un_data[i]}, {data.int_un_data[i]}, {data.bint_un_data[i]} + ''' + neg_row_data = f''' + {-1 * data.int_data[i]}, {-1 * data.bint_data[i]}, {-1 * data.sint_data[i]}, {-1 * data.tint_data[i]}, {-1 * data.float_data[i]}, {-1 * data.double_data[i]}, + {data.bool_data[i]}, '{data.binary_data[i]}', '{data.nchar_data[i]}', {data.ts_data[i]}, {1 * data.tint_un_data[i]}, + {1 * data.sint_un_data[i]}, {1 * data.int_un_data[i]}, {1 * data.bint_un_data[i]} + ''' + + tdSql.execute( + f"insert into ct1 values ( {NOW - i * TIME_STEP}, {row_data} )") + tdSql.execute( + f"insert into ct2 values ( {NOW - i * int(TIME_STEP * 0.6)}, {neg_row_data} )") + tdSql.execute( + f"insert into ct4 values ( {NOW - i * int(TIME_STEP * 0.8) }, {row_data} )") + tdSql.execute( + f"insert into t1 values ( {NOW - i * int(TIME_STEP * 1.2)}, {row_data} )") + + tdSql.execute( + f"insert into ct2 values ( {NOW + int(TIME_STEP * 0.6)}, {null_data} )") + tdSql.execute( + f"insert into ct2 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 0.6)}, {null_data} )") + tdSql.execute( + f"insert into ct2 values ( {NOW - self.rows * int(TIME_STEP * 0.29) }, {null_data} )") + + tdSql.execute( + f"insert into ct4 values ( {NOW + int(TIME_STEP * 0.8)}, {null_data} )") + tdSql.execute( + f"insert into ct4 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 0.8)}, {null_data} )") + tdSql.execute( + f"insert into ct4 values ( {NOW - self.rows * int(TIME_STEP * 0.39)}, {null_data} )") + + tdSql.execute( + f"insert into t1 values ( {NOW + int(TIME_STEP * 1.2)}, {null_data} )") + tdSql.execute( + f"insert into t1 values ( {NOW - (self.rows + 1) * int(TIME_STEP * 1.2)}, {null_data} )") + tdSql.execute( + f"insert into t1 values ( {NOW - self.rows * int(TIME_STEP * 0.59)}, {null_data} )") + + def run(self): + sma1 = SMAschema(func=("min(c1)","max(c2)")) + sql1 = self.__create_sma_index(sma1) + print("================") + print(sql1) + # a = DataSet() + # return + self.rows = 10 + + tdLog.printNoPrefix("==========step0:all check") + # self.all_test() + + tdLog.printNoPrefix("==========step1:create table in normal database") + tdSql.prepare() + self.__create_tb() + self.__insert_data() + return + + tdLog.printNoPrefix("==========step2:create table in rollup database") + tdSql.execute("create database db3 retentions 1s:4m,2s:8m,3s:12m") + tdSql.execute("use db3") + self.__create_tb() + self.__insert_data() + + tdSql.execute("drop database if exists db1 ") + tdSql.execute("drop database if exists db2 ") + + tdDnodes.stop(1) + tdDnodes.start(1) + + tdLog.printNoPrefix("==========step4:after wal, all check again ") + self.all_test() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/abs.py b/tests/system-test/2-query/abs.py index d779cc26cd7560b8ae543bc9596eee388dfb98cc..244cccb04143601d185757950bea2bc2e1181f5e 100644 --- a/tests/system-test/2-query/abs.py +++ b/tests/system-test/2-query/abs.py @@ -6,15 +6,63 @@ import inspect from util.log import * from util.sql import * from util.cases import * +import random + class TDTestCase: - updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , - "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, - "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143} + updatecfgDict = {'debugFlag': 143, "cDebugFlag": 143, "uDebugFlag": 143, "rpcDebugFlag": 143, "tmrDebugFlag": 143, + "jniDebugFlag": 143, "simDebugFlag": 143, "dDebugFlag": 143, "dDebugFlag": 143, "vDebugFlag": 143, "mDebugFlag": 143, "qDebugFlag": 143, + "wDebugFlag": 143, "sDebugFlag": 143, "tsdbDebugFlag": 143, "tqDebugFlag": 143, "fsDebugFlag": 143, "fnDebugFlag": 143} + def init(self, conn, logSql): tdLog.debug(f"start to excute {__file__}") tdSql.init(conn.cursor(), True) + self.tb_nums = 10 + self.row_nums = 20 + self.ts = 1434938400000 + self.time_step = 1000 + + def insert_datas_and_check_abs(self ,tbnums , rownums , time_step ): + tdLog.info(" prepare datas for auto check abs function ") + + tdSql.execute(" create database test ") + tdSql.execute(" use test ") + tdSql.execute(" create stable stb (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint,\ + c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) tags (t1 int)") + for tbnum in range(tbnums): + tbname = "sub_tb_%d"%tbnum + tdSql.execute(" create table %s using stb tags(%d) "%(tbname , tbnum)) + + ts = self.ts + for row in range(rownums): + ts += time_step*row + c1 = random.randint(0,10000) + c2 = random.randint(0,100000) + c3 = random.randint(0,125) + c4 = random.randint(0,125) + c5 = random.random()/1.0 + c6 = random.random()/1.0 + c7 = "'true'" + c8 = "'binary_val'" + c9 = "'nchar_val'" + c10 = ts + tdSql.execute(f" insert into {tbname} values ({ts},{c1},{c2},{c3},{c4},{c5},{c6},{c7},{c8},{c9},{c10})") + + tdSql.execute("use test") + tbnames = ["stb", "sub_tb_1"] + support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] + for tbname in tbnames: + tdSql.query("desc {}".format(tbname)) + coltypes = tdSql.queryResult + for coltype in coltypes: + colname = coltype[0] + abs_sql = "select abs({}) from {} order by tbname ".format(colname, tbname) + origin_sql = "select {} from {} order by tbname".format(colname, tbname) + if coltype[1] in support_types: + self.check_result_auto(origin_sql , abs_sql) + + def prepare_datas(self): tdSql.execute( '''create table stb1 @@ -22,7 +70,7 @@ class TDTestCase: tags (t1 int) ''' ) - + tdSql.execute( ''' create table t1 @@ -39,14 +87,21 @@ class TDTestCase: tdSql.execute( f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute( + "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( f'''insert into t1 values @@ -64,11 +119,11 @@ class TDTestCase: ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ''' ) - def prepare_tag_datas(self): - # prepare datas - tdSql.execute("create database if not exists testdb keep 3650 duration 1000") + # prepare datas + tdSql.execute( + "create database if not exists testdb keep 3650 duration 1000") tdSql.execute(" use testdb ") tdSql.execute( '''create table stb1 @@ -84,7 +139,8 @@ class TDTestCase: ''' ) for i in range(4): - tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + tdSql.execute( + f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') for i in range(9): tdSql.execute( @@ -93,14 +149,21 @@ class TDTestCase: tdSql.execute( f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" ) - tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") - tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute( + "insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute( + "insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") - tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") - tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute( + "insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") tdSql.execute( f'''insert into t1 values @@ -119,18 +182,18 @@ class TDTestCase: ''' ) - def check_result_auto(self ,origin_query , abs_query): + def check_result_auto(self, origin_query, abs_query): abs_result = tdSql.getResult(abs_query) origin_result = tdSql.getResult(origin_query) - auto_result =[] - + auto_result = [] + for row in origin_result: row_check = [] for elem in row: if elem == None: elem = None - elif elem >=0: + elif elem >= 0: elem = elem else: elem = -elem @@ -138,21 +201,23 @@ class TDTestCase: auto_result.append(row_check) check_status = True - for row_index , row in enumerate(abs_result): - for col_index , elem in enumerate(row): + for row_index, row in enumerate(abs_result): + for col_index, elem in enumerate(row): if auto_result[row_index][col_index] != elem: - check_status = False + check_status = False if not check_status: - tdLog.notice("abs function value has not as expected , sql is \"%s\" "%abs_query ) + tdLog.notice( + "abs function value has not as expected , sql is \"%s\" " % abs_query) sys.exit(1) else: - tdLog.info("abs value check pass , it work as expected ,sql is \"%s\" "%abs_query ) - + tdLog.info( + "abs value check pass , it work as expected ,sql is \"%s\" " % abs_query) + def test_errors(self): tdSql.execute("use testdb") error_sql_lists = [ "select abs from t1", - # "select abs(-+--+c1) from t1", + "select abs(-+--+c1) from t1", # "select +-abs(c1) from t1", # "select ++-abs(c1) from t1", # "select ++--abs(c1) from t1", @@ -182,93 +247,25 @@ class TDTestCase: ] for error_sql in error_sql_lists: tdSql.error(error_sql) - + def support_types(self): tdSql.execute("use testdb") - type_error_sql_lists = [ - "select abs(ts) from t1" , - "select abs(t0) from t1" , - "select abs(c7) from t1", - "select abs(c8) from t1", - "select abs(c9) from t1", - "select abs(t7) from t1", - "select abs(t8) from t1", - "select abs(t9) from t1", - "select abs(ts) from ct1" , - "select abs(c7) from ct1", - "select abs(c8) from ct1", - "select abs(c9) from ct1", - "select abs(ts) from ct3" , - "select abs(c7) from ct3", - "select abs(c8) from ct3", - "select abs(c9) from ct3", - "select abs(ts) from ct4" , - "select abs(c7) from ct4", - "select abs(c8) from ct4", - "select abs(c9) from ct4", - "select abs(ts) from stb1" , - "select abs(c7) from stb1", - "select abs(c8) from stb1", - "select abs(c9) from stb1" , - - "select abs(ts) from stbbb1" , - "select abs(c7) from stbbb1", - - "select abs(ts) from tbname", - "select abs(c9) from tbname" - - ] - - for type_sql in type_error_sql_lists: - tdSql.error(type_sql) - - - type_sql_lists = [ - "select abs(c1) from t1", - "select abs(c2) from t1", - "select abs(c3) from t1", - "select abs(c4) from t1", - "select abs(c5) from t1", - "select abs(c6) from t1", - - "select abs(t1) from ct1", - "select abs(t2) from ct1", - "select abs(t3) from ct1", - "select abs(t4) from ct1", - "select abs(t5) from ct1", - "select abs(t6) from ct1", - - "select abs(c1) from ct1", - "select abs(c2) from ct1", - "select abs(c3) from ct1", - "select abs(c4) from ct1", - "select abs(c5) from ct1", - "select abs(c6) from ct1", - - "select abs(c1) from ct3", - "select abs(c2) from ct3", - "select abs(c3) from ct3", - "select abs(c4) from ct3", - "select abs(c5) from ct3", - "select abs(c6) from ct3", - - "select abs(c1) from stb1", - "select abs(c2) from stb1", - "select abs(c3) from stb1", - "select abs(c4) from stb1", - "select abs(c5) from stb1", - "select abs(c6) from stb1", - - "select abs(c6) as alisb from stb1", - "select abs(c6) alisb from stb1", - ] + tbnames = ["stb1", "t1", "ct1", "ct2"] + support_types = ["BIGINT", "SMALLINT", "TINYINT", "FLOAT", "DOUBLE", "INT"] + for tbname in tbnames: + tdSql.query("desc {}".format(tbname)) + coltypes = tdSql.queryResult + for coltype in coltypes: + colname = coltype[0] + abs_sql = "select abs({}) from {}".format(colname, tbname) + if coltype[1] in support_types: + tdSql.query(abs_sql) + else: + tdSql.error(abs_sql) - for type_sql in type_sql_lists: - tdSql.query(type_sql) - def basic_abs_function(self): - # basic query + # basic query tdSql.query("select c1 from ct3") tdSql.checkRows(0) tdSql.query("select c1 from t1") @@ -288,13 +285,13 @@ class TDTestCase: tdSql.query("select abs(c5) from ct3") tdSql.checkRows(0) tdSql.query("select abs(c6) from ct3") - + # used for regular table tdSql.query("select abs(c1) from t1") tdSql.checkData(0, 0, None) - tdSql.checkData(1 , 0, 1) - tdSql.checkData(3 , 0, 3) - tdSql.checkData(5 , 0, None) + tdSql.checkData(1, 0, 1) + tdSql.checkData(3, 0, 3) + tdSql.checkData(5, 0, None) tdSql.query("select c1, c2, c3 , c4, c5 from t1") tdSql.checkData(1, 4, 1.11000) @@ -305,62 +302,66 @@ class TDTestCase: tdSql.checkData(3, 4, 33) tdSql.checkData(5, 5, None) - self.check_result_auto( "select c1, c2, c3 , c4, c5 from t1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from t1") - + self.check_result_auto("select c1, c2, c3 , c4, c5 from t1", + "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from t1") + # used for sub table tdSql.query("select abs(c1) from ct1") tdSql.checkData(0, 0, 8) - tdSql.checkData(1 , 0, 7) - tdSql.checkData(3 , 0, 5) - tdSql.checkData(5 , 0, 4) + tdSql.checkData(1, 0, 7) + tdSql.checkData(3, 0, 5) + tdSql.checkData(5, 0, 4) tdSql.query("select abs(c1) from ct1") - self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct1", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct1") - self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct1;","select c1 from ct1" ) + self.check_result_auto("select c1, c2, c3 , c4, c5 from ct1", + "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct1") + self.check_result_auto( + "select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct1;", "select c1 from ct1") + + # used for stable table - # used for stable table - tdSql.query("select abs(c1) from stb1") tdSql.checkRows(25) - self.check_result_auto( "select c1, c2, c3 , c4, c5 from ct4 ", "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct4") - self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct4;" , "select c1 from ct4" ) - + self.check_result_auto("select c1, c2, c3 , c4, c5 from ct4 ", + "select (c1), abs(c2) ,abs(c3), abs(c4), abs(c5) from ct4") + self.check_result_auto( + "select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from ct4;", "select c1 from ct4") # used for not exists table tdSql.error("select abs(c1) from stbbb1") tdSql.error("select abs(c1) from tbname") tdSql.error("select abs(c1) from ct5") - # mix with common col + # mix with common col tdSql.query("select c1, abs(c1) from ct1") - tdSql.checkData(0 , 0 ,8) - tdSql.checkData(0 , 1 ,8) - tdSql.checkData(4 , 0 ,0) - tdSql.checkData(4 , 1 ,0) + tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 1, 8) + tdSql.checkData(4, 0, 0) + tdSql.checkData(4, 1, 0) tdSql.query("select c1, abs(c1) from ct4") - tdSql.checkData(0 , 0 , None) - tdSql.checkData(0 , 1 ,None) - tdSql.checkData(4 , 0 ,5) - tdSql.checkData(4 , 1 ,5) - tdSql.checkData(5 , 0 ,None) - tdSql.checkData(5 , 1 ,None) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(4, 0, 5) + tdSql.checkData(4, 1, 5) + tdSql.checkData(5, 0, None) + tdSql.checkData(5, 1, None) tdSql.query("select c1, abs(c1) from ct4 ") - tdSql.checkData(0 , 0 ,None) - tdSql.checkData(0 , 1 ,None) - tdSql.checkData(4 , 0 ,5) - tdSql.checkData(4 , 1 ,5) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(4, 0, 5) + tdSql.checkData(4, 1, 5) # mix with common functions tdSql.query("select c1, abs(c1),c5, floor(c5) from ct4 ") - tdSql.checkData(0 , 0 ,None) - tdSql.checkData(0 , 1 ,None) - tdSql.checkData(0 , 2 ,None) - tdSql.checkData(0 , 3 ,None) - - tdSql.checkData(3 , 0 , 6) - tdSql.checkData(3 , 1 , 6) - tdSql.checkData(3 , 2 ,6.66000) - tdSql.checkData(3 , 3 ,6.00000) + tdSql.checkData(0, 0, None) + tdSql.checkData(0, 1, None) + tdSql.checkData(0, 2, None) + tdSql.checkData(0, 3, None) + + tdSql.checkData(3, 0, 6) + tdSql.checkData(3, 1, 6) + tdSql.checkData(3, 2, 6.66000) + tdSql.checkData(3, 3, 6.00000) tdSql.query("select c1, abs(c1),c5, floor(c5) from stb1 ") @@ -377,18 +378,17 @@ class TDTestCase: tdSql.query("select max(c5), count(c5) from stb1") tdSql.query("select max(c5), count(c5) from ct1") - # bug fix for count tdSql.query("select count(c1) from ct4 ") - tdSql.checkData(0,0,9) + tdSql.checkData(0, 0, 9) tdSql.query("select count(*) from ct4 ") - tdSql.checkData(0,0,12) + tdSql.checkData(0, 0, 12) tdSql.query("select count(c1) from stb1 ") - tdSql.checkData(0,0,22) + tdSql.checkData(0, 0, 22) tdSql.query("select count(*) from stb1 ") - tdSql.checkData(0,0,25) + tdSql.checkData(0, 0, 25) - # bug fix for compute + # bug fix for compute tdSql.query("select c1, abs(c1) -0 ,ceil(c1)-0 from ct4 ") tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, None) @@ -405,47 +405,48 @@ class TDTestCase: tdSql.checkData(1, 1, 8.000000000) tdSql.checkData(1, 2, 7.900000000) - - def abs_func_filter(self): tdSql.execute("use db") - tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") + tdSql.query( + "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>5 ") tdSql.checkRows(3) - tdSql.checkData(0,0,8) - tdSql.checkData(0,1,8.000000000) - tdSql.checkData(0,2,8.000000000) - tdSql.checkData(0,3,7.900000000) - tdSql.checkData(0,4,3.000000000) + tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 1, 8.000000000) + tdSql.checkData(0, 2, 8.000000000) + tdSql.checkData(0, 3, 7.900000000) + tdSql.checkData(0, 4, 3.000000000) - tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.query( + "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") tdSql.checkRows(1) - tdSql.checkData(0,0,5) - tdSql.checkData(0,1,5.000000000) - tdSql.checkData(0,2,5.000000000) - tdSql.checkData(0,3,4.900000000) - tdSql.checkData(0,4,2.000000000) - - tdSql.query("select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 5.000000000) + tdSql.checkData(0, 2, 5.000000000) + tdSql.checkData(0, 3, 4.900000000) + tdSql.checkData(0, 4, 2.000000000) + + tdSql.query( + "select c1, abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1=5 ") tdSql.checkRows(1) - tdSql.checkData(0,0,5) - tdSql.checkData(0,1,5.000000000) - tdSql.checkData(0,2,5.000000000) - tdSql.checkData(0,3,4.900000000) - tdSql.checkData(0,4,2.000000000) - - tdSql.query("select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") + tdSql.checkData(0, 0, 5) + tdSql.checkData(0, 1, 5.000000000) + tdSql.checkData(0, 2, 5.000000000) + tdSql.checkData(0, 3, 4.900000000) + tdSql.checkData(0, 4, 2.000000000) + + tdSql.query( + "select c1,c2 , abs(c1) -0 ,ceil(c1-0.1)-0 ,floor(c1+0.1)-0.1 ,ceil(log(c1,2)-0.5) from ct4 where c1>log(c1,2) limit 1 ") tdSql.checkRows(1) - tdSql.checkData(0,0,8) - tdSql.checkData(0,1,88888) - tdSql.checkData(0,2,8.000000000) - tdSql.checkData(0,3,8.000000000) - tdSql.checkData(0,4,7.900000000) - tdSql.checkData(0,5,3.000000000) - + tdSql.checkData(0, 0, 8) + tdSql.checkData(0, 1, 88888) + tdSql.checkData(0, 2, 8.000000000) + tdSql.checkData(0, 3, 8.000000000) + tdSql.checkData(0, 4, 7.900000000) + tdSql.checkData(0, 5, 3.000000000) def abs_Arithmetic(self): pass - + def check_boundary_values(self): tdSql.execute("drop database if exists bound_test") @@ -457,74 +458,84 @@ class TDTestCase: ) tdSql.execute(f'create table sub1_bound using stb_bound tags ( 1 )') tdSql.execute( - f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" - ) + f"insert into sub1_bound values ( now()-1s, 2147483647, 9223372036854775807, 32767, 127, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) tdSql.execute( - f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" - ) + f"insert into sub1_bound values ( now()-1s, -2147483647, -9223372036854775807, -32767, -127, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) tdSql.execute( - f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" - ) + f"insert into sub1_bound values ( now(), 2147483646, 9223372036854775806, 32766, 126, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) tdSql.execute( - f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" - ) + f"insert into sub1_bound values ( now(), -2147483646, -9223372036854775806, -32766, -126, -3.40E+38, -1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) tdSql.error( - f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" - ) - self.check_result_auto( "select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from sub1_bound") - self.check_result_auto( "select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", "select abs(c1), abs(c2) ,abs(c3), abs(c3), abs(c2) ,abs(c1) from sub1_bound") - self.check_result_auto("select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from sub1_bound;" , "select abs(c1) from sub1_bound" ) - + f"insert into sub1_bound values ( now()+1s, 2147483648, 9223372036854775808, 32768, 128, 3.40E+38, 1.7e+308, True, 'binary_tb1', 'nchar_tb1', now() )" + ) + self.check_result_auto("select c1, c2, c3 , c4, c5 ,c6 from sub1_bound ", + "select abs(c1), abs(c2) ,abs(c3), abs(c4), abs(c5) ,abs(c6) from sub1_bound") + self.check_result_auto("select c1, c2, c3 , c3, c2 ,c1 from sub1_bound ", + "select abs(c1), abs(c2) ,abs(c3), abs(c3), abs(c2) ,abs(c1) from sub1_bound") + self.check_result_auto( + "select abs(abs(abs(abs(abs(abs(abs(abs(abs(abs(c1)))))))))) nest_col_func from sub1_bound;", "select abs(c1) from sub1_bound") + # check basic elem for table per row - tdSql.query("select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ") - tdSql.checkData(0,0,2147483647) - tdSql.checkData(0,1,9223372036854775807) - tdSql.checkData(0,2,32767) - tdSql.checkData(0,3,127) - tdSql.checkData(0,4,339999995214436424907732413799364296704.00000) - tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) - tdSql.checkData(1,0,2147483647) - tdSql.checkData(1,1,9223372036854775807) - tdSql.checkData(1,2,32767) - tdSql.checkData(1,3,127) - tdSql.checkData(1,4,339999995214436424907732413799364296704.00000) - tdSql.checkData(1,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) - tdSql.checkData(3,0,2147483646) - tdSql.checkData(3,1,9223372036854775806) - tdSql.checkData(3,2,32766) - tdSql.checkData(3,3,126) - tdSql.checkData(3,4,339999995214436424907732413799364296704.00000) - tdSql.checkData(3,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.query( + "select abs(c1) ,abs(c2) , abs(c3) , abs(c4), abs(c5), abs(c6) from sub1_bound ") + tdSql.checkData(0, 0, 2147483647) + tdSql.checkData(0, 1, 9223372036854775807) + tdSql.checkData(0, 2, 32767) + tdSql.checkData(0, 3, 127) + tdSql.checkData(0, 4, 339999995214436424907732413799364296704.00000) + tdSql.checkData(0, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.checkData(1, 0, 2147483647) + tdSql.checkData(1, 1, 9223372036854775807) + tdSql.checkData(1, 2, 32767) + tdSql.checkData(1, 3, 127) + tdSql.checkData(1, 4, 339999995214436424907732413799364296704.00000) + tdSql.checkData(1, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + tdSql.checkData(3, 0, 2147483646) + tdSql.checkData(3, 1, 9223372036854775806) + tdSql.checkData(3, 2, 32766) + tdSql.checkData(3, 3, 126) + tdSql.checkData(3, 4, 339999995214436424907732413799364296704.00000) + tdSql.checkData(3, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) # check + - * / in functions - tdSql.query("select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") - tdSql.checkData(0,0,2147483648.000000000) - tdSql.checkData(0,1,9223372036854775807) - tdSql.checkData(0,2,32767.000000000) - tdSql.checkData(0,3,63.500000000) - tdSql.checkData(0,4,169999997607218212453866206899682148352.000000000) - tdSql.checkData(0,5,169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) - - - tdSql.checkData(1,0,2147483646.000000000) - tdSql.checkData(1,1,9223372036854775808.000000000) - tdSql.checkData(1,2,32767.000000000) - tdSql.checkData(1,3,63.500000000) - tdSql.checkData(1,4,169999997607218212453866206899682148352.000000000) + tdSql.query( + "select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") + tdSql.checkData(0, 0, 2147483648.000000000) + tdSql.checkData(0, 1, 9223372036854775807) + tdSql.checkData(0, 2, 32767.000000000) + tdSql.checkData(0, 3, 63.500000000) + tdSql.checkData( + 0, 4, 169999997607218212453866206899682148352.000000000) + tdSql.checkData(0, 5, 169999999999999993883079578865998174333346074304075874502773119193537729178160565864330091787584707988572262467983188919169916105593357174268369962062473635296474636515660464935663040684957844303524367815028553272712298986386310828644513212353921123253311675499856875650512437415429217994623324794855339589632.000000000) + + tdSql.checkData(1, 0, 2147483646.000000000) + tdSql.checkData(1, 1, 9223372036854775808.000000000) + tdSql.checkData(1, 2, 32767.000000000) + tdSql.checkData(1, 3, 63.500000000) + tdSql.checkData( + 1, 4, 169999997607218212453866206899682148352.000000000) + + self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound", + "select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") - self.check_result_auto("select c1+1 ,c2 , c3*1 , c4/2, c5/2, c6 from sub1_bound" ,"select abs(c1+1) ,abs(c2) , abs(c3*1) , abs(c4/2), abs(c5)/2, abs(c6) from sub1_bound ") - def test_tag_compute_for_scalar_function(self): - + tdSql.execute("use testdb") - self.check_result_auto( "select c1, t2, t3 , t4, t5 from ct4 ", "select (c1), abs(t2) ,abs(t3), abs(t4), abs(t5) from ct4") - self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from ct4 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from ct4") - self.check_result_auto( "select c1+2, t2+2, t3 , t4, t5 from stb1 order by t1 ", "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from stb1 order by t1") + self.check_result_auto("select c1, t2, t3 , t4, t5 from ct4 ", + "select (c1), abs(t2) ,abs(t3), abs(t4), abs(t5) from ct4") + self.check_result_auto("select c1+2, t2+2, t3 , t4, t5 from ct4 ", + "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from ct4") + self.check_result_auto("select c1+2, t2+2, t3 , t4, t5 from stb1 order by t1 ", + "select (c1)+2, abs(t2)+2 ,abs(t3), abs(t4), abs(t5) from stb1 order by t1") - # bug need fix + # bug need fix - # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash + # tdSql.query(" select sum(c1) from stb1 where t1+10 >1; ") #　taosd crash tdSql.query("select c1 ,t1 from stb1 where t1 =0 ") tdSql.checkRows(13) # tdSql.query("select t1 from stb1 where t1 >0 ") @@ -532,7 +543,7 @@ class TDTestCase: # tdSql.query("select sum(t1) from (select c1 ,t1 from stb1)") # tdSql.checkData(0,0,61) # tdSql.query("select distinct(c1) ,t1 from stb1") - # tdSql.checkRows(11) + # tdSql.checkRows(20) # tdSql.query("select max(t2) , t1 ,c1, t2 from stb1") # tdSql.checkData(0,3,33333) @@ -543,45 +554,49 @@ class TDTestCase: tdSql.checkRows(1) # tdSql.query("select t1 from stb1 where abs(t1+c1)=1") # tdSql.checkRows(1) - tdSql.query("select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") - - + tdSql.query( + "select abs(c1+t1)*t1 from stb1 where abs(c1)/floor(abs(ceil(t1))) ==1") def run(self): # sourcery skip: extract-duplicate-method, remove-redundant-fstring tdSql.prepare() tdLog.printNoPrefix("==========step1:create table ==============") - + self.prepare_datas() self.prepare_tag_datas() - tdLog.printNoPrefix("==========step2:test errors ==============") + tdLog.printNoPrefix("==========step2:test errors ==============") self.test_errors() - - tdLog.printNoPrefix("==========step3:support types ============") + + tdLog.printNoPrefix("==========step3:support types ============") self.support_types() - tdLog.printNoPrefix("==========step4: abs basic query ============") + tdLog.printNoPrefix("==========step4: abs basic query ============") self.basic_abs_function() - tdLog.printNoPrefix("==========step5: abs boundary query ============") + tdLog.printNoPrefix("==========step5: abs boundary query ============") self.check_boundary_values() - tdLog.printNoPrefix("==========step6: abs filter query ============") + tdLog.printNoPrefix("==========step6: abs filter query ============") self.abs_func_filter() - tdLog.printNoPrefix("==========step6: tag coumpute query ============") - + tdLog.printNoPrefix("==========step6: tag coumpute query ============") + self.test_tag_compute_for_scalar_function() + tdLog.printNoPrefix("==========step7: check result of query ============") + + self.insert_datas_and_check_abs(self.tb_nums,self.row_nums,self.time_step) + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") + tdCases.addLinux(__file__, TDTestCase()) tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/distribute_agg_avg.py b/tests/system-test/2-query/distribute_agg_avg.py new file mode 100644 index 0000000000000000000000000000000000000000..647a2625582f24ec9baeedeae84c5b3034b08f2c --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_avg.py @@ -0,0 +1,279 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random ,os ,sys +import platform + + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_avg_functions(self, tbname , col_name): + + avg_sql = f"select avg({col_name}) from {tbname};" + + same_sql = f"select {col_name} from {tbname} where {col_name} is not null " + + tdSql.query(same_sql) + pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') + pre_avg = np.sum(pre_data)/len(pre_data) + + tdSql.query(avg_sql) + tdSql.checkData(0,0,pre_avg) + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_avg_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + avg_sql = f"select avg({col_name}) from stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null " + + tdSql.query(same_sql) + pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') + pre_avg = np.sum(pre_data)/len(pre_data) + + tdSql.query(avg_sql) + tdSql.checkData(0,0,pre_avg) + + def check_avg_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + self.check_avg_functions(tablename,colname) + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_avg_distribute_diff_vnode(colname) + else: + # self.check_avg_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query(" select avg(c1) from stb1 ") + tdSql.checkData(0,0,14.086956522) + + tdSql.query(" select avg(a) from (select avg(c1) a from stb1 partition by tbname) ") + tdSql.checkData(0,0,14.292307692) + + tdSql.query(" select avg(c1) from stb1 where t1=1") + tdSql.checkData(0,0,6.000000000) + + tdSql.query("select avg(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,11112.000000000) + + tdSql.query("select avg(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,6.000000000) + + tdSql.query("select avg(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select avg(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select avg(c1) from stb1 union all select avg(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,14.086956522) + + tdSql.query("select avg(a) from (select avg(c1) a from stb1 union all select avg(c1) a from stb1)") + tdSql.checkRows(1) + tdSql.checkData(0,0,14.086956522) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select avg(tb1.c1), avg(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,4.500000000) + tdSql.checkData(0,1,4.500000000) + + # group by + tdSql.execute(" use testdb ") + + # partition by tbname or partition by tag + tdSql.query("select avg(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + # nest query for support max + tdSql.query("select avg(c2+2)+1 from (select avg(c1) c2 from stb1)") + tdSql.checkData(0,0,17.086956522) + tdSql.query("select avg(c1+2) as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,16.086956522) + tdSql.query("select avg(a+2) as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,16.086956522) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3),sum(c1+c2),avg(c1) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + tdSql.checkData(0,4,28202310.000000000) + tdSql.checkData(0,5,14.086956522) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_avg_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/distribute_agg_max.py b/tests/system-test/2-query/distribute_agg_max.py index ae0ab5aafab4b64ac9072b4de392d4359a4129de..5c9760cbcf0a027b4a3f0c9fa684a83a36fd8d97 100644 --- a/tests/system-test/2-query/distribute_agg_max.py +++ b/tests/system-test/2-query/distribute_agg_max.py @@ -245,6 +245,27 @@ class TDTestCase: tdSql.query(" select max(c1),c2 from stb1 group by c2 ") tdSql.checkRows(31) + # selective common cols of datas + tdSql.query("select max(c1),c2,c3,c5 from stb1") + tdSql.checkRows(1) + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,311108) + tdSql.checkData(0,2,3108) + tdSql.checkData(0,3,31.08000) + + tdSql.query("select max(c1),t1,c2,t3 from stb1") + tdSql.checkRows(1) + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,19) + tdSql.checkData(0,2,311108) + + tdSql.query("select max(c1),ceil(t1),pow(c2,1)+2,abs(t3) from stb1") + tdSql.checkRows(1) + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,19) + tdSql.checkData(0,2,311110.000000000) + tdSql.checkData(0,3,2109) + # partition by tbname or partition by tag tdSql.query("select max(c1),tbname from stb1 partition by tbname") query_data = tdSql.queryResult diff --git a/tests/system-test/2-query/distribute_agg_min.py b/tests/system-test/2-query/distribute_agg_min.py index 8a458c74df4f46929bb93abbebee2aad3dd7b76f..dd20d882295019dbc5af94db96d8ffb8ffe5d760 100644 --- a/tests/system-test/2-query/distribute_agg_min.py +++ b/tests/system-test/2-query/distribute_agg_min.py @@ -245,6 +245,24 @@ class TDTestCase: tdSql.query(" select min(c1),c2 from stb1 group by c2 ") tdSql.checkRows(31) + # selective common cols of datas + tdSql.query("select min(c1),c2,c3,c5 from stb1") + tdSql.checkRows(1) + tdSql.checkData(0,0,0) + tdSql.checkData(0,1,0) + tdSql.checkData(0,2,0) + tdSql.checkData(0,3,0) + + tdSql.query("select min(c1),t1,c2,t3 from stb1 where c1 >5") + tdSql.checkRows(1) + tdSql.checkData(0,0,6) + tdSql.checkData(0,2,66666) + + tdSql.query("select min(c1),ceil(t1),pow(c2,1)+2,abs(t3) from stb1 where c1>12") + tdSql.checkRows(1) + tdSql.checkData(0,0,13) + tdSql.checkData(0,2,144445.000000000) + # partition by tbname or partition by tag tdSql.query("select min(c1),tbname from stb1 partition by tbname") query_data = tdSql.queryResult diff --git a/tests/system-test/2-query/distribute_agg_stddev.py b/tests/system-test/2-query/distribute_agg_stddev.py new file mode 100644 index 0000000000000000000000000000000000000000..5050e6e94013c0c60e9150637d54777802e99e4d --- /dev/null +++ b/tests/system-test/2-query/distribute_agg_stddev.py @@ -0,0 +1,304 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random ,os ,sys +import platform +import math + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + + + def check_stddev_functions(self, tbname , col_name): + + stddev_sql = f"select stddev({col_name}) from {tbname};" + + same_sql = f"select {col_name} from {tbname} where {col_name} is not null " + + tdSql.query(same_sql) + pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') + pre_avg = np.sum(pre_data)/len(pre_data) + + # Calculate variance + stddev_result = 0 + for num in tdSql.queryResult: + stddev_result += (num-pre_avg)*(num-pre_avg)/len(tdSql.queryResult) + + stddev_result = math.sqrt(stddev_result) + + tdSql.query(stddev_sql) + + if -0.0001 < tdSql.queryResult[0][0]-stddev_result < 0.0001: + tdLog.info(" sql:%s; row:0 col:0 data:%d , expect:%d"%(stddev_sql,tdSql.queryResult[0][0],stddev_result)) + else: + tdLog.exit(" sql:%s; row:0 col:0 data:%d , expect:%d"%(stddev_sql,tdSql.queryResult[0][0],stddev_result)) + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + tdSql.execute( + ''' + create table t1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + ''' + ) + for i in range(20): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + + for i in range(9): + tdSql.execute( + f"insert into ct1 values ( now()-{i*10}s, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + tdSql.execute( + f"insert into ct4 values ( now()-{i*90}d, {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, 'binary{i}', 'nchar{i}', now()+{1*i}a )" + ) + + for i in range(1,21): + if i ==1 or i == 4: + continue + else: + tbname = "ct"+f'{i}' + for j in range(9): + tdSql.execute( + f"insert into {tbname} values ( now()-{(i+j)*10}s, {1*(j+i)}, {11111*(j+i)}, {111*(j+i)}, {11*(j)}, {1.11*(j+i)}, {11.11*(j+i)}, {(j+i)%2}, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + tdSql.execute("insert into ct1 values (now()-45s, 0, 0, 0, 0, 0, 0, 0, 'binary0', 'nchar0', now()+8a )") + tdSql.execute("insert into ct1 values (now()+10s, 9, -99999, -999, -99, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+15s, 9, -99999, -999, -99, -9.99, NULL, 1, 'binary9', 'nchar9', now()+9a )") + tdSql.execute("insert into ct1 values (now()+20s, 9, -99999, -999, NULL, -9.99, -99.99, 1, 'binary9', 'nchar9', now()+9a )") + + tdSql.execute("insert into ct4 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct4 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdSql.execute( + f'''insert into t1 values + ( '2020-04-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2020-10-21 01:01:01.000', 1, 11111, 111, 11, 1.11, 11.11, 1, "binary1", "nchar1", now()+1a ) + ( '2020-12-31 01:01:01.000', 2, 22222, 222, 22, 2.22, 22.22, 0, "binary2", "nchar2", now()+2a ) + ( '2021-01-01 01:01:06.000', 3, 33333, 333, 33, 3.33, 33.33, 0, "binary3", "nchar3", now()+3a ) + ( '2021-05-07 01:01:10.000', 4, 44444, 444, 44, 4.44, 44.44, 1, "binary4", "nchar4", now()+4a ) + ( '2021-07-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ( '2021-09-30 01:01:16.000', 5, 55555, 555, 55, 5.55, 55.55, 0, "binary5", "nchar5", now()+5a ) + ( '2022-02-01 01:01:20.000', 6, 66666, 666, 66, 6.66, 66.66, 1, "binary6", "nchar6", now()+6a ) + ( '2022-10-28 01:01:26.000', 7, 00000, 000, 00, 0.00, 00.00, 1, "binary7", "nchar7", "1970-01-01 08:00:00.000" ) + ( '2022-12-01 01:01:30.000', 8, -88888, -888, -88, -8.88, -88.88, 0, "binary8", "nchar8", "1969-01-01 01:00:00.000" ) + ( '2022-12-31 01:01:36.000', 9, -99999999999999999, -999, -99, -9.99, -999999999999999999999.99, 1, "binary9", "nchar9", "1900-01-01 00:00:00.000" ) + ( '2023-02-21 01:01:01.000', NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) + ''' + ) + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def check_stddev_distribute_diff_vnode(self,col_name): + + vgroup_ids = [] + for k ,v in self.vnode_disbutes.items(): + if len(v)>=2: + vgroup_ids.append(k) + + distribute_tbnames = [] + + for vgroup_id in vgroup_ids: + vnode_tables = self.vnode_disbutes[vgroup_id] + distribute_tbnames.append(random.sample(vnode_tables,1)[0]) + tbname_ins = "" + for tbname in distribute_tbnames: + tbname_ins += "'%s' ,"%tbname + + tbname_filters = tbname_ins[:-1] + + stddev_sql = f"select stddev({col_name}) from stb1 where tbname in ({tbname_filters});" + + same_sql = f"select {col_name} from stb1 where tbname in ({tbname_filters}) and {col_name} is not null " + + tdSql.query(same_sql) + pre_data = np.array(tdSql.queryResult)[np.array(tdSql.queryResult) != None] + if (platform.system().lower() == 'windows' and pre_data.dtype == 'int32'): + pre_data = np.array(pre_data, dtype = 'int64') + pre_avg = np.sum(pre_data)/len(pre_data) + + # Calculate variance + stddev_result = 0 + for num in tdSql.queryResult: + stddev_result += (num-pre_avg)*(num-pre_avg)/len(tdSql.queryResult) + + stddev_result = math.sqrt(stddev_result) + + tdSql.query(stddev_sql) + tdSql.checkData(0,0,stddev_result) + + + def check_stddev_status(self): + # check max function work status + + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + tablenames.append(table_name[0]) + + tdSql.query("desc stb1") + col_names = tdSql.queryResult + + colnames = [] + for col_name in col_names: + if col_name[1] in ["INT" ,"BIGINT" ,"SMALLINT" ,"TINYINT" , "FLOAT" ,"DOUBLE"]: + colnames.append(col_name[0]) + + for tablename in tablenames: + for colname in colnames: + if colname.startswith("c"): + self.check_stddev_functions(tablename,colname) + else: + # self.check_stddev_functions(tablename,colname) + pass + + + # check max function for different vnode + + for colname in colnames: + if colname.startswith("c"): + self.check_stddev_distribute_diff_vnode(colname) + else: + # self.check_stddev_distribute_diff_vnode(colname) # bug for tag + pass + + + def distribute_agg_query(self): + # basic filter + tdSql.query(" select stddev(c1) from stb1 ") + tdSql.checkData(0,0,6.694663959) + + tdSql.query(" select stddev(a) from (select stddev(c1) a from stb1 partition by tbname) ") + tdSql.checkData(0,0,0.156797505) + + tdSql.query(" select stddev(c1) from stb1 where t1=1") + tdSql.checkData(0,0,2.581988897) + + tdSql.query("select stddev(c1+c2) from stb1 where c1 =1 ") + tdSql.checkData(0,0,0.000000000) + + tdSql.query("select stddev(c1) from stb1 where tbname=\"ct2\"") + tdSql.checkData(0,0,2.581988897) + + tdSql.query("select stddev(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + tdSql.query("select stddev(c1) from stb1 where t1> 4 partition by tbname") + tdSql.checkRows(15) + + # union all + tdSql.query("select stddev(c1) from stb1 union all select stddev(c1) from stb1 ") + tdSql.checkRows(2) + tdSql.checkData(0,0,6.694663959) + + tdSql.query("select stddev(a) from (select stddev(c1) a from stb1 union all select stddev(c1) a from stb1)") + tdSql.checkRows(1) + tdSql.checkData(0,0,0.000000000) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query("select stddev(tb1.c1), stddev(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts") + tdSql.checkRows(1) + tdSql.checkData(0,0,2.872281323) + tdSql.checkData(0,1,2.872281323) + + # group by + tdSql.execute(" use testdb ") + + # partition by tbname or partition by tag + tdSql.query("select stddev(c1) from stb1 partition by tbname") + tdSql.checkRows(20) + + # nest query for support max + tdSql.query("select stddev(c2+2)+1 from (select stddev(c1) c2 from stb1)") + tdSql.checkData(0,0,1.000000000) + tdSql.query("select stddev(c1+2) as c2 from (select ts ,c1 ,c2 from stb1)") + tdSql.checkData(0,0,6.694663959) + tdSql.query("select stddev(a+2) as c2 from (select ts ,abs(c1) a ,c2 from stb1)") + tdSql.checkData(0,0,6.694663959) + + # mixup with other functions + tdSql.query("select max(c1),count(c1),last(c2,c3),sum(c1+c2),avg(c1),stddev(c1) from stb1") + tdSql.checkData(0,0,28) + tdSql.checkData(0,1,184) + tdSql.checkData(0,2,-99999) + tdSql.checkData(0,3,-999) + tdSql.checkData(0,4,28202310.000000000) + tdSql.checkData(0,5,14.086956522) + tdSql.checkData(0,6,6.694663959) + + def run(self): + + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.check_stddev_status() + self.distribute_agg_query() + + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/2-query/join.py b/tests/system-test/2-query/join.py index 140808d3874915b56cc3a0ee559e352a1a0589ae..5ff11c84dd66ce1b88843b9a666318278778a6e2 100644 --- a/tests/system-test/2-query/join.py +++ b/tests/system-test/2-query/join.py @@ -238,7 +238,7 @@ class TDTestCase: ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp - ) tags (t1 int) + ) tags (tag1 int) ''' create_ntb_sql = f'''create table t1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, diff --git a/tests/system-test/2-query/json_tag.py b/tests/system-test/2-query/json_tag.py index 957e916e344d6bb35288521717d39f40ce39360b..0c649f20080c73b1a26ac419037cc8904b8e1119 100644 --- a/tests/system-test/2-query/json_tag.py +++ b/tests/system-test/2-query/json_tag.py @@ -68,7 +68,7 @@ class TDTestCase: tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('[1,true]')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{222}')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"fe\"}')") - # + # test invalidate json key, key must can be printed assic char tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":[1,true]}')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"tag1\":{}}')") @@ -79,7 +79,7 @@ class TDTestCase: # test invalidate json value, value number can not be inf,nan TD-12166 tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":1.8e308}')") tdSql.error("CREATE TABLE if not exists jsons1_14 using jsons1 tags('{\"k\":-1.8e308}')") - # + #test length limit char1= ''.join(['abcd']*64) char3= ''.join(['abcd']*1021) @@ -87,15 +87,15 @@ class TDTestCase: tdSql.error("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s1\":5}')" % char1) # len(key)=257 tdSql.execute("CREATE TABLE if not exists jsons1_15 using jsons1 tags('{\"%s\":5}')" % char1) # len(key)=256 tdSql.error("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSSS\":\"%s\"}')" % char3) # len(object)=4096 - #tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSS\":\"%s\"}')" % char3) # len(object)=4095 + tdSql.execute("CREATE TABLE if not exists jsons1_16 using jsons1 tags('{\"TSSS\":\"%s\"}')" % char3) # len(object)=4095 tdSql.execute("drop table if exists jsons1_15") tdSql.execute("drop table if exists jsons1_16") - # + print("============== STEP 2 ===== alter table json tag") tdSql.error("ALTER STABLE jsons1 add tag tag2 nchar(20)") tdSql.error("ALTER STABLE jsons1 drop tag jtag") tdSql.error("ALTER TABLE jsons1 MODIFY TAG jtag nchar(128)") - # + tdSql.execute("ALTER TABLE jsons1_1 SET TAG jtag='{\"tag1\":\"femail\",\"tag2\":35,\"tag3\":true}'") tdSql.query("select jtag from jsons1_1") tdSql.checkData(0, 0, '{"tag1":"femail","tag2":35,"tag3":true}') @@ -105,9 +105,9 @@ class TDTestCase: tdSql.execute("create table st(ts timestamp, i int) tags(t int)") tdSql.error("ALTER STABLE st add tag jtag json") tdSql.error("ALTER STABLE st add column jtag json") - # - # print("============== STEP 3 ===== query table") - # # test error syntax + + print("============== STEP 3 ===== query table") + # test error syntax tdSql.error("select * from jsons1 where jtag->tag1='beijing'") tdSql.error("select -> from jsons1") tdSql.error("select * from jsons1 where contains") @@ -115,17 +115,17 @@ class TDTestCase: tdSql.error("select jtag->location from jsons1") tdSql.error("select jtag contains location from jsons1") tdSql.error("select * from jsons1 where jtag contains location") - #tdSql.error("select * from jsons1 where jtag contains''") + tdSql.query("select * from jsons1 where jtag contains''") tdSql.error("select * from jsons1 where jtag contains 'location'='beijing'") - # - # # test function error + + # test function error tdSql.error("select avg(jtag->'tag1') from jsons1") tdSql.error("select avg(jtag) from jsons1") tdSql.error("select min(jtag->'tag1') from jsons1") tdSql.error("select min(jtag) from jsons1") tdSql.error("select ceil(jtag->'tag1') from jsons1") tdSql.error("select ceil(jtag) from jsons1") - # + #test scalar operation tdSql.query("select jtag contains 'tag1',jtag->'tag1' from jsons1 order by jtag->'tag1'") @@ -158,10 +158,11 @@ class TDTestCase: tdSql.checkData(0, 0, None) tdSql.checkData(0, 1, False) tdSql.checkData(7, 0, "false") - tdSql.checkData(7, 1, True) + tdSql.checkData(7, 1, False) + tdSql.checkData(8, 1, False) tdSql.checkData(12, 1, True) - # # test select normal column + # test select normal column tdSql.query("select dataint from jsons1 order by dataint") tdSql.checkRows(9) tdSql.checkData(1, 0, 1) @@ -180,7 +181,7 @@ class TDTestCase: tdSql.query("select jtag from jsons1_9") tdSql.checkData(0, 0, None) - # # test select json tag->'key', value is string + # test select json tag->'key', value is string tdSql.query("select jtag->'tag1' from jsons1_1") tdSql.checkData(0, 0, '"femail"') tdSql.query("select jtag->'tag2' from jsons1_6") @@ -200,7 +201,7 @@ class TDTestCase: # test select json tag->'key', key is not exist tdSql.query("select jtag->'tag10' from jsons1_4") tdSql.checkData(0, 0, None) - # + tdSql.query("select jtag->'tag1' from jsons1") tdSql.checkRows(13) # test header name @@ -210,24 +211,25 @@ class TDTestCase: tdSql.checkColNameList(res, cname_list) - # # test where with json tag + # test where with json tag tdSql.query("select * from jsons1_1 where jtag is not null") - # tdSql.error("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") - # tdSql.error("select * from jsons1 where jtag->'tag1'={}") - # - # # where json value is string + # tdSql.query("select * from jsons1 where jtag='{\"tag1\":11,\"tag2\":\"\"}'") + tdSql.error("select * from jsons1 where jtag->'tag1'={}") + + # where json value is string tdSql.query("select * from jsons1 where jtag->'tag2'='beijing'") tdSql.checkRows(2) - tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing'") + tdSql.query("select dataint,tbname,jtag->'tag1',jtag from jsons1 where jtag->'tag2'='beijing' order by dataint") tdSql.checkRows(2) - # out of order, cannot compare value - #tdSql.checkData(0, 0, 2) - #tdSql.checkData(0, 1, 'jsons1_2') - #tdSql.checkData(0, 2, 5) - #tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}') - #tdSql.checkData(1, 0, 3) - #tdSql.checkData(1, 1, 'jsons1_3') - #tdSql.checkData(1, 2, 'false') + tdSql.checkData(0, 0, 2) + tdSql.checkData(0, 1, 'jsons1_2') + tdSql.checkData(0, 2, "5.000000000") + tdSql.checkData(0, 3, '{"tag1":5,"tag2":"beijing"}') + tdSql.checkData(1, 0, 3) + tdSql.checkData(1, 1, 'jsons1_3') + tdSql.checkData(1, 2, 'false') + + tdSql.query("select * from jsons1 where jtag->'tag1'='beijing'") tdSql.checkRows(0) tdSql.query("select * from jsons1 where jtag->'tag1'='收到货'") @@ -236,72 +238,73 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag2'>='beijing'") tdSql.checkRows(3) - # open - #tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'") - #tdSql.checkRows(2) - tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'") + tdSql.query("select * from jsons1 where jtag->'tag2'<'beijing'") tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag2'<='beijing'") + tdSql.checkRows(4) tdSql.query("select * from jsons1 where jtag->'tag2'!='beijing'") - tdSql.checkRows(5) + tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag2'=''") tdSql.checkRows(2) - # - # # where json value is int + + # where json value is int tdSql.query("select * from jsons1 where jtag->'tag1'=5") tdSql.checkRows(1) tdSql.checkData(0, 1, 2) tdSql.query("select * from jsons1 where jtag->'tag1'=10") tdSql.checkRows(0) tdSql.query("select * from jsons1 where jtag->'tag1'<54") - tdSql.checkRows(4) + tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'<=11") - tdSql.checkRows(4) + tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'>4") tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'>=5") tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'!=5") - tdSql.checkRows(6) + tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'!=55") - tdSql.checkRows(7) - # - # # where json value is double + tdSql.checkRows(3) + + # where json value is double tdSql.query("select * from jsons1 where jtag->'tag1'=1.232") tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'<1.232") - tdSql.checkRows(1) + tdSql.checkRows(0) tdSql.query("select * from jsons1 where jtag->'tag1'<=1.232") - tdSql.checkRows(2) + tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'>1.23") tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'>=1.232") tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'!=1.232") - tdSql.checkRows(6) + tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1'!=3.232") - tdSql.checkRows(7) - #tdSql.error("select * from jsons1 where jtag->'tag1'/0=3") - #tdSql.error("select * from jsons1 where jtag->'tag1'/5=1") - # - # # where json value is bool + tdSql.checkRows(3) + tdSql.query("select * from jsons1 where jtag->'tag1'/0=3") + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'/5=1") + tdSql.checkRows(1) + + # where json value is bool tdSql.query("select * from jsons1 where jtag->'tag1'=true") tdSql.checkRows(0) - #tdSql.query("select * from jsons1 where jtag->'tag1'=false") - #tdSql.checkRows(1) + tdSql.query("select * from jsons1 where jtag->'tag1'=false") + tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'!=false") - tdSql.checkRows(3) - #tdSql.error("select * from jsons1 where jtag->'tag1'>false") - # - # # where json value is null - # open - #tdSql.query("select * from jsons1 where jtag->'tag1'=null") # only json suport =null. This synatx will change later. - #tdSql.checkRows(1) - # - # # where json key is null + tdSql.checkRows(0) + tdSql.query("select * from jsons1 where jtag->'tag1'>false") + tdSql.checkRows(0) + + # where json value is null + tdSql.query("select * from jsons1 where jtag->'tag1'=null") + tdSql.checkRows(0) + + # where json key is null tdSql.query("select * from jsons1 where jtag->'tag_no_exist'=3") tdSql.checkRows(0) - # - # # where json value is not exist + + # where json value is not exist tdSql.query("select * from jsons1 where jtag->'tag1' is null") tdSql.checkData(0, 0, 'jsons1_9') tdSql.checkRows(2) @@ -309,16 +312,16 @@ class TDTestCase: tdSql.checkRows(9) tdSql.query("select * from jsons1 where jtag->'tag3' is not null") tdSql.checkRows(3) - # - # # test contains + + # test contains tdSql.query("select * from jsons1 where jtag contains 'tag1'") - tdSql.checkRows(7) + tdSql.checkRows(8) tdSql.query("select * from jsons1 where jtag contains 'tag3'") - tdSql.checkRows(3) + tdSql.checkRows(4) tdSql.query("select * from jsons1 where jtag contains 'tag_no_exist'") tdSql.checkRows(0) - # - # # test json tag in where condition with and/or + + # test json tag in where condition with and/or tdSql.query("select * from jsons1 where jtag->'tag1'=false and jtag->'tag2'='beijing'") tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1'=false or jtag->'tag2'='beijing'") @@ -335,15 +338,15 @@ class TDTestCase: tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1'='femail' and jtag contains 'tag3'") tdSql.checkRows(2) - # - # - # # test with between and + + + # test with between and tdSql.query("select * from jsons1 where jtag->'tag1' between 1 and 30") tdSql.checkRows(3) tdSql.query("select * from jsons1 where jtag->'tag1' between 'femail' and 'beijing'") tdSql.checkRows(2) - # - # # test with tbname/normal column + + # test with tbname/normal column tdSql.query("select * from jsons1 where tbname = 'jsons1_1'") tdSql.checkRows(2) tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3'") @@ -352,20 +355,18 @@ class TDTestCase: tdSql.checkRows(0) tdSql.query("select * from jsons1 where tbname = 'jsons1_1' and jtag contains 'tag3' and dataint=23") tdSql.checkRows(1) - # - # - # # test where condition like - # open - # syntax error - #tdSql.query("select *,tbname from jsons1 where jtag->'tag2' like 'bei%'") - #tdSql.checkRows(2) - #tdSql.query("select *,tbname from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null") - #tdSql.checkRows(2) - # - # # test where condition in no support in + + + # test where condition like + tdSql.query("select * from jsons1 where jtag->'tag2' like 'bei%'") + tdSql.checkRows(2) + tdSql.query("select * from jsons1 where jtag->'tag1' like 'fe%' and jtag->'tag2' is not null") + tdSql.checkRows(2) + + # test where condition in no support in # tdSql.error("select * from jsons1 where jtag->'tag1' in ('beijing')") - # - # # test where condition match/nmath + + # test where condition match/nmath tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma'") tdSql.checkRows(2) tdSql.query("select * from jsons1 where jtag->'tag1' match 'ma$'") @@ -376,23 +377,22 @@ class TDTestCase: tdSql.checkRows(1) tdSql.query("select * from jsons1 where jtag->'tag1' nmatch 'ma'") tdSql.checkRows(1) - # - # # test distinct + + # test distinct tdSql.execute("insert into jsons1_14 using jsons1 tags('{\"tag1\":\"收到货\",\"tag2\":\"\",\"tag3\":null}') values(1591062628000, 2, NULL, '你就会', 'dws')") tdSql.query("select distinct jtag->'tag1' from jsons1") tdSql.checkRows(8) tdSql.query("select distinct jtag from jsons1") tdSql.checkRows(9) - # - # #test dumplicate key with normal colomn + + #test dumplicate key with normal colomn tdSql.execute("INSERT INTO jsons1_15 using jsons1 tags('{\"tbname\":\"tt\",\"databool\":true,\"datastr\":\"是是是\"}') values(1591060828000, 4, false, 'jjsf', \"你就会\")") - #tdSql.query("select *,tbname,jtag from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") - #tdSql.checkRows(1) - # open - #tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") - #tdSql.checkRows(0) - # - # # test join + tdSql.query("select * from jsons1 where jtag->'datastr' match '是' and datastr match 'js'") + tdSql.checkRows(1) + # tdSql.query("select tbname,jtag->'tbname' from jsons1 where jtag->'tbname'='tt' and tbname='jsons1_14'") + # tdSql.checkRows(1) + + # test join tdSql.execute("create table if not exists jsons2(ts timestamp, dataInt int, dataBool bool, dataStr nchar(50), dataStrBin binary(150)) tags(jtag json)") tdSql.execute("insert into jsons2_1 using jsons2 tags('{\"tag1\":\"fff\",\"tag2\":5, \"tag3\":true}') values(1591060618000, 2, false, 'json2', '你是2')") tdSql.execute("insert into jsons2_2 using jsons2 tags('{\"tag1\":5,\"tag2\":null}') values (1591060628000, 2, true, 'json2', 'sss')") @@ -460,19 +460,18 @@ class TDTestCase: tdSql.checkColNameList(res, cname_list) # test top/bottom with group by json tag - # random failure - #tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") - #tdSql.checkRows(11) - #tdSql.checkData(0, 1, None) - #tdSql.checkData(2, 0, 4) - #tdSql.checkData(3, 0, 3) - #tdSql.checkData(3, 1, "false") - #tdSql.checkData(10, 0, 23) - #tdSql.checkData(10, 1, '"femail"') + # tdSql.query("select top(dataint,2),jtag->'tag1' from jsons1 group by jtag->'tag1' order by jtag->'tag1'") + # tdSql.checkRows(11) + # tdSql.checkData(0, 1, None) + # tdSql.checkData(2, 0, 4) + # tdSql.checkData(3, 0, 3) + # tdSql.checkData(3, 1, "false") + # tdSql.checkData(8, 0, 2) + # tdSql.checkData(10, 1, '"femail"') # test having - #tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") - #tdSql.checkRows(3) + tdSql.query("select count(*),jtag->'tag1' from jsons1 group by jtag->'tag1' having count(*) > 1") + tdSql.checkRows(3) # subquery with json tag tdSql.query("select * from (select jtag, dataint from jsons1) order by dataint") @@ -480,22 +479,13 @@ class TDTestCase: tdSql.checkData(1, 1, 1) tdSql.checkData(5, 0, '{"tag1":false,"tag2":"beijing"}') - # tdSql.query("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") - # tdSql.checkRows(11) - # tdSql.checkData(1, 0, '"femail"') - # tdSql.checkData(2, 0, 5) - # - # res = tdSql.getColNameList("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") - # cname_list = [] - # cname_list.append("jtag->'tag1'") - # tdSql.checkColNameList(res, cname_list) - # - # tdSql.query("select ts,tbname,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)") + tdSql.error("select jtag->'tag1' from (select jtag->'tag1', dataint from jsons1)") + # tdSql.query("select ts,jtag->'tag1' from (select jtag->'tag1',tbname,ts from jsons1 order by ts)") # tdSql.checkRows(11) # tdSql.checkData(1, 1, "jsons1_1") # tdSql.checkData(1, 2, '"femail"') - # - # # union all + + # union all tdSql.query("select jtag->'tag1' from jsons1 union all select jtag->'tag2' from jsons2") tdSql.checkRows(17) tdSql.query("select jtag->'tag1' from jsons1_1 union all select jtag->'tag2' from jsons2_1") @@ -508,24 +498,25 @@ class TDTestCase: tdSql.query("select dataint,jtag,tbname from jsons1 union all select dataint,jtag,tbname from jsons2") tdSql.checkRows(13) - # #show create table - # tdSql.query("show create table jsons1") - # tdSql.checkData(0, 1, 'CREATE TABLE `jsons1` (`ts` TIMESTAMP,`dataint` INT,`databool` BOOL,`datastr` NCHAR(50),`datastrbin` BINARY(150)) TAGS (`jtag` JSON)') - # - # #test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares + #show create table + tdSql.query("show create table jsons1") + tdSql.checkData(0, 1, 'CREATE STABLE `jsons1` (`ts` TIMESTAMP, `dataint` INT, `databool` BOOL, `datastr` NCHAR(50), `datastrbin` VARCHAR(150)) TAGS (`jtag` JSON) WATERMARK 5000a, 5000a') + + #test aggregate function:count/avg/twa/irate/sum/stddev/leastsquares tdSql.query("select count(*) from jsons1 where jtag is not null") tdSql.checkData(0, 0, 10) tdSql.query("select avg(dataint) from jsons1 where jtag is not null") tdSql.checkData(0, 0, 5.3) - #tdSql.error("select twa(dataint) from jsons1 where jtag is not null") - tdSql.error("select irate(dataint) from jsons1 where jtag is not null") - #tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") - #tdSql.checkData(0, 0, 49) + # tdSql.query("select twa(dataint) from jsons1 where jtag is not null") + # tdSql.checkData(0, 0, 36) + # tdSql.error("select irate(dataint) from jsons1 where jtag is not null") + tdSql.query("select sum(dataint) from jsons1 where jtag->'tag1' is not null") + tdSql.checkData(0, 0, 45) tdSql.query("select stddev(dataint) from jsons1 where jtag->'tag1'>1") tdSql.checkData(0, 0, 4.496912521) - #tdSql.error("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null") - # - # #test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp + tdSql.query("SELECT LEASTSQUARES(dataint, 1, 1) from jsons1 where jtag is not null") + + #test selection function:min/max/first/last/top/bottom/percentile/apercentile/last_row/interp tdSql.query("select min(dataint) from jsons1 where jtag->'tag1'>1") tdSql.checkData(0, 0, 1) tdSql.query("select max(dataint) from jsons1 where jtag->'tag1'>1") @@ -541,13 +532,16 @@ class TDTestCase: tdSql.query("select percentile(dataint,20) from jsons1 where jtag->'tag1'>1") tdSql.query("select apercentile(dataint, 50) from jsons1 where jtag->'tag1'>1") tdSql.checkData(0, 0, 1.5) - #tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1") - #tdSql.checkData(0, 0, 11) - #tdSql.error("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1") - # - # #test calculation function:diff/derivative/spread/ceil/floor/round/ - #tdSql.error("select diff(dataint) from jsons1 where jtag->'tag1'>1") - #tdSql.error("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1") + # tdSql.query("select last_row(dataint) from jsons1 where jtag->'tag1'>1") + # tdSql.query("select interp(dataint) from jsons1 where ts = '2020-06-02 09:17:08.000' and jtag->'tag1'>1") + + #test calculation function:diff/derivative/spread/ceil/floor/round/ + tdSql.query("select diff(dataint) from jsons1 where jtag->'tag1'>1") + # tdSql.checkRows(2) + # tdSql.checkData(0, 0, -1) + # tdSql.checkData(1, 0, 10) + tdSql.query("select derivative(dataint, 10m, 0) from jsons1 where jtag->'tag1'>1") + tdSql.checkData(0, 0, -2) tdSql.query("select spread(dataint) from jsons1 where jtag->'tag1'>1") tdSql.checkData(0, 0, 10) tdSql.query("select ceil(dataint) from jsons1 where jtag->'tag1'>1") diff --git a/tests/system-test/2-query/sample.py b/tests/system-test/2-query/sample.py index a84d93404a9f082ffa85c50d1eb8c015309308ec..33ef7e65db0fbb3d40730f9cc012fd5d0d7b9d15 100644 --- a/tests/system-test/2-query/sample.py +++ b/tests/system-test/2-query/sample.py @@ -624,8 +624,7 @@ class TDTestCase: tdLog.info(" sample data is in datas groups ,successed sql is : %s" % sample_query ) else: tdLog.exit(" sample data is not in datas groups ,failed sql is : %s" % sample_query ) - - + def basic_sample_query(self): tdSql.execute(" drop database if exists db ") tdSql.execute(" create database if not exists db duration 300 ") @@ -760,6 +759,14 @@ class TDTestCase: self.check_sample("select sample( c1 ,3 ) from t1 where c1 between 1 and 10" ,"select c1 from t1 where c1 between 1 and 10") + tdSql.query("select sample(c1,2) ,c2,c3 ,c5 from stb1") + tdSql.checkRows(2) + tdSql.checkCols(4) + + self.check_sample("select sample( c1 ,3 ),c2,c3,c4,c5 from t1 where c1 between 1 and 10" ,"select c1,c2,c3,c4,c5 from t1 where c1 between 1 and 10") + self.check_sample("select sample( c1 ,3 ),c2,c3,c4,c5 from stb1 where c1 between 1 and 10" ,"select c1,c2,c3,c4,c5 from stb1 where c1 between 1 and 10") + self.check_sample("select sample( c1 ,3 ),t1 from stb1 where c1 between 1 and 10" ,"select c1,t1 from stb1 where c1 between 1 and 10") + # join tdSql.query("select sample( ct4.c1 , 1 ) from ct1, ct4 where ct4.ts=ct1.ts") @@ -772,8 +779,8 @@ class TDTestCase: self.check_sample("select sample(c1,2) from stb1 partition by tbname" , "select c1 from stb1 partition by tbname") # nest query - # tdSql.query("select sample(c1,2) from (select c1 from t1); ") - # tdSql.checkRows(2) + tdSql.query("select sample(c1,2) from (select c1 from t1); ") + tdSql.checkRows(2) # union all tdSql.query("select sample(c1,2) from t1 union all select sample(c1,3) from t1") diff --git a/tests/system-test/2-query/substr.py b/tests/system-test/2-query/substr.py index e78606826b610223f5ac30f09560e8ec73800891..79b5ac515bc9810284a4eb4820e24dd946d60591 100644 --- a/tests/system-test/2-query/substr.py +++ b/tests/system-test/2-query/substr.py @@ -23,6 +23,9 @@ CHAR_COL = [ BINARY_COL, NCHAR_COL, ] BOOLEAN_COL = [ BOOL_COL, ] TS_TYPE_COL = [ TS_COL, ] +ERR_POS = 0 +CURRENT_POS = 1 +LENS = 6 class TDTestCase: @@ -76,6 +79,7 @@ class TDTestCase: if pos < 1: tdSql.error(f"select substr( {condition}, {pos}, {lens}) , {condition} from {tbname} ") + break tdSql.query(f"select substr( {condition}, {pos}, {lens}) , {condition} from {tbname} ") for j in range(tdSql.queryRows): @@ -127,7 +131,7 @@ class TDTestCase: tdLog.printNoPrefix("==========current sql condition check , must return query ok==========") tbname = ["ct1", "ct2", "ct4", "t1", "stb1"] for tb in tbname: - self.__substr_check(tb, 1, 6) + self.__substr_check(tb, CURRENT_POS, LENS) tdLog.printNoPrefix(f"==========current sql condition check in {tb} over==========") def __test_error(self): @@ -137,7 +141,7 @@ class TDTestCase: for tb in tbname: for errsql in self.__substr_err_check(tb): tdSql.error(sql=errsql) - self.__substr_check(tb, 0, 6) + self.__substr_check(tb, ERR_POS, LENS) tdLog.printNoPrefix(f"==========err sql condition check in {tb} over==========") @@ -154,7 +158,7 @@ class TDTestCase: ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, {FLOAT_COL} float, {DOUBLE_COL} double, {BOOL_COL} bool, {BINARY_COL} binary(16), {NCHAR_COL} nchar(32), {TS_COL} timestamp - ) tags (t1 int) + ) tags (tag1 int) ''' create_ntb_sql = f'''create table t1( ts timestamp, {INT_COL} int, {BINT_COL} bigint, {SINT_COL} smallint, {TINT_COL} tinyint, diff --git a/tests/system-test/2-query/sum.py b/tests/system-test/2-query/sum.py index ea0e1f7fae214fa009e7230b524959b7afef59da..f6ff4989e7e07c1bcaf508960ac609d8d05d5657 100644 --- a/tests/system-test/2-query/sum.py +++ b/tests/system-test/2-query/sum.py @@ -73,7 +73,7 @@ class TDTestCase: f"select sum(ceil( {un_num_col} )) from {tbanme} ", ) ) - sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL ) + # sqls.extend( f"select sum( {un_num_col} + {un_num_col_2} ) from {tbanme} " for un_num_col_2 in UN_NUM_COL ) sqls.extend( f"select sum( {num_col} + {ts_col} ) from {tbanme} " for num_col in NUM_COL for ts_col in TS_TYPE_COL) sqls.extend( diff --git a/tests/system-test/2-query/top.py b/tests/system-test/2-query/top.py index 83f535856ee8b9456d7759128063d5149df82ea1..86e201ea9a6cdf1d2e0cccd8904cbede19b41282 100644 --- a/tests/system-test/2-query/top.py +++ b/tests/system-test/2-query/top.py @@ -11,103 +11,159 @@ # -*- coding: utf-8 -*- -import random import string from util.common import * from util.log import * from util.cases import * from util.sql import * - +from util.sqlset import * class TDTestCase: def init(self, conn, logSql): tdLog.debug("start to execute %s" % __file__) tdSql.init(conn.cursor()) - + self.setsql = TDSetSql() + self.ntbname = 'ntb' self.rowNum = 10 self.tbnum = 20 self.ts = 1537146000000 self.binary_str = 'taosdata' self.nchar_str = '涛思数据' - def top_check_base(self): - tdSql.prepare() - tdSql.execute('''create table stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - tdSql.execute("create table stb_1 using stb tags('beijing')") - for i in range(self.rowNum): - tdSql.execute(f"insert into stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) - column_list = ['col1','col2','col3','col4','col5','col6','col7','col8'] - error_column_list = ['col11','col12','col13'] - error_param_list = [0,101] - for i in column_list: - tdSql.query(f'select top({i},2) from stb_1') - tdSql.checkRows(2) - tdSql.checkEqual(tdSql.queryResult,[(9,),(10,)]) - for j in error_param_list: - tdSql.error(f'select top({i},{j}) from stb_1') - for i in error_column_list: - tdSql.error(f'select top({i},10) from stb_1') - tdSql.query("select ts,top(col1, 2),ts from stb_1 group by tbname") - tdSql.checkRows(2) - tdSql.query('select top(col2,1) from stb_1 interval(1y) order by col2') - tdSql.checkData(0,0,10) - tdSql.error("select * from stb_1 where top(col2,1)=1") - tdSql.execute('drop database db') - def top_check_stb_distribute(self): - # prepare data for vgroup 4 + self.column_dict = { + 'ts' : 'timestamp', + 'col1': 'tinyint', + 'col2': 'smallint', + 'col3': 'int', + 'col4': 'bigint', + 'col5': 'tinyint unsigned', + 'col6': 'smallint unsigned', + 'col7': 'int unsigned', + 'col8': 'bigint unsigned', + 'col9': 'float', + 'col10': 'double', + 'col11': 'bool', + 'col12': 'binary(20)', + 'col13': 'nchar(20)' + } + + self.param_list = [1,100] + + def insert_data(self,column_dict,tbname,row_num): + sql = '' + for k, v in column_dict.items(): + if v.lower() == 'timestamp' or v.lower() == 'tinyint' or v.lower() == 'smallint' or v.lower() == 'int' or v.lower() == 'bigint' or \ + v.lower() == 'tinyint unsigned' or v.lower() == 'smallint unsigned' or v.lower() == 'int unsigned' or v.lower() == 'bigint unsigned' or v.lower() == 'bool': + sql += '%d,' + elif v.lower() == 'float' or v.lower() == 'double': + sql += '%f,' + elif 'binary' in v.lower(): + sql += f'"{self.binary_str}%d",' + elif 'nchar' in v.lower(): + sql += f'"{self.nchar_str}%d",' + insert_sql = f'insert into {tbname} values({sql[:-1]})' + for i in range(row_num): + insert_list = [] + for k, v in column_dict.items(): + if v.lower() in[ 'tinyint' , 'smallint' , 'int', 'bigint' , 'tinyint unsigned' , 'smallint unsigned' , 'int unsigned' , 'bigint unsigned'] or\ + 'binary' in v.lower() or 'nchar' in v.lower(): + insert_list.append(1 + i) + elif v.lower() == 'float' or v.lower() == 'double': + insert_list.append(0.1 + i) + elif v.lower() == 'bool': + insert_list.append(i % 2) + elif v.lower() == 'timestamp': + insert_list.append(self.ts + i) + tdSql.execute(insert_sql%(tuple(insert_list))) + pass + def top_check_data(self,tbname,tb_type): + new_column_dict = {} + for param in self.param_list: + for k,v in self.column_dict.items(): + if v.lower() in ['tinyint','smallint','int','bigint','tinyint unsigned','smallint unsigned','int unsigned','bigint unsigned']: + tdSql.query(f'select top({k},{param}) from {tbname}') + if param >= self.rowNum: + if tb_type in ['normal_table','child_table']: + tdSql.checkRows(self.rowNum) + values_list = [] + for i in range(self.rowNum): + tp = (self.rowNum-i,) + values_list.insert(0,tp) + tdSql.checkEqual(tdSql.queryResult,values_list) + elif tb_type == 'stable': + tdSql.checkRows(param) + elif param < self.rowNum: + if tb_type in ['normal_table','child_table']: + tdSql.checkRows(param) + values_list = [] + for i in range(param): + tp = (self.rowNum-i,) + values_list.insert(0,tp) + tdSql.checkEqual(tdSql.queryResult,values_list) + elif tb_type == 'stable': + tdSql.checkRows(param) + for i in [self.param_list[0]-1,self.param_list[-1]+1]: + tdSql.error(f'select top({k},{i}) from {tbname}') + new_column_dict.update({k:v}) + elif v.lower() == 'bool' or 'binary' in v.lower() or 'nchar' in v.lower(): + tdSql.error(f'select top({k},{param}) from {tbname}') + tdSql.error(f'select * from {tbname} where top({k},{param})=1') + for key in new_column_dict.keys(): + for k in self.column_dict.keys(): + if key == k : + continue + else: + tdSql.query(f'select top({key},2),{k} from {tbname} group by tbname') + if tb_type == 'normal_table' or tb_type == 'child_table': + tdSql.checkRows(2) + else: + tdSql.checkRows(2*self.tbnum) + def top_check_stb(self): dbname = tdCom.getLongName(10, "letters") stbname = tdCom.getLongName(5, "letters") + tag_dict = { + 't0':'int' + } + tag_values = [ + f'1' + ] tdSql.execute(f"create database if not exists {dbname} vgroups 2") tdSql.execute(f'use {dbname}') - # build 20 child tables,every table insert 10 rows - tdSql.execute(f'''create table {stbname}(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') + tdSql.execute(self.setsql.set_create_stable_sql(stbname,self.column_dict,tag_dict)) + for i in range(self.tbnum): - tdSql.execute(f"create table {stbname}_{i} using {stbname} tags('beijing')") - tdSql.execute(f"insert into {stbname}_{i}(ts) values(%d)" % (self.ts - 1-i)) - column_list = ['col1','col2','col3','col4','col5','col6','col7','col8'] - for i in [f'{stbname}', f'{dbname}.{stbname}']: - for j in column_list: - tdSql.query(f"select top({j},1) from {i}") - tdSql.checkRows(0) + tdSql.execute(f"create table {stbname}_{i} using {stbname} tags({tag_values[0]})") + tdSql.execute(self.insert_data(self.column_dict,f'{stbname}_{i}',self.rowNum)) tdSql.query('show tables') vgroup_list = [] for i in range(len(tdSql.queryResult)): vgroup_list.append(tdSql.queryResult[i][6]) - vgroup_list_set = set(vgroup_list) + vgroup_list_set = set(vgroup_list) for i in vgroup_list_set: vgroups_num = vgroup_list.count(i) - if vgroups_num >=2: + if vgroups_num >= 2: tdLog.info(f'This scene with {vgroups_num} vgroups is ok!') - continue + else: - tdLog.exit(f'This scene does not meet the requirements with {vgroups_num} vgroup!\n') - for i in range(self.rowNum): - for j in range(self.tbnum): - tdSql.execute(f"insert into {stbname}_{j} values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" - % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) - - error_column_list = ['col11','col12','col13'] - error_param_list = [0,101] - for i in column_list: - tdSql.query(f'select top({i},2) from {stbname}') - tdSql.checkRows(2) - tdSql.checkEqual(tdSql.queryResult,[(10,),(10,)]) - for j in error_param_list: - tdSql.error(f'select top({i},{j}) from {stbname}') - for i in error_column_list: - tdSql.error(f'select top({i},10) from {stbname}') + tdLog.exit( + 'This scene does not meet the requirements with {vgroups_num} vgroup!\n') + for i in range(self.tbnum): + self.top_check_data(f'{stbname}_{i}','child_table') + self.top_check_data(stbname,'stable') + tdSql.execute(f'drop database {dbname}') + + def top_check_ntb(self): + tdSql.prepare() + tdSql.execute(self.setsql.set_create_normaltable_sql(self.ntbname,self.column_dict)) + self.insert_data(self.column_dict,self.ntbname,self.rowNum) + self.top_check_data(self.ntbname,'normal_table') + tdSql.execute('drop database db') - tdSql.query(f"select ts,top(col1, 2),ts from {stbname} group by tbname") - tdSql.checkRows(2*self.tbnum) - tdSql.query(f'select top(col2,1) from {stbname} interval(1y) order by col2') - tdSql.checkData(0,0,10) - tdSql.error(f"select * from {stbname} where top(col2,1)=1") def run(self): - self.top_check_base() - self.top_check_stb_distribute() + self.top_check_ntb() + self.top_check_stb() + + def stop(self): tdSql.close() tdLog.success("%s successfully executed" % __file__) diff --git a/tests/system-test/2-query/twa.py b/tests/system-test/2-query/twa.py new file mode 100644 index 0000000000000000000000000000000000000000..b400e503d9b7197718c3cd7feb09a8a7bac8b6a7 --- /dev/null +++ b/tests/system-test/2-query/twa.py @@ -0,0 +1,150 @@ +from util.log import * +from util.cases import * +from util.sql import * +import numpy as np +import random ,os ,sys +import platform +import math + +class TDTestCase: + updatecfgDict = {'debugFlag': 143 ,"cDebugFlag":143,"uDebugFlag":143 ,"rpcDebugFlag":143 , "tmrDebugFlag":143 , + "jniDebugFlag":143 ,"simDebugFlag":143,"dDebugFlag":143, "dDebugFlag":143,"vDebugFlag":143,"mDebugFlag":143,"qDebugFlag":143, + "wDebugFlag":143,"sDebugFlag":143,"tsdbDebugFlag":143,"tqDebugFlag":143 ,"fsDebugFlag":143 ,"fnDebugFlag":143, + "maxTablesPerVnode":2 ,"minTablesPerVnode":2,"tableIncStepPerVnode":2 } + + def init(self, conn, logSql): + tdLog.debug("start to execute %s" % __file__) + tdSql.init(conn.cursor()) + self.vnode_disbutes = None + self.ts = 1537146000000 + self.tb_nums = 20 + self.row_nums = 100 + self.time_step = 1000 + + def prepare_datas_of_distribute(self): + + # prepate datas for 20 tables distributed at different vgroups + tdSql.execute("create database if not exists testdb keep 3650 duration 1000 vgroups 5") + tdSql.execute(" use testdb ") + tdSql.execute( + '''create table stb1 + (ts timestamp, c1 int, c2 bigint, c3 smallint, c4 tinyint, c5 float, c6 double, c7 bool, c8 binary(16),c9 nchar(32), c10 timestamp) + tags (t0 timestamp, t1 int, t2 bigint, t3 smallint, t4 tinyint, t5 float, t6 double, t7 bool, t8 binary(16),t9 nchar(32)) + ''' + ) + + for i in range(self.tb_nums): + tdSql.execute(f'create table ct{i+1} using stb1 tags ( now(), {1*i}, {11111*i}, {111*i}, {11*i}, {1.11*i}, {11.11*i}, {i%2}, "binary{i}", "nchar{i}" )') + ts = self.ts + for j in range(self.row_nums): + ts+=j*self.time_step + tdSql.execute( + f"insert into ct{i+1} values({ts}, 1, 11111, 111, 1, 1.11, 11.11, 2, 'binary{j}', 'nchar{j}', now()+{1*j}a )" + ) + + tdSql.execute("insert into ct1 values (now()-810d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct1 values (now()-400d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + tdSql.execute("insert into ct1 values (now()+90d, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL, NULL ) ") + + tdLog.info(" prepare data for distributed_aggregate done! ") + + def twa_support_types(self): + tdSql.query("desc stb1 ") + schema_list = tdSql.queryResult + for col_type in schema_list: + if col_type[1] in ["TINYINT" ,"SMALLINT","BIGINT" ,"INT","FLOAT","DOUBLE"]: + tdSql.query(f" select twa({col_type[0]}) from stb1 partition by tbname ") + else: + tdSql.error(f" select twa({col_type[0]}) from stb1 partition by tbname ") + + + def check_distribute_datas(self): + # get vgroup_ids of all + tdSql.query("show vgroups ") + vgroups = tdSql.queryResult + + vnode_tables={} + + for vgroup_id in vgroups: + vnode_tables[vgroup_id[0]]=[] + + # check sub_table of per vnode ,make sure sub_table has been distributed + tdSql.query("show tables like 'ct%'") + table_names = tdSql.queryResult + tablenames = [] + for table_name in table_names: + vnode_tables[table_name[6]].append(table_name[0]) + self.vnode_disbutes = vnode_tables + + count = 0 + for k ,v in vnode_tables.items(): + if len(v)>=2: + count+=1 + if count < 2: + tdLog.exit(" the datas of all not satisfy sub_table has been distributed ") + + def distribute_twa_query(self): + # basic filter + tdSql.query(" select twa(c1) from ct1 ") + tdSql.checkData(0,0,1.000000000) + + tdSql.query(" select twa(c1) from stb1 partition by tbname ") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,0,1.000000000) + + tdSql.query(" select twa(c2) from stb1 group by tbname ") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,0,11111.000000000) + + tdSql.query("select twa(c1+c2) from stb1 partition by tbname ") + tdSql.checkData(0,0,11112.000000000) + + tdSql.query("select twa(c1) from stb1 partition by t1") + tdSql.checkRows(self.tb_nums) + tdSql.checkData(0,0,1.000000000) + + # union all + tdSql.query(" select twa(c1) from stb1 partition by tbname union all select twa(c1) from stb1 partition by tbname ") + tdSql.checkRows(40) + tdSql.checkData(0,0,1.000000000) + + # join + + tdSql.execute(" create database if not exists db ") + tdSql.execute(" use db ") + tdSql.execute(" create stable st (ts timestamp , c1 int ,c2 float) tags(t1 int) ") + tdSql.execute(" create table tb1 using st tags(1) ") + tdSql.execute(" create table tb2 using st tags(2) ") + + + for i in range(10): + ts = i*10 + self.ts + tdSql.execute(f" insert into tb1 values({ts},{i},{i}.0)") + tdSql.execute(f" insert into tb2 values({ts},{i},{i}.0)") + + tdSql.query(" select twa(tb1.c1), twa(tb2.c2) from tb1, tb2 where tb1.ts=tb2.ts ") + tdSql.checkRows(1) + tdSql.checkData(0,0,4.500000000) + tdSql.checkData(0,1,4.500000000) + + # group by + tdSql.execute(" use testdb ") + + # mixup with other functions + tdSql.query(" select twa(c1),twa(c2),max(c1),elapsed(ts) from stb1 ") + tdSql.checkData(0,0,1.000000000) + tdSql.checkData(0,1,11111.000000000) + tdSql.checkData(0,2,1) + + def run(self): + self.prepare_datas_of_distribute() + self.check_distribute_datas() + self.twa_support_types() + self.distribute_twa_query() + + def stop(self): + tdSql.close() + tdLog.success("%s successfully executed" % __file__) + +tdCases.addWindows(__file__, TDTestCase()) +tdCases.addLinux(__file__, TDTestCase()) diff --git a/tests/system-test/6-cluster/5dnode3mnodeStop.py b/tests/system-test/6-cluster/5dnode3mnodeStop.py index 654b27bfc078db0a1fb3e50a84037e5938271d23..e1f66ceacb9a9e2f5f4f17547243262a227c4eb7 100644 --- a/tests/system-test/6-cluster/5dnode3mnodeStop.py +++ b/tests/system-test/6-cluster/5dnode3mnodeStop.py @@ -7,31 +7,25 @@ import os from util.log import * from util.sql import * from util.cases import * +from util.dnodes import * from util.dnodes import TDDnodes from util.dnodes import TDDnode +from util.cluster import * +from test import tdDnodes + import time import socket import subprocess from multiprocessing import Process -class MyDnodes(TDDnodes): - def __init__(self ,dnodes_lists): - super(MyDnodes,self).__init__() - self.dnodes = dnodes_lists # dnode must be TDDnode instance - self.simDeployed = False + class TDTestCase: def init(self,conn ,logSql): tdLog.debug(f"start to excute {__file__}") - self.TDDnodes = None - - def buildcluster(self,dnodenumber): - self.depoly_cluster(dnodenumber) - self.master_dnode = self.TDDnodes.dnodes[0] - self.host=self.master_dnode.cfgDict["fqdn"] - conn1 = taos.connect(self.master_dnode.cfgDict["fqdn"] , config=self.master_dnode.cfgDir) - tdSql.init(conn1.cursor()) - + tdSql.init(conn.cursor()) + self.host = socket.gethostname() + def getBuildPath(self): selfPath = os.path.dirname(os.path.realpath(__file__)) @@ -70,72 +64,34 @@ class TDTestCase: for i in range(4): tdSql.execute(f'create table ct{i+1} using stb1 tags ( {i+1} )') - def depoly_cluster(self ,dnodes_nums): - - testCluster = False - valgrind = 0 - hostname = socket.gethostname() - dnodes = [] - start_port = 6030 - for num in range(1, dnodes_nums+1): - dnode = TDDnode(num) - dnode.addExtraCfg("firstEp", f"{hostname}:{start_port}") - dnode.addExtraCfg("fqdn", f"{hostname}") - dnode.addExtraCfg("serverPort", f"{start_port + (num-1)*100}") - dnode.addExtraCfg("monitorFqdn", hostname) - dnode.addExtraCfg("monitorPort", 7043) - dnodes.append(dnode) - - self.TDDnodes = MyDnodes(dnodes) - self.TDDnodes.init("") - self.TDDnodes.setTestCluster(testCluster) - self.TDDnodes.setValgrind(valgrind) - self.TDDnodes.stopAll() - for dnode in self.TDDnodes.dnodes: - self.TDDnodes.deploy(dnode.index,{}) - - for dnode in self.TDDnodes.dnodes: - self.TDDnodes.starttaosd(dnode.index) - - # create cluster - for dnode in self.TDDnodes.dnodes[1:]: - # tdLog.debug(dnode.cfgDict) - dnode_id = dnode.cfgDict["fqdn"] + ":" +dnode.cfgDict["serverPort"] - dnode_first_host = dnode.cfgDict["firstEp"].split(":")[0] - dnode_first_port = dnode.cfgDict["firstEp"].split(":")[-1] - cmd = f" taos -h {dnode_first_host} -P {dnode_first_port} -s ' create dnode \"{dnode_id} \" ' ;" - tdLog.debug(cmd) - os.system(cmd) - - time.sleep(2) - tdLog.info(" create cluster with %d dnode done! " %dnodes_nums) - + def check3mnode(self): count=0 while count < 10: time.sleep(1) tdSql.query("show mnodes;") if tdSql.checkRows(3) : - tdLog.debug("mnode is three nodes") + tdLog.debug("mnode is three nodes") + else: + tdLog.exit("mnode number is correct") if tdSql.queryResult[0][2]=='leader' : if tdSql.queryResult[1][2]=='follower': if tdSql.queryResult[2][2]=='follower': tdLog.debug("three mnodes is ready in 10s") break - elif tdSql.queryResult[0][2]=='follower' : - if tdSql.queryResult[1][2]=='leader': + elif tdSql.queryResult[1][2]=='leader' : + if tdSql.queryResult[0][2]=='follower': if tdSql.queryResult[2][2]=='follower': tdLog.debug("three mnodes is ready in 10s") break - elif tdSql.queryResult[0][2]=='follower' : + elif tdSql.queryResult[2][2]=='leader' : if tdSql.queryResult[1][2]=='follower': - if tdSql.queryResult[2][2]=='leader': + if tdSql.queryResult[0][2]=='follower': tdLog.debug("three mnodes is ready in 10s") break count+=1 else: - tdLog.debug("three mnodes is not ready in 10s ") - return -1 + tdLog.exit("three mnodes is not ready in 10s ") tdSql.query("show mnodes;") tdSql.checkRows(3) @@ -153,6 +109,8 @@ class TDTestCase: tdSql.query("show mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") + else: + tdLog.exit("mnode number is correct") if tdSql.queryResult[0][2]=='offline' : if tdSql.queryResult[1][2]=='leader': if tdSql.queryResult[2][2]=='follower': @@ -164,8 +122,8 @@ class TDTestCase: break count+=1 else: - tdLog.debug("stop mnodes on dnode 2 failed in 10s ") - return -1 + tdLog.exit("stop mnodes on dnode 2 failed in 10s ") + tdSql.error("drop mnode on dnode 1;") tdSql.query("show mnodes;") @@ -185,6 +143,8 @@ class TDTestCase: tdSql.query("show mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") + else: + tdLog.exit("mnode number is correct") if tdSql.queryResult[0][2]=='leader' : if tdSql.queryResult[1][2]=='offline': if tdSql.queryResult[2][2]=='follower': @@ -192,8 +152,8 @@ class TDTestCase: break count+=1 else: - tdLog.debug("stop mnodes on dnode 2 failed in 10s ") - return -1 + tdLog.exit("stop mnodes on dnode 2 failed in 10s ") + tdSql.error("drop mnode on dnode 2;") tdSql.query("show mnodes;") @@ -215,6 +175,8 @@ class TDTestCase: tdSql.query("show mnodes;") if tdSql.checkRows(3) : tdLog.debug("mnode is three nodes") + else: + tdLog.exit("mnode number is correct") if tdSql.queryResult[0][2]=='leader' : if tdSql.queryResult[2][2]=='offline': if tdSql.queryResult[1][2]=='follower': @@ -222,8 +184,8 @@ class TDTestCase: break count+=1 else: - tdLog.debug("stop mnodes on dnode 3 failed in 10s") - return -1 + tdLog.exit("stop mnodes on dnode 3 failed in 10s") + tdSql.error("drop mnode on dnode 3;") tdSql.query("show mnodes;") tdSql.checkRows(3) @@ -237,15 +199,30 @@ class TDTestCase: tdSql.checkData(2,2,'offline') tdSql.checkData(2,3,'ready') - + + def check_dnodes_status(self,dnodeNumbers): + count=0 + while count < 5: + tdSql.query("show dnodes") + # tdLog.debug(tdSql.queryResult) + status=0 + for i in range(dnodeNumbers): + if tdSql.queryResult[i][4] == "ready": + status+=1 + tdLog.debug(status) + + if status == dnodeNumbers: + tdLog.debug(" create cluster with %d dnode and check cluster dnode all ready within 5s! " %dnodeNumbers) + break + count+=1 + time.sleep(1) + else: + tdLog.exit("create cluster with %d dnode but check dnode not ready within 5s ! "%dnodeNumbers) def five_dnode_three_mnode(self,dnodenumber): - tdSql.query("show dnodes;") - tdSql.checkData(0,1,'%s:6030'%self.host) - tdSql.checkData(4,1,'%s:6430'%self.host) - tdSql.checkData(0,4,'ready') - tdSql.checkData(4,4,'ready') + self.check_dnodes_status(5) tdSql.query("show mnodes;") + tdLog.debug(self.host) tdSql.checkRows(1) tdSql.checkData(0,1,'%s:6030'%self.host) tdSql.checkData(0,2,'leader') @@ -262,26 +239,32 @@ class TDTestCase: tdSql.error("create mnode on dnode 2") tdSql.query("show dnodes;") - tdLog.debug(tdSql.queryResult) + # tdLog.debug(tdSql.queryResult) tdLog.debug("stop and follower of mnode") - self.TDDnodes.stoptaosd(2) + tdDnodes=cluster.dnodes + # tdLog.debug(tdDnodes[0]) + + tdDnodes[1].stoptaosd() self.check3mnode2off() - self.TDDnodes.starttaosd(2) + tdDnodes[1].starttaosd() + self.check3mnode() - self.TDDnodes.stoptaosd(3) + tdDnodes[2].stoptaosd() self.check3mnode3off() - self.TDDnodes.starttaosd(3) + tdDnodes[2].starttaosd() + self.check3mnode() - self.TDDnodes.stoptaosd(1) + tdDnodes[0].stoptaosd() self.check3mnode1off() - self.TDDnodes.starttaosd(1) + tdDnodes[0].starttaosd() + self.check3mnode() self.check3mnode() stopcount =0 while stopcount <= 2: for i in range(dnodenumber): - self.TDDnodes.stoptaosd(i+1) + tdDnodes[i].stoptaosd() # if i == 1 : # self.check3mnode2off() # elif i == 2 : @@ -289,24 +272,14 @@ class TDTestCase: # elif i == 0: # self.check3mnode1off() - self.TDDnodes.starttaosd(i+1) + tdDnodes[i].starttaosd() # self.check3mnode() stopcount+=1 self.check3mnode() - - def getConnection(self, dnode): - host = dnode.cfgDict["fqdn"] - port = dnode.cfgDict["serverPort"] - config_dir = dnode.cfgDir - return taos.connect(host=host, port=int(port), config=config_dir) - - def run(self): - # tdLog.debug(self.master_dnode.cfgDict) - self.buildcluster(5) self.five_dnode_three_mnode(5) - + def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/7-tmq/stbFilter.py b/tests/system-test/7-tmq/stbFilter.py new file mode 100644 index 0000000000000000000000000000000000000000..542894574bcba9818ef8c859538a3c41a6f02876 --- /dev/null +++ b/tests/system-test/7-tmq/stbFilter.py @@ -0,0 +1,260 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 4 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 5000" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+9000) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + # if expectRowsList[2] != resultList[0]: + # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + # tdLog.exit("2 tmq consume rows error!") + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def tmqCase2(self): + tdLog.printNoPrefix("======== test case 2: ") + paraDict = {'dbName': 'db2', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 10, + 'rowsPerTbl': 10000, + 'batchNum': 100, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tmqCom.create_ctable(tdSql, dbName=paraDict["dbName"],stbName=paraDict["stbName"],ctbPrefix=paraDict['ctbPrefix'], ctbNum=paraDict['ctbNum']) + tdLog.info("insert data") + tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + # sqlString = "create topic %s as select ts, sin(c1), pow(c2,3) from %s.%s where c2 >= 0" %(topicNameList[0], paraDict['dbName'], paraDict['stbName']) + queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where c2 >= 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + queryString = "select ts, sin(c1), pow(c2,3) from %s.%s where sin(c2) >= 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + queryString = "select ts, sin(c1), floor(pow(c2,3)), c2 from %s.%s where (sin(c2) >= 0) and (floor(pow(c2,3)) %% 9 == 0)" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # start tmq consume processor + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 0 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + # # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[2] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + tdLog.exit("2 tmq consume rows error!") + + # time.sleep(10) + # for i in range(len(topicNameList)): + # tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 2 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + self.tmqCase2() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py new file mode 100644 index 0000000000000000000000000000000000000000..0e55dfa19d33a14703b9d985446bf532537b4925 --- /dev/null +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -0,0 +1,182 @@ + +import taos +import sys +import time +import socket +import os +import threading + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 10000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2', 'topic3'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + queryString = "select ts, log(c1), ceil(pow(c1,3)) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + + queryString = "select ts, log(c1), cos(c1) from %s.%s where c1 > 3169" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + + queryString = "select ts, log(c1), atan(c1) from %s.%s where ts >= %d" %(paraDict['dbName'], paraDict['stbName'], paraDict["startTs"]+6137) + sqlString = "create topic %s as %s" %(topicNameList[2], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 2 + topicList = topicNameList[2] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + # if expectRowsList[2] != resultList[0]: + # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) + # tdLog.exit("2 tmq consume rows error!") + + # self.checkFileContent(consumerId, queryString) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 68adaa412603e71413152257ad7d005d4017c753..9b23056546de3d7e74cd2bae9c3503d95df53a44 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -42,9 +42,11 @@ class TMQCom: tdSql.query("create database if not exists %s vgroups 1"%(cdbName)) tdSql.query("drop table if exists %s.consumeinfo "%(cdbName)) tdSql.query("drop table if exists %s.consumeresult "%(cdbName)) + tdSql.query("drop table if exists %s.notifyinfo "%(cdbName)) tdSql.query("create table %s.consumeinfo (ts timestamp, consumerid int, topiclist binary(1024), keylist binary(1024), expectmsgcnt bigint, ifcheckdata int, ifmanualcommit int)"%cdbName) tdSql.query("create table %s.consumeresult (ts timestamp, consumerid int, consummsgcnt bigint, consumrowcnt bigint, checkresult int)"%cdbName) + tdSql.query("create table %s.notifyinfo (ts timestamp, cmdid int, consumerid int)"%cdbName) def initConsumerInfoTable(self,cdbName='cdb'): tdLog.info("drop consumeinfo table") @@ -92,13 +94,190 @@ class TMQCom: tdLog.info(shellCmd) os.system(shellCmd) + def getStartConsumeNotifyFromTmqsim(self,cdbName='cdb'): + while 1: + tdSql.query("select * from %s.notifyinfo"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if (tdSql.getRows() == 1) and (tdSql.getData(0, 1) == 0): + break + else: + time.sleep(0.1) + return + + def getStartCommitNotifyFromTmqsim(self,cdbName='cdb'): + while 1: + tdSql.query("select * from %s.notifyinfo"%cdbName) + #tdLog.info("row: %d, %l64d, %l64d"%(tdSql.getData(0, 1),tdSql.getData(0, 2),tdSql.getData(0, 3)) + if tdSql.getRows() == 2 : + print(tdSql.getData(0, 1), tdSql.getData(1, 1)) + if tdSql.getData(1, 1) == 1: + break + time.sleep(0.1) + return + + def create_database(self,tsql, dbName,dropFlag=1,vgroups=4,replica=1): + if dropFlag == 1: + tsql.execute("drop database if exists %s"%(dbName)) + + tsql.execute("create database if not exists %s vgroups %d replica %d"%(dbName, vgroups, replica)) + tdLog.debug("complete to create database %s"%(dbName)) + return + + def create_stable(self,tsql, dbName,stbName): + tsql.execute("create table if not exists %s.%s (ts timestamp, c1 int, c2 int, c3 binary(16)) tags(t1 int, t2 binary(32))"%(dbName, stbName)) + tdLog.debug("complete to create %s.%s" %(dbName, stbName)) + return + + def create_ctable(self,tsql=None, dbName='dbx',stbName='stb',ctbPrefix='ctb',ctbNum=1): + tsql.execute("use %s" %dbName) + pre_create = "create table" + sql = pre_create + #tdLog.debug("doing create one stable %s and %d child table in %s ..." %(stbname, count ,dbname)) + for i in range(ctbNum): + tagValue = 'beijing' + if (i % 2 == 0): + tagValue = 'shanghai' + + sql += " %s%d using %s tags(%d, '%s')"%(ctbPrefix,i,stbName,i+1, tagValue) + if (i > 0) and (i%100 == 0): + tsql.execute(sql) + sql = pre_create + if sql != pre_create: + tsql.execute(sql) + + tdLog.debug("complete to create %d child tables in %s.%s" %(ctbNum, dbName, stbName)) + return + + def insert_data(self,tsql,dbName,stbName,ctbNum,rowsPerTbl,batchNum,startTs=None): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs is None: + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s%d values "%(stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_1(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + t = time.time() + startTs = int(round(t * 1000)) + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + for i in range(ctbNum): + sql += " %s%d values "%(ctbPrefix,i) + for j in range(rowsPerTbl): + if (j % 2 == 0): + sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, j, j) + else: + sql += "(%d, %d, %d, 'tmqrow_%d') "%(startTs + j, j, -j, j) + if (j > 0) and ((j%batchNum == 0) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + if j < rowsPerTbl - 1: + sql = "insert into %s%d values " %(ctbPrefix,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_interlaceByMultiTbl(self,tsql,dbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + ctbDict = {} + for i in range(ctbNum): + ctbDict[i] = 0 + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfCtb = 0 + while rowsOfCtb < rowsPerTbl: + for i in range(ctbNum): + sql += " %s.%s_%d values "%(dbName,ctbPrefix,i) + for k in range(batchNum): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + ctbDict[i], ctbDict[i], ctbDict[i]) + ctbDict[i] += 1 + if (0 == ctbDict[i]%batchNum) or (ctbDict[i] == rowsPerTbl): + tsql.execute(sql) + sql = "insert into " + break + rowsOfCtb = ctbDict[0] + + tdLog.debug("insert data ............ [OK]") + return + + def insert_data_with_autoCreateTbl(self,tsql,dbName,stbName,ctbPrefix,ctbNum,rowsPerTbl,batchNum,startTs=0): + tdLog.debug("start to insert data wiht auto create child table ............") + tsql.execute("use %s" %dbName) + pre_insert = "insert into " + sql = pre_insert + + if startTs == 0: + t = time.time() + startTs = int(round(t * 1000)) + + #tdLog.debug("doing insert data into stable:%s rows:%d ..."%(stbName, allRows)) + rowsOfSql = 0 + for i in range(ctbNum): + sql += " %s.%s_%d using %s.%s tags (%d) values "%(dbName,ctbPrefix,i,dbName,stbName,i) + for j in range(rowsPerTbl): + sql += "(%d, %d, 'tmqrow_%d') "%(startTs + j, j, j) + rowsOfSql += 1 + if (j > 0) and ((rowsOfSql == batchNum) or (j == rowsPerTbl - 1)): + tsql.execute(sql) + rowsOfSql = 0 + if j < rowsPerTbl - 1: + sql = "insert into %s.%s_%d using %s.%s tags (%d) values " %(dbName,ctbPrefix,i,dbName,stbName,i) + else: + sql = "insert into " + #end sql + if sql != pre_insert: + #print("insert sql:%s"%sql) + tsql.execute(sql) + tdLog.debug("insert data ............ [OK]") + return + def syncCreateDbStbCtbInsertData(self, tsql, paraDict): - tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"], paraDict['precision']) - tdCom.create_stable(tsql, paraDict["dbName"],paraDict["stbName"], paraDict["columnDict"], paraDict["tagDict"]) - tdCom.create_ctables(tsql, paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["tagDict"]) + tdCom.create_database(tsql, paraDict["dbName"],paraDict["dropFlag"]) + tdCom.create_stable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdCom.create_ctable(tsql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) if "event" in paraDict and type(paraDict['event']) == type(threading.Event()): paraDict["event"].set() - tdCom.insert_data(tsql,paraDict["dbName"],paraDict["stbName"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + ctbPrefix = paraDict['ctbPrefix'] + ctbNum = paraDict["ctbNum"] + for i in range(ctbNum): + tbName = '%s%s'%(ctbPrefix,i) + tdCom.insert_rows(tsql,dbname=paraDict["dbName"],tbname=tbName,start_ts_value=paraDict['startTs'],count=paraDict['rowsPerTbl']) return def threadFunction(self, **paraDict): diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py new file mode 100644 index 0000000000000000000000000000000000000000..f1c451de859d1321e917ed0441ed144d297854c0 --- /dev/null +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -0,0 +1,194 @@ +from distutils.log import error +import taos +import sys +import time +import socket +import os +import threading +import subprocess +import platform + +from util.log import * +from util.sql import * +from util.cases import * +from util.dnodes import * +from util.common import * +sys.path.append("./7-tmq") +from tmqCommon import * + +class TDTestCase: + def init(self, conn, logSql): + tdLog.debug(f"start to excute {__file__}") + tdSql.init(conn.cursor()) + #tdSql.init(conn.cursor(), logSql) # output sql.txt file + + def prepare_udf_so(self): + selfPath = os.path.dirname(os.path.realpath(__file__)) + + if ("community" in selfPath): + projPath = selfPath[:selfPath.find("community")] + else: + projPath = selfPath[:selfPath.find("tests")] + print(projPath) + + if platform.system().lower() == 'windows': + self.libudf1 = subprocess.Popen('(for /r %s %%i in ("udf1.d*") do @echo %%i)|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + if (not tdDnodes.dnodes[0].remoteIP == ""): + tdDnodes.dnodes[0].remote_conn.get(tdDnodes.dnodes[0].config["path"]+'/debug/build/lib/libudf1.so',projPath+"\\debug\\build\\lib\\") + self.libudf1 = self.libudf1.replace('udf1.dll','libudf1.so') + else: + self.libudf1 = subprocess.Popen('find %s -name "libudf1.so"|grep lib|head -n1'%projPath , shell=True, stdout=subprocess.PIPE,stderr=subprocess.STDOUT).stdout.read().decode("utf-8") + self.libudf1 = self.libudf1.replace('\r','').replace('\n','') + return + + def create_udf_function(self): + # create scalar functions + tdSql.execute("create function udf1 as '%s' outputtype int bufSize 8;"%self.libudf1) + + functions = tdSql.getResult("show functions") + function_nums = len(functions) + if function_nums == 1: + tdLog.info("create one udf functions success ") + else: + tdLog.exit("create udf functions fail") + return + + def checkFileContent(self, consumerId, queryString): + buildPath = tdCom.getBuildPath() + cfgPath = tdCom.getClientCfgPath() + dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) + cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) + tdLog.info(cmdStr) + os.system(cmdStr) + + consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) + tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) + + consumeFile = open(consumeRowsFile, mode='r') + queryFile = open(dstFile, mode='r') + + # skip first line for it is schema + queryFile.readline() + + while True: + dst = queryFile.readline() + src = consumeFile.readline() + + if dst: + if dst != src: + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) + else: + break + return + + def tmqCase1(self): + tdLog.printNoPrefix("======== test case 1: ") + paraDict = {'dbName': 'db1', + 'dropFlag': 1, + 'event': '', + 'vgroups': 4, + 'stbName': 'stb', + 'colPrefix': 'c', + 'tagPrefix': 't', + 'colSchema': [{'type': 'INT', 'count':2}, {'type': 'binary', 'len':20, 'count':1}], + 'tagSchema': [{'type': 'INT', 'count':1}, {'type': 'binary', 'len':20, 'count':1}], + 'ctbPrefix': 'ctb', + 'ctbNum': 1, + 'rowsPerTbl': 1000, + 'batchNum': 10, + 'startTs': 1640966400000, # 2022-01-01 00:00:00.000 + 'pollDelay': 10, + 'showMsg': 1, + 'showRow': 1} + + topicNameList = ['topic1', 'topic2'] + expectRowsList = [] + tmqCom.initConsumerTable() + tdCom.create_database(tdSql, paraDict["dbName"],paraDict["dropFlag"], vgroups=4,replica=1) + tdLog.info("create stb") + tdCom.create_stable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"], column_elm_list=paraDict['colSchema'], tag_elm_list=paraDict['tagSchema']) + tdLog.info("create ctb") + tdCom.create_ctable(tdSql, dbname=paraDict["dbName"],stbname=paraDict["stbName"],tag_elm_list=paraDict['tagSchema'],count=paraDict["ctbNum"], default_ctbname_prefix=paraDict['ctbPrefix']) + tdLog.info("insert data") + tmqCom.insert_data_1(tdSql,paraDict["dbName"],paraDict["ctbPrefix"],paraDict["ctbNum"],paraDict["rowsPerTbl"],paraDict["batchNum"],paraDict["startTs"]) + + tdLog.info("create topics from stb with filter") + queryString = "select ts, c1,udf1(c1),c2,udf1(c2) from %s.%s where c1 %% 7 == 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[0], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + # init consume info, and start tmq_sim, then check consume result + tdLog.info("insert consume info to consume processor") + consumerId = 0 + expectrowcnt = paraDict["rowsPerTbl"] * paraDict["ctbNum"] + topicList = topicNameList[0] + ifcheckdata = 1 + ifManualCommit = 1 + keyList = 'group.id:cgrp1, enable.auto.commit:false, auto.commit.interval.ms:6000, auto.offset.reset:earliest' + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + + if expectRowsList[0] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) + tdLog.exit("0 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) + + + # reinit consume info, and start tmq_sim, then check consume result + tmqCom.initConsumerTable() + + queryString = "select ts, c1,udf1(c1),sin(udf1(c2)), log(udf1(c2)) from %s.%s where udf1(c1) == 88 or sin(udf1(c1)) > 0" %(paraDict['dbName'], paraDict['stbName']) + sqlString = "create topic %s as %s" %(topicNameList[1], queryString) + tdLog.info("create topic sql: %s"%sqlString) + tdSql.execute(sqlString) + tdSql.query(queryString) + expectRowsList.append(tdSql.getRows()) + + consumerId = 1 + topicList = topicNameList[1] + tmqCom.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) + + tdLog.info("start consume processor") + tmqCom.startTmqSimProcess(paraDict['pollDelay'],paraDict["dbName"],paraDict['showMsg'], paraDict['showRow']) + + tdLog.info("wait the consume result") + expectRows = 1 + resultList = tmqCom.selectConsumeResult(expectRows) + if expectRowsList[1] != resultList[0]: + tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) + tdLog.exit("1 tmq consume rows error!") + + self.checkFileContent(consumerId, queryString) + tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) + + time.sleep(10) + for i in range(len(topicNameList)): + tdSql.query("drop topic %s"%topicNameList[i]) + + tdLog.printNoPrefix("======== test case 1 end ...... ") + + def run(self): + tdSql.prepare() + self.prepare_udf_so() + self.create_udf_function() + self.tmqCase1() + + def stop(self): + tdSql.close() + tdLog.success(f"{__file__} successfully executed") + +event = threading.Event() + +tdCases.addLinux(__file__, TDTestCase()) +tdCases.addWindows(__file__, TDTestCase()) diff --git a/tests/system-test/fulltest.sh b/tests/system-test/fulltest.sh index db3f7216abb7bad42b2ca0328808b733f93573b1..19a67e924c055afa6c470f69f957804d15b80e66 100755 --- a/tests/system-test/fulltest.sh +++ b/tests/system-test/fulltest.sh @@ -35,6 +35,7 @@ python3 ./test.py -f 2-query/lower.py python3 ./test.py -f 2-query/join.py python3 ./test.py -f 2-query/join2.py python3 ./test.py -f 2-query/cast.py +python3 ./test.py -f 2-query/substr.py python3 ./test.py -f 2-query/union.py python3 ./test.py -f 2-query/union1.py python3 ./test.py -f 2-query/concat.py @@ -105,11 +106,14 @@ python3 ./test.py -f 2-query/distribute_agg_min.py python3 ./test.py -f 2-query/distribute_agg_sum.py python3 ./test.py -f 2-query/distribute_agg_spread.py python3 ./test.py -f 2-query/distribute_agg_apercentile.py +python3 ./test.py -f 2-query/distribute_agg_avg.py +python3 ./test.py -f 2-query/distribute_agg_stddev.py +python3 ./test.py -f 2-query/twa.py python3 ./test.py -f 6-cluster/5dnode1mnode.py python3 ./test.py -f 6-cluster/5dnode2mnode.py -#python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -#python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -N 5 +# python3 ./test.py -f 6-cluster/5dnode3mnodeDrop.py -N 5 # BUG python3 ./test.py -f 6-cluster/5dnode3mnodeStopInsert.py python3 ./test.py -f 7-tmq/basic5.py @@ -127,3 +131,6 @@ python3 ./test.py -f 7-tmq/subscribeStb4.py python3 ./test.py -f 7-tmq/db.py python3 ./test.py -f 7-tmq/tmqError.py python3 ./test.py -f 7-tmq/schema.py +python3 ./test.py -f 7-tmq/stbFilter.py +python3 ./test.py -f 7-tmq/tmqCheckData.py +python3 ./test.py -f 7-tmq/tmqUdf.py diff --git a/tests/system-test/simpletest.bat b/tests/system-test/simpletest.bat index c8084c3ad50be4aad782e5c2f05e3d1bac9214d4..0b1f16d2164e7509efaa0b82dbdeb38c2859af2f 100644 --- a/tests/system-test/simpletest.bat +++ b/tests/system-test/simpletest.bat @@ -5,7 +5,7 @@ python3 .\test.py -f 0-others\taosShellNetChk.py python3 .\test.py -f 0-others\telemetry.py python3 .\test.py -f 0-others\taosdMonitor.py python3 .\test.py -f 0-others\udfTest.py -python3 .\test.py -f 0-others\udf_create.py +@REM python3 .\test.py -f 0-others\udf_create.py @REM python3 .\test.py -f 0-others\udf_restart_taosd.py @REM python3 .\test.py -f 0-others\cachelast.py diff --git a/tests/system-test/test.py b/tests/system-test/test.py index 55bfd8c945082ef53bdb66de8c080c25f6208621..4ca16e7a5766d61bd569d9f9ae31ee98c14f48ac 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -28,6 +28,7 @@ sys.path.append("../pytest") from util.log import * from util.dnodes import * from util.cases import * +from util.cluster import * import taos @@ -58,10 +59,11 @@ if __name__ == "__main__": logSql = True stop = 0 restart = False + dnodeNums = 1 updateCfgDict = {} execCmd = "" - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd']) + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -77,6 +79,8 @@ if __name__ == "__main__": tdLog.printNoPrefix('-d update cfg dict, base64 json str') tdLog.printNoPrefix('-k not kill valgrind processer') tdLog.printNoPrefix('-e eval str to run') + tdLog.printNoPrefix('-N create dnodes numbers clusters') + sys.exit(0) if key in ['-r', '--restart']: @@ -126,6 +130,9 @@ if __name__ == "__main__": print('updateCfgDict convert fail.') sys.exit(0) + if key in ['-N', '--dnodeNums']: + dnodeNums = value + if not execCmd == "": tdDnodes.init(deployPath) print(execCmd) @@ -232,11 +239,36 @@ if __name__ == "__main__": updateCfgDict = ucase.updatecfgDict except: pass - tdDnodes.deploy(1,updateCfgDict) - tdDnodes.start(1) - - tdCases.logSql(logSql) - + if dnodeNums == 1 : + tdDnodes.deploy(1,updateCfgDict) + tdDnodes.start(1) + tdCases.logSql(logSql) + else : + print("start cluster and dnodes number") + + dnodeslist = cluster.configure_cluster(dnodes_nums=dnodeNums,independent=True) + tdDnodes = ClusterDnodes(dnodeslist) + tdDnodes.init(deployPath, masterIp) + tdDnodes.setTestCluster(testCluster) + tdDnodes.setValgrind(valgrind) + tdDnodes.stopAll() + for dnode in tdDnodes.dnodes: + tdDnodes.deploy(dnode.index,{}) + for dnode in tdDnodes.dnodes: + tdDnodes.starttaosd(dnode.index) + tdCases.logSql(logSql) + conn = taos.connect( + host, + config=tdDnodes.getSimCfgPath()) + print(tdDnodes.getSimCfgPath(),host) + cluster.create_dnode(conn) + try: + if cluster.check_dnode(conn) : + print("check dnode ready") + except Exception as r: + print(r) + + if testCluster: tdLog.info("Procedures for testing cluster") if fileName == "all": @@ -248,10 +280,12 @@ if __name__ == "__main__": conn = taos.connect( host, config=tdDnodes.getSimCfgPath()) + if fileName == "all": tdCases.runAllLinux(conn) else: tdCases.runOneLinux(conn, fileName) + if restart: if fileName == "all": tdLog.info("not need to query ") diff --git a/tests/test/c/tmqSim.c b/tests/test/c/tmqSim.c index 948df3a40ac9a41ee0d6eac67958327d4877ff10..053cd79432319dc35dbd50bb088d0d918b1e2099 100644 --- a/tests/test/c/tmqSim.c +++ b/tests/test/c/tmqSim.c @@ -22,8 +22,10 @@ #include #include "taos.h" +#include "taosdef.h" #include "taoserror.h" #include "tlog.h" +#include "types.h" #define GREEN "\033[1;32m" #define NC "\033[0m" @@ -34,11 +36,7 @@ #define MAX_CONSUMER_THREAD_CNT (16) #define MAX_VGROUP_CNT (32) -typedef enum { - NOTIFY_CMD_START_CONSUM, - NOTIFY_CMD_START_COMMIT, - NOTIFY_CMD_ID_BUTT -}NOTIFY_CMD_ID; +typedef enum { NOTIFY_CMD_START_CONSUM, NOTIFY_CMD_START_COMMIT, NOTIFY_CMD_ID_BUTT } NOTIFY_CMD_ID; typedef struct { TdThread thread; @@ -49,8 +47,9 @@ typedef struct { // char autoCommit[8]; // true, false // char autoOffsetRest[16]; // none, earliest, latest - int32_t ifCheckData; - int64_t expectMsgCnt; + TdFilePtr pConsumeRowsFile; + int32_t ifCheckData; + int64_t expectMsgCnt; int64_t consumeMsgCnt; int64_t consumeRowCnt; @@ -86,6 +85,7 @@ typedef struct { int32_t saveRowFlag; int32_t consumeDelay; // unit s int32_t numOfThread; + int32_t useSnapshot; SThreadInfo stThreads[MAX_CONSUMER_THREAD_CNT]; } SConfInfo; @@ -93,6 +93,8 @@ static SConfInfo g_stConfInfo; TdFilePtr g_fp = NULL; static int running = 1; +int8_t useSnapshot = 0; + // char* g_pRowValue = NULL; // TdFilePtr g_fp = NULL; @@ -129,7 +131,6 @@ void initLogFile() { char tmpString[128]; sprintf(filename, "%s/../log/tmqlog_%s.txt", configDir, getCurrentTimeString(tmpString)); - // sprintf(filename, "%s/../log/tmqlog.txt", configDir); #ifdef WINDOWS for (int i = 2; i < sizeof(filename); i++) { if (filename[i] == ':') filename[i] = '-'; @@ -203,6 +204,8 @@ void parseArgument(int32_t argc, char* argv[]) { g_stConfInfo.saveRowFlag = atol(argv[++i]); } else if (strcmp(argv[i], "-y") == 0) { g_stConfInfo.consumeDelay = atol(argv[++i]); + } else if (strcmp(argv[i], "-e") == 0) { + useSnapshot = (int8_t)atol(argv[++i]); } else { pError("%s unknow para: %s %s", GREEN, argv[++i], NC); exit(-1); @@ -296,17 +299,148 @@ int32_t saveConsumeContentToTbl(SThreadInfo* pInfo, char* buf) { return 0; } +static char* shellFormatTimestamp(char* buf, int64_t val, int32_t precision) { + // if (shell.args.is_raw_time) { + // sprintf(buf, "%" PRId64, val); + // return buf; + // } + + time_t tt; + int32_t ms = 0; + if (precision == TSDB_TIME_PRECISION_NANO) { + tt = (time_t)(val / 1000000000); + ms = val % 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + tt = (time_t)(val / 1000000); + ms = val % 1000000; + } else { + tt = (time_t)(val / 1000); + ms = val % 1000; + } + + /* + comment out as it make testcases like select_with_tags.sim fail. + but in windows, this may cause the call to localtime crash if tt < 0, + need to find a better solution. + if (tt < 0) { + tt = 0; + } + */ + +#ifdef WINDOWS + if (tt < 0) tt = 0; +#endif + if (tt <= 0 && ms < 0) { + tt--; + if (precision == TSDB_TIME_PRECISION_NANO) { + ms += 1000000000; + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + ms += 1000000; + } else { + ms += 1000; + } + } + + struct tm* ptm = taosLocalTime(&tt, NULL); + size_t pos = strftime(buf, 35, "%Y-%m-%d %H:%M:%S", ptm); + + if (precision == TSDB_TIME_PRECISION_NANO) { + sprintf(buf + pos, ".%09d", ms); + } else if (precision == TSDB_TIME_PRECISION_MICRO) { + sprintf(buf + pos, ".%06d", ms); + } else { + sprintf(buf + pos, ".%03d", ms); + } + + return buf; +} + +static void shellDumpFieldToFile(TdFilePtr pFile, const char* val, TAOS_FIELD* field, int32_t length, + int32_t precision) { + if (val == NULL) { + taosFprintfFile(pFile, "%s", TSDB_DATA_NULL_STR); + return; + } + + int n; + char buf[TSDB_MAX_BYTES_PER_ROW]; + switch (field->type) { + case TSDB_DATA_TYPE_BOOL: + taosFprintfFile(pFile, "%d", ((((int32_t)(*((char*)val))) == 1) ? 1 : 0)); + break; + case TSDB_DATA_TYPE_TINYINT: + taosFprintfFile(pFile, "%d", *((int8_t*)val)); + break; + case TSDB_DATA_TYPE_UTINYINT: + taosFprintfFile(pFile, "%u", *((uint8_t*)val)); + break; + case TSDB_DATA_TYPE_SMALLINT: + taosFprintfFile(pFile, "%d", *((int16_t*)val)); + break; + case TSDB_DATA_TYPE_USMALLINT: + taosFprintfFile(pFile, "%u", *((uint16_t*)val)); + break; + case TSDB_DATA_TYPE_INT: + taosFprintfFile(pFile, "%d", *((int32_t*)val)); + break; + case TSDB_DATA_TYPE_UINT: + taosFprintfFile(pFile, "%u", *((uint32_t*)val)); + break; + case TSDB_DATA_TYPE_BIGINT: + taosFprintfFile(pFile, "%" PRId64, *((int64_t*)val)); + break; + case TSDB_DATA_TYPE_UBIGINT: + taosFprintfFile(pFile, "%" PRIu64, *((uint64_t*)val)); + break; + case TSDB_DATA_TYPE_FLOAT: + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + break; + case TSDB_DATA_TYPE_DOUBLE: + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } + break; + case TSDB_DATA_TYPE_BINARY: + case TSDB_DATA_TYPE_NCHAR: + case TSDB_DATA_TYPE_JSON: + memcpy(buf, val, length); + buf[length] = 0; + taosFprintfFile(pFile, "\'%s\'", buf); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + shellFormatTimestamp(buf, *(int64_t*)val, precision); + taosFprintfFile(pFile, "'%s'", buf); + break; + default: + break; + } +} + +static void dumpToFileForCheck(TdFilePtr pFile, TAOS_ROW row, TAOS_FIELD* fields, int32_t* length, int32_t num_fields, + int32_t precision) { + for (int32_t i = 0; i < num_fields; i++) { + if (i > 0) { + taosFprintfFile(pFile, "\n"); + } + shellDumpFieldToFile(pFile, (const char*)row[i], fields + i, length[i], precision); + } + taosFprintfFile(pFile, "\n"); +} + static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) { char buf[1024]; int32_t totalRows = 0; // printf("topic: %s\n", tmq_get_topic_name(msg)); - int32_t vgroupId = tmq_get_vgroup_id(msg); + int32_t vgroupId = tmq_get_vgroup_id(msg); + const char* dbName = tmq_get_db_name(msg); - taosFprintfFile(g_fp, "msg index:%" PRId64 ", consumerId: %d\n", msgIndex, pInfo->consumerId); - // taosFprintfFile(g_fp, "topic: %s, vgroupId: %d, tableName: %s\n", tmq_get_topic_name(msg), vgroupId, - // tmq_get_table_name(msg)); - taosFprintfFile(g_fp, "topic: %s, vgroupId: %d\n", tmq_get_topic_name(msg), vgroupId); + taosFprintfFile(g_fp, "consumerId: %d, msg index:%" PRId64 "\n", pInfo->consumerId, msgIndex); + taosFprintfFile(g_fp, "dbName: %s, topic: %s, vgroupId: %d\n", dbName != NULL ? dbName : "invalid table", + tmq_get_topic_name(msg), vgroupId); while (1) { TAOS_ROW row = taos_fetch_row(msg); @@ -315,16 +449,18 @@ static int32_t msg_process(TAOS_RES* msg, SThreadInfo* pInfo, int32_t msgIndex) TAOS_FIELD* fields = taos_fetch_fields(msg); int32_t numOfFields = taos_field_count(msg); + int32_t* length = taos_fetch_lengths(msg); + int32_t precision = taos_result_precision(msg); + const char* tbName = tmq_get_table_name(msg); + dumpToFileForCheck(pInfo->pConsumeRowsFile, row, fields, length, numOfFields, precision); taos_print_row(buf, row, fields, numOfFields); - const char* tbName = tmq_get_table_name(msg); - if (0 != g_stConfInfo.showRowFlag) { taosFprintfFile(g_fp, "tbname:%s, rows[%d]: %s\n", (tbName != NULL ? tbName : "null table"), totalRows, buf); - if (0 != g_stConfInfo.saveRowFlag) { - saveConsumeContentToTbl(pInfo, buf); - } + // if (0 != g_stConfInfo.saveRowFlag) { + // saveConsumeContentToTbl(pInfo, buf); + // } } totalRows++; @@ -347,8 +483,7 @@ int queryDB(TAOS* taos, char* command) { return 0; } -static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) { -} +static void appNothing(void* param, TAOS_RES* res, int32_t numOfRows) {} int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { char sqlStr[1024] = {0}; @@ -356,11 +491,8 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { int64_t now = taosGetTimestampMs(); // schema: ts timestamp, consumerid int, consummsgcnt bigint, checkresult int - sprintf(sqlStr, "insert into %s.notifyinfo values (%"PRId64", %d, %d)", - g_stConfInfo.cdbName, - now, - cmdId, - pInfo->consumerId); + sprintf(sqlStr, "insert into %s.notifyinfo values (%" PRId64 ", %d, %d)", g_stConfInfo.cdbName, now, cmdId, + pInfo->consumerId); taos_query_a(pInfo->taos, sqlStr, appNothing, NULL); @@ -370,12 +502,12 @@ int32_t notifyMainScript(SThreadInfo* pInfo, int32_t cmdId) { } static int32_t g_once_commit_flag = 0; -static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { - pError("tmq_commit_cb_print() commit %d\n", code); +static void tmq_commit_cb_print(tmq_t* tmq, int32_t code, void* param) { + pError("tmq_commit_cb_print() commit %d\n", code); if (0 == g_once_commit_flag) { g_once_commit_flag = 1; - notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); + notifyMainScript((SThreadInfo*)param, (int32_t)NOTIFY_CMD_START_COMMIT); } taosFprintfFile(g_fp, "tmq_commit_cb_print() be called\n"); } @@ -409,6 +541,10 @@ void build_consumer(SThreadInfo* pInfo) { // tmq_conf_set(conf, "auto.offset.reset", "none"); // tmq_conf_set(conf, "auto.offset.reset", "earliest"); // tmq_conf_set(conf, "auto.offset.reset", "latest"); + // + if (useSnapshot) { + tmq_conf_set(conf, "experiment.use.snapshot", "true"); + } pInfo->tmq = tmq_consumer_new(conf, NULL, 0); @@ -464,6 +600,19 @@ void loop_consume(SThreadInfo* pInfo) { pInfo->ts = taosGetTimestampMs(); + if (pInfo->ifCheckData) { + char filename[256] = {0}; + char tmpString[128]; + // sprintf(filename, "%s/../log/consumerid_%d_%s.txt", configDir, pInfo->consumerId, + // getCurrentTimeString(tmpString)); + sprintf(filename, "%s/../log/consumerid_%d.txt", configDir, pInfo->consumerId); + pInfo->pConsumeRowsFile = taosOpenFile(filename, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_TRUNC | TD_FILE_STREAM); + if (pInfo->pConsumeRowsFile == NULL) { + taosFprintfFile(g_fp, "%s create file fail for save rows content\n", getCurrentTimeString(tmpString)); + return; + } + } + while (running) { TAOS_RES* tmqMsg = tmq_consumer_poll(pInfo->tmq, g_stConfInfo.consumeDelay * 1000); if (tmqMsg) { @@ -475,10 +624,10 @@ void loop_consume(SThreadInfo* pInfo) { totalMsgs++; - if (0 == once_flag) { + if (0 == once_flag) { once_flag = 1; - notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); - } + notifyMainScript(pInfo, NOTIFY_CMD_START_CONSUM); + } if (totalRows >= pInfo->expectMsgCnt) { char tmpString[128]; @@ -507,7 +656,7 @@ void* consumeThreadFunc(void* param) { pInfo->taos = taos_connect(NULL, "root", "taosdata", NULL, 0); if (pInfo->taos == NULL) { taosFprintfFile(g_fp, "taos_connect() fail, can not notify and save consume result to main scripte\n"); - exit(-1); + exit(-1); } build_consumer(pInfo); diff --git a/tests/tsim/src/simExe.c b/tests/tsim/src/simExe.c index f97a13d2c5867ef35ceda35e716461400544371c..aaad76bb530cbb4f4ae1fa6f401f08db56adedbc 100644 --- a/tests/tsim/src/simExe.c +++ b/tests/tsim/src/simExe.c @@ -39,13 +39,26 @@ void simLogSql(char *sql, bool useSharp) { char *simParseArbitratorName(char *varName) { static char hostName[140]; +#ifdef WINDOWS + taosGetFqdn(hostName); + sprintf(&hostName[strlen(hostName)], ":%d", 8000); +#else sprintf(hostName, "%s:%d", "localhost", 8000); +#endif return hostName; } char *simParseHostName(char *varName) { static char hostName[140]; +#ifdef WINDOWS + hostName[0] = '\"'; + taosGetFqdn(&hostName[1]); + int strEndIndex = strlen(hostName); + hostName[strEndIndex] = '\"'; + hostName[strEndIndex + 1] = '\0'; +#else sprintf(hostName, "%s", "localhost"); +#endif return hostName; } @@ -399,7 +412,8 @@ bool simExecuteSystemCmd(SScript *script, char *option) { sprintf(buf, "cd %s; ", simScriptDir); simVisuallizeOption(script, option, buf + strlen(buf)); #else - sprintf(buf, "%s%s", simScriptDir, option); + sprintf(buf, "%s", simScriptDir); + simVisuallizeOption(script, option, buf + strlen(buf)); simReplaceStr(buf, ".sh", ".bat"); #endif diff --git a/tools/shell/src/shellCommand.c b/tools/shell/src/shellCommand.c index ef71f3fce6f8b21f09f85ed373233cc5714e6316..f236c1eb88699f81bec125a57ed640d8485a1ba7 100644 --- a/tools/shell/src/shellCommand.c +++ b/tools/shell/src/shellCommand.c @@ -21,6 +21,7 @@ #define UP 3 #define DOWN 4 #define PSIZE shell.info.promptSize +#define SHELL_INPUT_MAX_COMMAND_SIZE 10000 typedef struct { char *buffer; @@ -227,6 +228,7 @@ void shellPrintChar(char c, int32_t times) { } void shellPositionCursor(int32_t step, int32_t direction) { +#ifndef WINDOWS if (step > 0) { if (direction == LEFT) { fprintf(stdout, "\033[%dD", step); @@ -239,6 +241,7 @@ void shellPositionCursor(int32_t step, int32_t direction) { } fflush(stdout); } +#endif } void shellUpdateBuffer(SShellCmd *cmd) { @@ -330,10 +333,14 @@ void shellClearScreen(int32_t ecmd_pos, int32_t cursor_pos) { int32_t command_x = ecmd_pos / ws_col; shellPositionCursor(cursor_y, LEFT); shellPositionCursor(command_x - cursor_x, DOWN); +#ifndef WINDOWS fprintf(stdout, "\033[2K"); +#endif for (int32_t i = 0; i < command_x; i++) { shellPositionCursor(1, UP); + #ifndef WINDOWS fprintf(stdout, "\033[2K"); + #endif } fflush(stdout); } @@ -394,6 +401,41 @@ void shellShowOnScreen(SShellCmd *cmd) { fflush(stdout); } +char taosGetConsoleChar() { +#ifdef WINDOWS + static void *console = NULL; + if (console == NULL) { + console = GetStdHandle(STD_INPUT_HANDLE); + } + static TdWchar buf[SHELL_INPUT_MAX_COMMAND_SIZE]; + static char mbStr[5]; + static unsigned long bufLen = 0; + static uint16_t bufIndex = 0, mbStrIndex = 0, mbStrLen = 0; + if (bufLen == 0) { + ReadConsoleW(console, buf, SHELL_INPUT_MAX_COMMAND_SIZE, &bufLen, NULL); + bufIndex = 0; + } + if (mbStrLen == 0){ + if (buf[bufIndex] == '\r') { + bufIndex++; + } + mbStrLen = WideCharToMultiByte(CP_UTF8, 0, &buf[bufIndex], 1, mbStr, sizeof(mbStr), NULL, NULL); + mbStrIndex = 0; + bufIndex++; + } + mbStrIndex++; + if (mbStrIndex == mbStrLen) { + mbStrLen = 0; + if (bufIndex == bufLen) { + bufLen = 0; + } + } + return mbStr[mbStrIndex-1]; +#else + return (char)getchar(); // getchar() return an 'int32_t' value +#endif +} + int32_t shellReadCommand(char *command) { SShellHistory *pHistory = &shell.history; SShellCmd cmd = {0}; @@ -407,7 +449,7 @@ int32_t shellReadCommand(char *command) { // Read input. char c; while (1) { - c = (char)getchar(); // getchar() return an 'int32_t' value + c = taosGetConsoleChar(); if (c == EOF) { return c; @@ -417,7 +459,7 @@ int32_t shellReadCommand(char *command) { int32_t count = shellCountPrefixOnes(c); utf8_array[0] = c; for (int32_t k = 1; k < count; k++) { - c = (char)getchar(); + c = taosGetConsoleChar(); utf8_array[k] = c; } shellInsertChar(&cmd, utf8_array, count); @@ -472,10 +514,10 @@ int32_t shellReadCommand(char *command) { break; } } else if (c == '\033') { - c = (char)getchar(); + c = taosGetConsoleChar(); switch (c) { case '[': - c = (char)getchar(); + c = taosGetConsoleChar(); switch (c) { case 'A': // Up arrow if (hist_counter != pHistory->hstart) { @@ -502,35 +544,35 @@ int32_t shellReadCommand(char *command) { shellMoveCursorLeft(&cmd); break; case '1': - if ((c = (char)getchar()) == '~') { + if ((c = taosGetConsoleChar()) == '~') { // Home key shellPositionCursorHome(&cmd); } break; case '2': - if ((c = (char)getchar()) == '~') { + if ((c = taosGetConsoleChar()) == '~') { // Insert key } break; case '3': - if ((c = (char)getchar()) == '~') { + if ((c = taosGetConsoleChar()) == '~') { // Delete key shellDeleteChar(&cmd); } break; case '4': - if ((c = (char)getchar()) == '~') { + if ((c = taosGetConsoleChar()) == '~') { // End key shellPositionCursorEnd(&cmd); } break; case '5': - if ((c = (char)getchar()) == '~') { + if ((c = taosGetConsoleChar()) == '~') { // Page up key } break; case '6': - if ((c = (char)getchar()) == '~') { + if ((c = taosGetConsoleChar()) == '~') { // Page down key } break; diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 1f29237d38b0441507bef05a39586d8a3b80ca56..8a017d378de05feae3ef49624a617a453572d98d 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -393,15 +393,11 @@ void shellPrintNChar(const char *str, int32_t length, int32_t width) { break; } int w = 0; -#ifdef WINDOWS - w = bytes; -#else if(*(str + pos) == '\t' || *(str + pos) == '\n' || *(str + pos) == '\r'){ w = bytes; }else{ w = taosWcharWidth(wc); } -#endif pos += bytes; if (w <= 0) { @@ -524,6 +520,16 @@ bool shellIsLimitQuery(const char *sql) { return false; } +bool shellIsShowQuery(const char *sql) { + //todo refactor + if (taosStrCaseStr(sql, "show ") != NULL) { + return true; + } + + return false; +} + + int32_t shellVerticalPrintResult(TAOS_RES *tres, const char *sql) { TAOS_ROW row = taos_fetch_row(tres); if (row == NULL) { @@ -686,7 +692,7 @@ int32_t shellHorizontalPrintResult(TAOS_RES *tres, const char *sql) { uint64_t resShowMaxNum = UINT64_MAX; - if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsLimitQuery(sql)) { + if (shell.args.commands == NULL && shell.args.file[0] == 0 && !shellIsLimitQuery(sql) && !shellIsShowQuery(sql)) { resShowMaxNum = SHELL_DEFAULT_RES_SHOW_NUM; } diff --git a/tools/taosadapter b/tools/taosadapter index 29926478edd87533a043f91c1a9ed0e27671e626..c1dc11f3cae64adf31dbd9a954ef8372d1e8f671 160000 --- a/tools/taosadapter +++ b/tools/taosadapter @@ -1 +1 @@ -Subproject commit 29926478edd87533a043f91c1a9ed0e27671e626 +Subproject commit c1dc11f3cae64adf31dbd9a954ef8372d1e8f671