diff --git a/cmake/cmake.version b/cmake/cmake.version index 5150ee3b75e2ac429961850995d06b5b2fd40cc9..232e86d891afebc524c1a6b2df78fac6a9137211 100644 --- a/cmake/cmake.version +++ b/cmake/cmake.version @@ -2,7 +2,7 @@ IF (DEFINED VERNUMBER) SET(TD_VER_NUMBER ${VERNUMBER}) ELSE () - SET(TD_VER_NUMBER "3.0.3.2") + SET(TD_VER_NUMBER "3.0.4.0") ENDIF () IF (DEFINED VERCOMPATIBLE) diff --git a/cmake/taosadapter_CMakeLists.txt.in b/cmake/taosadapter_CMakeLists.txt.in index b2f335e1f7b8a30a46713a322c2ef95b5605c9d9..4a8f4864b301101fd04214674069e8c3196ba49d 100644 --- a/cmake/taosadapter_CMakeLists.txt.in +++ b/cmake/taosadapter_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taosadapter ExternalProject_Add(taosadapter GIT_REPOSITORY https://github.com/taosdata/taosadapter.git - GIT_TAG cb1e89c + GIT_TAG ae8d51c SOURCE_DIR "${TD_SOURCE_DIR}/tools/taosadapter" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/cmake/taostools_CMakeLists.txt.in b/cmake/taostools_CMakeLists.txt.in index 0110b27b325e9479455bbbf1e8c77e73f69d950d..4838e97dd79c55d58c0442a065a3c1e3171895b2 100644 --- a/cmake/taostools_CMakeLists.txt.in +++ b/cmake/taostools_CMakeLists.txt.in @@ -2,7 +2,7 @@ # taos-tools ExternalProject_Add(taos-tools GIT_REPOSITORY https://github.com/taosdata/taos-tools.git - GIT_TAG 149ac34 + GIT_TAG ffc2e6f SOURCE_DIR "${TD_SOURCE_DIR}/tools/taos-tools" BINARY_DIR "" #BUILD_IN_SOURCE TRUE diff --git a/compile_flags.txt b/compile_flags.txt new file mode 100644 index 0000000000000000000000000000000000000000..c61f9701abb7750b5e36c10eac480f61886fd5ca --- /dev/null +++ b/compile_flags.txt @@ -0,0 +1,9 @@ +-DLINUX +-DWEBSOCKET +-I/usr/include +-Iinclude +-Iinclude/os +-Iinclude/common +-Iinclude/util +-Iinclude/libs/transport +-Itools/shell/inc diff --git a/docs/en/07-develop/07-tmq.mdx b/docs/en/07-develop/07-tmq.mdx index 7465cc0a12ecc9c92a48a2308fe101d057f80492..739eabd1cf7e0168251e134daf101a9288333efe 100644 --- a/docs/en/07-develop/07-tmq.mdx +++ b/docs/en/07-develop/07-tmq.mdx @@ -222,7 +222,7 @@ A database including one supertable and two subtables is created as follows: ```sql DROP DATABASE IF EXISTS tmqdb; -CREATE DATABASE tmqdb; +CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600; CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16)); CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0"); CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); diff --git a/docs/en/12-taos-sql/07-tag-index.md b/docs/en/12-taos-sql/07-tag-index.md index cb2a61d3e813ad130476f8f3309d30b7e208a009..af1d0a352ec8950e49a1269c5b2192228fd35d32 100644 --- a/docs/en/12-taos-sql/07-tag-index.md +++ b/docs/en/12-taos-sql/07-tag-index.md @@ -6,7 +6,7 @@ description: Use Tag Index to Improve Query Performance ## Introduction -Prior to TDengine 3.0.3.0 (excluded)，only one index is created by default on the first tag of each super talbe, but it's not allowed to dynamically create index on any other tags. From version 3.0.30, you can dynamically create index on any tag of any type. The index created automatically by TDengine is still valid. Query performance can benefit from indexes if you use properly. +Prior to TDengine 3.0.3.0 (excluded)，only one index is created by default on the first tag of each super table, but it's not allowed to dynamically create index on any other tags. From version 3.0.30, you can dynamically create index on any tag of any type. The index created automatically by TDengine is still valid. Query performance can benefit from indexes if you use properly. ## Syntax @@ -48,4 +48,4 @@ You can also add filter conditions to limit the results. 6. You can' create index on a normal table or a child table. -7. If the unique values of a tag column are too few, it's better not to create index on such tag columns, the benefit would be very small. \ No newline at end of file +7. If the unique values of a tag column are too few, it's better not to create index on such tag columns, the benefit would be very small. diff --git a/docs/en/12-taos-sql/10-function.md b/docs/en/12-taos-sql/10-function.md index 8dfa9c2851c16ac16d43559555e492991b44dd15..0b5f9103c79a6633a4aaa5cb58a1c6ee77ec4550 100644 --- a/docs/en/12-taos-sql/10-function.md +++ b/docs/en/12-taos-sql/10-function.md @@ -5,9 +5,9 @@ description: This document describes the standard SQL functions available in TDe toc_max_heading_level: 4 --- -## Single Row Functions +## Scalar Functions -Single row functions return a result for each row. +Scalar functions return one result for each row. ### Mathematical Functions @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **Description**: UNIX timestamp converted from a string of date/time format -**Return value type**: BIGINT +**Return value type**: BIGINT, TIMESTAMP **Applicable column types**: VARCHAR and NCHAR @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - The input string must be compatible with ISO8601/RFC3339 standard, NULL will be returned if the string can't be converted - The precision of the returned timestamp is same as the precision set for the current data base in use +- return_timestamp indicates whether the returned value type is TIMESTAMP or not. If this parameter set to 1, function will return TIMESTAMP type. Otherwise function will return BIGINT type. If parameter is omitted, default return value type is BIGINT. ### Time and Date Functions diff --git a/docs/en/12-taos-sql/14-stream.md b/docs/en/12-taos-sql/14-stream.md index b8f6c3a163e1c2fa37e437725652eabb1a71dbe6..d99c5bdae4344670d5143b7962b76464e5c2d697 100644 --- a/docs/en/12-taos-sql/14-stream.md +++ b/docs/en/12-taos-sql/14-stream.md @@ -13,8 +13,11 @@ Because stream processing is built in to TDengine, you are no longer reliant on ```sql CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name SUBTABLE(expression) AS subquery stream_options: { - TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] - WATERMARK time + TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + WATERMARK time + IGNORE EXPIRED [0|1] + DELETE_MARK time + FILL_HISTORY [0|1] } ``` @@ -141,3 +144,27 @@ The data in expired windows is tagged as expired. TDengine stream processing pro 2. Recalculate the data. In this method, all data in the window is reobtained from the database and recalculated. The latest results are then returned. In both of these methods, configuring the watermark is essential for obtaining accurate results (if expired data is dropped) and avoiding repeated triggers that affect system performance (if expired data is recalculated). + +## Supported functions + +All [scalar functions](../function/#scalar-functions) are available in stream processing. All [System information functions](../function/#system-information-functions) are not allowed in stream processing. All [Aggregate functions](../function/#aggregate-functions) and [Selection functions](../function/#selection-functions) are available in stream processing, except the followings: + - [leastsquares](../function/#leastsquares) + - [percentile](../function/#percentile) + - [top](../function/#top) + - [bottom](../function/#bottom) + - [elapsed](../function/#elapsed) + - [interp](../function/#interp) + - [derivative](../function/#derivative) + - [irate](../function/#irate) + - [twa](../function/#twa) + - [histogram](../function/#histogram) + - [diff](../function/#diff) + - [statecount](../function/#statecount) + - [stateduration](../function/#stateduration) + - [csum](../function/#csum) + - [mavg](../function/#mavg) + - [sample](../function/#sample) + - [tail](../function/#tail) + - [unique](../function/#unique) + - [mode](../function/#mode) + diff --git a/docs/en/12-taos-sql/23-perf.md b/docs/en/12-taos-sql/23-perf.md index fc369ec663cf9130bd7d1af650f1f47a1054fad6..43ff8e3091309e4f55ca1e7b59d1edca09424e3d 100644 --- a/docs/en/12-taos-sql/23-perf.md +++ b/docs/en/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ Provides information about SQL queries currently running. Similar to SHOW QUERIE | 1 | consumer_id | BIGINT | Consumer ID | | 2 | consumer_group | BINARY(192) | Consumer group | | 3 | client_id | BINARY(192) | Client ID (user-defined) | -| 4 | status | BINARY(20) | Consumer status | +| 4 | status | BINARY(20) | Consumer status. All possible status include: ready(consumer is in normal state), lost(the connection between consumer and mnode is broken), rebalance(the redistribution of vgroups that belongs to current consumer is now in progress), unknown(consumer is in invalid state) | 5 | topics | BINARY(204) | Subscribed topic. Returns one row for each topic. | | 6 | up_time | TIMESTAMP | Time of first connection to TDengine Server | | 7 | subscribe_time | TIMESTAMP | Time of first subscription | diff --git a/docs/en/12-taos-sql/29-changes.md b/docs/en/12-taos-sql/29-changes.md index a695a2cae18f28e090816ec98a978674a028df30..086aee59fe0583e905c04320c9fdd3a4a93351c7 100644 --- a/docs/en/12-taos-sql/29-changes.md +++ b/docs/en/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ The following data types can be used in the schema for standard tables. | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | Deprecated| This Enterprise Edition-only statement has been removed. It returns the error "This statement is no longer supported." | 2 | ALTER ALL DNODES | Added | Modifies the configuration of all dnodes. -| 3 | ALTER DATABASE | Modified | Deprecated +| 3 | ALTER DATABASE | Modified | Deprecated | 4 | ALTER STABLE | Modified | Deprecated | 5 | ALTER TABLE | Modified | Deprecated | 6 | ALTER USER | Modified | Deprecated diff --git a/docs/en/13-operation/10-monitor.md b/docs/en/13-operation/10-monitor.md index 346b874059a11c5608027b4ea6e2550765d64186..19107240bff5942ebb28e9a8353bc227ccb2b00e 100644 --- a/docs/en/13-operation/10-monitor.md +++ b/docs/en/13-operation/10-monitor.md @@ -42,3 +42,304 @@ An existing Grafana Notification Channel can be specified with parameter `-E`, t Launch `TDinsight.sh` with the command above and restart Grafana, then open Dashboard `http://localhost:3000/d/tdinsight`. For more use cases and restrictions please refer to [TDinsight](/reference/tdinsight/). + +## log database + +The data of tdinsight dashboard is stored in `log` database (default. You can change it in taoskeeper's config file. For more infrmation, please reference to [taoskeeper document](/reference/taosKeeper)). The taoskeeper will create log database on taoskeeper startup. + +### cluster\_info table + +`cluster_info` table contains cluster information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|first\_ep|VARCHAR||first ep of cluster| +|first\_ep\_dnode\_id|INT||dnode id or first\_ep| +|version|VARCHAR||tdengine version. such as: 3.0.4.0| +|master\_uptime|FLOAT||days of master's uptime| +|monitor\_interval|INT||monitor interval in second| +|dbs\_total|INT||total number of databases in cluster| +|tbs\_total|BIGINT||total number of tables in cluster| +|stbs\_total|INT||total number of stables in cluster| +|dnodes\_total|INT||total number of dnodes in cluster| +|dnodes\_alive|INT||total number of dnodes in ready state| +|mnodes\_total|INT||total number of mnodes in cluster| +|mnodes\_alive|INT||total number of mnodes in ready state| +|vgroups\_total|INT||total number of vgroups in cluster| +|vgroups\_alive|INT||total number of vgroups in ready state| +|vnodes\_total|INT||total number of vnode in cluster| +|vnodes\_alive|INT||total number of vnode in ready state| +|connections\_total|INT||total number of connections to cluster| +|topics\_total|INT||total number of topics in cluster| +|streams\_total|INT||total number of streams in cluster| +|protocol|INT||protocol version| +|cluster\_id|NCHAR|TAG|cluster id| + +### d\_info table + +`d_info` table contains dnodes information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|status|VARCHAR||dnode status| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### m\_info table + +`m_info` table contains mnode information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|role|VARCHAR||the role of mnode. leader or follower| +|mnode\_id|INT|TAG|master node id| +|mnode\_ep|NCHAR|TAG|master node endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### dnodes\_info table + +`dnodes_info` table contains dnodes information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|uptime|FLOAT||dnode uptime| +|cpu\_engine|FLOAT||cpu usage of tdengine. read from `/proc//stat`| +|cpu\_system|FLOAT||cpu usage of server. read from `/proc/stat`| +|cpu\_cores|FLOAT||cpu cores of server| +|mem\_engine|INT||memory usage of tdengine. read from `/proc//status`| +|mem\_system|INT||available memory on the server| +|mem\_total|INT||total memory of server in `KB`| +|disk\_engine|INT||| +|disk\_used|BIGINT||usage of data dir in `bytes`| +|disk\_total|BIGINT||the capacity of data dir in `bytes`| +|net\_in|FLOAT||network throughput rate in kb/s. read from `/proc/net/dev`| +|net\_out|FLOAT||network throughput rate in kb/s. read from `/proc/net/dev`| +|io\_read|FLOAT||io throughput rate in kb/s. read from `/proc//io`| +|io\_write|FLOAT||io throughput rate in kb/s. read from `/proc//io`| +|io\_read\_disk|FLOAT||io throughput rate of disk in kb/s. read from `/proc//io`| +|io\_write\_disk|FLOAT||io throughput rate of disk in kb/s. read from `/proc//io`| +|req\_select|INT||number of select queries received per dnode| +|req\_select\_rate|FLOAT||number of select queries received per dnode divided by monitor interval.| +|req\_insert|INT||number of insert queries received per dnode| +|req\_insert\_success|INT||number of successfully insert queries received per dnode| +|req\_insert\_rate|FLOAT||number of insert queries received per dnode divided by monitor interval| +|req\_insert\_batch|INT||number of batch insertions| +|req\_insert\_batch\_success|INT||number of successful batch insertions| +|req\_insert\_batch\_rate|FLOAT||number of batch insertions divided by monitor interval| +|errors|INT||dnode errors| +|vnodes\_num|INT||number of vnodes per dnode| +|masters|INT||number of master vnodes| +|has\_mnode|INT||if the dnode has mnode| +|has\_qnode|INT||if the dnode has qnode| +|has\_snode|INT||if the dnode has snode| +|has\_bnode|INT||if the dnode has bnode| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### data\_dir table + +`data_dir` table contains data directory information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||data directory. default is `/var/lib/taos`| +|level|INT||level for multi-level storage| +|avail|BIGINT||available space for data directory| +|used|BIGINT||used space for data directory| +|total|BIGINT||total space for data directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_dir table + +`log_dir` table contains log directory information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||log directory. default is `/var/log/taos/`| +|avail|BIGINT||available space for log directory| +|used|BIGINT||used space for data directory| +|total|BIGINT||total space for data directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### temp\_dir table + +`temp_dir` table contains temp dir information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||temp directory. default is `/tmp/`| +|avail|BIGINT||available space for temp directory| +|used|BIGINT||used space for temp directory| +|total|BIGINT||total space for temp directory| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vgroups\_info table + +`vgroups_info` table contains vgroups information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vgroup\_id|INT||vgroup id| +|database\_name|VARCHAR||database for the vgroup| +|tables\_num|BIGINT||number of tables per vgroup| +|status|VARCHAR||status| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vnodes\_role table + +`vnodes_role` table contains vnode role information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vnode\_role|VARCHAR||role. leader or follower| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### logs table + +`logs` table contains login information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|level|VARCHAR||log level| +|content|NCHAR||log content| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_summary table + +`log_summary` table contains log summary information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|error|INT||error count| +|info|INT||info count| +|debug|INT||debug count| +|trace|INT||trace count| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### grants\_info table + +`grants_info` table contains grants information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|expire\_time|BIGINT||time until grants expire in seconds| +|timeseries\_used|BIGINT||timeseries used| +|timeseries\_total|BIGINT||total timeseries| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### keeper\_monitor table + +`keeper_monitor` table contains keeper monitor information records. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|cpu|FLOAT||cpu usage| +|mem|FLOAT||memory usage| +|identify|NCHAR|TAG|| + +### taosadapter\_restful\_http\_request\_total table + +`taosadapter_restful_http_request_total` table contains taosadapter rest request information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_fail table + +`taosadapter_restful_http_request_fail` table contains taosadapter failed rest request information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_in\_flight table + +`taosadapter_restful_http_request_in_flight` table contains taosadapter rest request information record in real time. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_restful\_http\_request\_summary\_milliseconds table + +`taosadapter_restful_http_request_summary_milliseconds` table contains the summary or rest information record. The timestamp column of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|count|DOUBLE||| +|sum|DOUBLE||| +|0.5|DOUBLE||| +|0.9|DOUBLE||| +|0.99|DOUBLE||| +|0.1|DOUBLE||| +|0.2|DOUBLE||| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| + +### taosadapter\_system\_mem\_percent table + +`taosadapter_system_mem_percent` table contains taosadapter memory usage information. The timestamp of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||metric value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_system\_cpu\_percent table + +`taosadapter_system_cpu_percent` table contains taosadapter cup usage information. The timestamp of this table is `_ts`. + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||mertic value| +|endpoint|NCHAR|TAG|taosadpater endpoint| + diff --git a/docs/en/14-reference/02-rest-api/02-rest-api.mdx b/docs/en/14-reference/02-rest-api/02-rest-api.mdx index 1691b8be8b53480199898ab4ec6b5917eba7979f..409e079b92d5c8375ce6b3da60c0b2aa053ef662 100644 --- a/docs/en/14-reference/02-rest-api/02-rest-api.mdx +++ b/docs/en/14-reference/02-rest-api/02-rest-api.mdx @@ -382,6 +382,130 @@ Response body: } ``` +## REST API between TDengine 2.x and 3.0 + +### URI + +| URI | TDengine 2.x | TDengine 3.0 | +| :--------------------| :------------------: | :--------------------------------------------------: | +| /rest/sql | Supported | Supported (with different response code and body) | +| /rest/sqlt | Supported | No more supported | +| /rest/sqlutc | Supported | No more supported | + +### HTTP code + +| HTTP code | TDengine 2.x | TDengine 3.0 | note | +| :--------------------| :------------------: | :----------: | :-----------------------------------: | +| 200 | Supported | Supported | Success or taosc return error | +| 400 | Not supported | Supported | Parameter error | +| 401 | Not supported | Supported | Authentication failure | +| 404 | Supported | Supported | URI not exist | +| 500 | Not supported | Supported | Internal error | +| 503 | Supported | Supported | Insufficient system resources | + +### Response body + +#### REST response body return from TDengine 2.x + +```JSON +{ + "status": "succ", + "head": [ + "name", + "created_time", + "ntables", + "vgroups", + "replica", + "quorum", + "days", + "keep1,keep2,keep(D)", + "cache(MB)", + "blocks", + "minrows", + "maxrows", + "wallevel", + "fsync", + "comp", + "precision", + "status" + ], + "data": [ + [ + "log", + "2020-09-02 17:23:00.039", + 4, + 1, + 1, + 1, + 10, + "30,30,30", + 1, + 3, + 100, + 4096, + 1, + 3000, + 2, + "us", + "ready" + ] + ], + "rows": 1 +} +``` +``` + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], +``` + +#### REST response body return from TDengine 3.0 + +```JSON +{ + "code": 0, + "column_meta": [ + [ + "name", + "VARCHAR", + 64 + ], + [ + "ntables", + "BIGINT", + 8 + ], + [ + "status", + "VARCHAR", + 10 + ] + ], + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], + "rows": 2 +} +``` + ## Reference [taosAdapter](/reference/taosadapter/) diff --git a/docs/en/14-reference/03-connector/03-cpp.mdx b/docs/en/14-reference/03-connector/03-cpp.mdx index b543879b3cf8814d21d0a6f940762c67cc2519b0..014109b77e65f435d6d4cfb985daf74d664e12ae 100644 --- a/docs/en/14-reference/03-connector/03-cpp.mdx +++ b/docs/en/14-reference/03-connector/03-cpp.mdx @@ -423,6 +423,6 @@ In addition to writing data using the SQL method or the parameter binding API, w **Description** - The above seven interfaces are extension interfaces, which are mainly used to pass ttl and reqid parameters, and can be used as needed. - - Withing _raw interfaces represent data through the passed parameters lines and len. In order to solve the problem that the original interface data contains '\0' and is truncated. The totalRows pointer returns the number of parsed data rows. - - Withing _ttl interfaces can pass the ttl parameter to control the ttl expiration time of the table. - - Withing _reqid interfaces can track the entire call chain by passing the reqid parameter. + - Within _raw interfaces represent data through the passed parameters lines and len. In order to solve the problem that the original interface data contains '\0' and is truncated. The totalRows pointer returns the number of parsed data rows. + - Within _ttl interfaces can pass the ttl parameter to control the ttl expiration time of the table. + - Within _reqid interfaces can track the entire call chain by passing the reqid parameter. diff --git a/docs/en/14-reference/03-connector/04-java.mdx b/docs/en/14-reference/03-connector/04-java.mdx index fd4b4641d75e2116267b3f7f13a995079913f663..85d79ef0e0c150958590ce75ce3de9457f572c45 100644 --- a/docs/en/14-reference/03-connector/04-java.mdx +++ b/docs/en/14-reference/03-connector/04-java.mdx @@ -82,7 +82,7 @@ Add following dependency in the `pom.xml` file of your Maven project: com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.1.0 ``` @@ -227,7 +227,7 @@ In addition to getting the connection from the specified URL, you can use Proper Note: - The client parameter set in the application is process-level. If you want to update the parameters of the client, you need to restart the application. This is because the client parameter is a global parameter that takes effect only the first time the application is set. -- The following sample code is based on taos-jdbcdriver-3.0.0. +- The following sample code is based on taos-jdbcdriver-3.1.0. ```java public Connection getConn() throws Exception{ @@ -364,7 +364,7 @@ TDengine has significantly improved the bind APIs to support data writing (INSER **Note:** - JDBC REST connections do not currently support bind interface -- The following sample code is based on taos-jdbcdriver-3.0.0 +- The following sample code is based on taos-jdbcdriver-3.1.0 - The setString method should be called for binary type data, and the setNString method should be called for nchar type data - both setString and setNString require the user to declare the width of the corresponding column in the size parameter of the table definition @@ -632,7 +632,7 @@ TDengine supports schemaless writing. It is compatible with InfluxDB's Line Prot Note: - JDBC REST connections do not currently support schemaless writes -- The following sample code is based on taos-jdbcdriver-3.0.0 +- The following sample code is based on taos-jdbcdriver-3.1.0 ```java public class SchemalessInsertTest { diff --git a/docs/en/14-reference/03-connector/07-python.mdx b/docs/en/14-reference/03-connector/07-python.mdx index bfbdd929c263a59df42e1f310b267ba1521032c2..cc5c8f4e69e208c98c773e94c6aa2ca1ace334e7 100644 --- a/docs/en/14-reference/03-connector/07-python.mdx +++ b/docs/en/14-reference/03-connector/07-python.mdx @@ -459,6 +459,56 @@ For a more detailed description of the `sql()` method, please refer to [RestClie +### Schemaless Insert + +Connector support schemaless insert. + + + + +Simple insert + +```python +{{#include docs/examples/python/schemaless_insert.py}} +``` + +Insert with ttl argument + +```python +{{#include docs/examples/python/schemaless_insert_ttl.py}} +``` + +Insert with req_id argument + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + + + + + +Simple insert + +```python +{{#include docs/examples/python/schemaless_insert_raw.py}} +``` + +Insert with ttl argument + +```python +{{#include docs/examples/python/schemaless_insert_raw_ttl.py}} +``` + +Insert with req_id argument + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + + + + ### Other sample programs | Example program links | Example program content | diff --git a/docs/en/14-reference/03-connector/_category_.yml b/docs/en/14-reference/03-connector/_category_.yml index e470f64aa013b137f05f03db112641faf2956297..6a766e96574844db87a70ff7eb5f0005cb6acbfb 100644 --- a/docs/en/14-reference/03-connector/_category_.yml +++ b/docs/en/14-reference/03-connector/_category_.yml @@ -1 +1 @@ -label: "connector" \ No newline at end of file +label: "Connector" diff --git a/docs/en/14-reference/03-connector/index.mdx b/docs/en/14-reference/03-connector/index.mdx index a35d5bc2d1ed4a69f9750a1153d15efe815f674d..28b7b83b58f1c755d147981dc80a443f21e9860d 100644 --- a/docs/en/14-reference/03-connector/index.mdx +++ b/docs/en/14-reference/03-connector/index.mdx @@ -62,7 +62,7 @@ The different database framework specifications for various programming language | **Regular Query** | Support | Support | Support | Support | Support | Support | | **Parameter Binding** | Not Supported | Not Supported | Support | Support | Not Supported | Support | | **Subscription (TMQ) ** | Supported | Support | Support | Not Supported | Not Supported | Support | -| **Schemaless** | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | +| **Schemaless** | Supported | Not Supported | Not Supported | Not Supported | Not Supported | Not Supported | | **Bulk Pulling (based on WebSocket) ** | Support | Support | Support | Support | Support | Support | | **DataFrame** | Not Supported | Support | Not Supported | Not Supported | Not Supported | Not Supported | diff --git a/docs/en/14-reference/04-taosadapter.md b/docs/en/14-reference/04-taosadapter.md index 7ab894a1c72651e3568f3effa351b044c397b2dd..6bc49768c692a47ea2796fb006d9ee942ef818ba 100644 --- a/docs/en/14-reference/04-taosadapter.md +++ b/docs/en/14-reference/04-taosadapter.md @@ -54,94 +54,91 @@ Command-line arguments take precedence over environment variables over configura ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when taosd returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" + --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) + --version Print the version and exit ``` Note: @@ -332,6 +329,10 @@ This parameter controls the number of results returned by the following interfac taosAdapter uses the parameter `httpCodeServerError` to set whether to return a non-200 http status code http status code other than when the C interface returns an error. When set to true, different http status codes will be returned according to the error code returned by C. For details, see [RESTful API](https://docs.tdengine.com/reference/rest-api/) HTTP Response Code chapter. +## Configure whether schemaless writes automatically create DBs + +Starting from version 3.0.4.0, the taosAdapter provides the parameter "smlAutoCreateDB" to control whether to automatically create DBs when writing with the schemaless protocol. The default value is false, which means that the DB will not be automatically created and the user needs to manually create the DB before performing schemaless writing. + ## Troubleshooting You can check the taosAdapter running status with the `systemctl status taosadapter` command. diff --git a/docs/en/14-reference/12-config/index.md b/docs/en/14-reference/12-config/index.md index 430487a3af9f8729e199251b9cb6c913ee059769..2e6f5ec1e27b7cbba035ad745e4d1f6ec3633f22 100644 --- a/docs/en/14-reference/12-config/index.md +++ b/docs/en/14-reference/12-config/index.md @@ -628,6 +628,16 @@ The charset that takes effect is UTF-8. | Default Value | 1 | | Note | The core file is generated under root directory `systemctl start taosd`/`launchctl start com.tdengine.taosd` is used to start, or under the working directory if `taosd` is started directly on Linux/macOS Shell. | +### enableScience + +| Attribute | Description | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| Applicable | Only taos-CLI client | +| Meaning | Whether to show float and double with the scientific notation | +| Value Range | 0: false, 1: true | +| Default Value | 0 | + + ### udf | Attribute | Description | diff --git a/docs/en/14-reference/13-schemaless/13-schemaless.md b/docs/en/14-reference/13-schemaless/13-schemaless.md index 3f75364081d7ec242d96a30f3adf0861637a06eb..aad0e63a4228ca303302d4a3970182355f750d53 100644 --- a/docs/en/14-reference/13-schemaless/13-schemaless.md +++ b/docs/en/14-reference/13-schemaless/13-schemaless.md @@ -3,13 +3,11 @@ title: Schemaless Writing description: This document describes how to use the schemaless write component of TDengine. --- -In IoT applications, data is collected for many purposes such as intelligent control, business analysis, device monitoring and so on. Due to changes in business or functional requirements or changes in device hardware, the application logic and even the data collected may change. Schemaless writing automatically creates storage structures for your data as it is being written to TDengine, so that you do not need to create supertables in advance. When necessary, schemaless writing -will automatically add the required columns to ensure that the data written by the user is stored correctly. +In IoT applications, data is collected for many purposes such as intelligent control, business analysis, device monitoring and so on. Due to changes in business or functional requirements or changes in device hardware, the application logic and even the data collected may change. Schemaless writing automatically creates storage structures for your data as it is being written to TDengine, so that you do not need to create supertables in advance. When necessary, schemaless writing will automatically add the required columns to ensure that the data written by the user is stored correctly. The schemaless writing method creates super tables and their corresponding subtables. These are completely indistinguishable from the super tables and subtables created directly via SQL. You can write data directly to them via SQL statements. Note that the names of tables created by schemaless writing are based on fixed mapping rules for tag values, so they are not explicitly ideographic and they lack readability. -Tips: -The schemaless write will automatically create a table. You do not need to create a table manually, or an unknown error may occur. +Note: Schemaless writing creates tables automatically. Creating tables manually is not supported with schemaless writing. ## Schemaless Writing Line Protocol @@ -50,8 +48,7 @@ In the schemaless writing data line protocol, each data item in the field_set ne - `t`, `T`, `true`, `True`, `TRUE`, `f`, `F`, `false`, and `False` will be handled directly as BOOL types. -For example, the following data rows write c1 column as 3 (BIGINT), c2 column as false (BOOL), c3 column -as "passit" (BINARY), c4 column as 4 (DOUBLE), and the primary key timestamp as 1626006833639000000 to child table with the t1 label as "3" (NCHAR), the t2 label as "4" (NCHAR), and the t3 label as "t3" (NCHAR) and the super table named `st`. +For example, the following string indicates that the one row of data is written to the st supertable with the t1 tag as "3" (NCHAR), the t2 tag as "4" (NCHAR), and the t3 tag as "t3" (NCHAR); the c1 column is 3 (BIGINT), the c2 column is false (BOOL), the c3 column is "passit" (BINARY), the c4 column is 4 (DOUBLE), and the primary key timestamp is 1626006833639000000. ```json st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 @@ -69,23 +66,31 @@ Schemaless writes process row data according to the following principles. "measurement,tag_key1=tag_value1,tag_key2=tag_value2" ``` +:::tip Note that tag_key1, tag_key2 are not the original order of the tags entered by the user but the result of using the tag names in ascending order of the strings. Therefore, tag_key1 is not the first tag entered in the line protocol. -The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t_" is a fixed prefix that every table generated by this mapping relationship has. +The string's MD5 hash value "md5_val" is calculated after the ranking is completed. The calculation result is then combined with the string to generate the table name: "t_md5_val". "t\_" is a fixed prefix that every table generated by this mapping relationship has. +::: + You can configure smlChildTableName in taos.cfg to specify table names, for example, `smlChildTableName=tname`. You can insert `st,tname=cpul,t1=4 c1=3 1626006833639000000` and the cpu1 table will be automatically created. Note that if multiple rows have the same tname but different tag_set values, the tag_set of the first row is used to create the table and the others are ignored. 2. If the super table obtained by parsing the line protocol does not exist, this super table is created. + **Important:** Manually creating supertables for schemaless writing is not supported. Schemaless writing creates appropriate supertables automatically. + 3. If the subtable obtained by the parse line protocol does not exist, Schemaless creates the sub-table according to the subtable name determined in steps 1 or 2. + 4. If the specified tag or regular column in the data row does not exist, the corresponding tag or regular column is added to the super table (only incremental). -5. If there are some tag columns or regular columns in the super table that are not specified to take values in a data row, then the values of these columns are set to - NULL. + +5. If there are some tag columns or regular columns in the super table that are not specified to take values in a data row, then the values of these columns are set to NULL. + 6. For BINARY or NCHAR columns, if the length of the value provided in a data row exceeds the column type limit, the maximum length of characters allowed to be stored in the column is automatically increased (only incremented and not decremented) to ensure complete preservation of the data. + 7. Errors encountered throughout the processing will interrupt the writing process and return an error code. -8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur.(smlDataFormat in taos.cfg default to false after version of 3.0.1.3, discarded since 3.0.3.0) -:::tip -All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed -48KB, and the total length of tag value cannot exceed 16KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. +8. It is assumed that the order of field_set in a supertable is consistent, meaning that the first record contains all fields and subsequent records store fields in the same order. If the order is not consistent, set smlDataFormat in taos.cfg to false. Otherwise, data will be written out of order and a database error will occur. + Note: TDengine 3.0.3.0 and later automatically detect whether order is consistent. This parameter is no longer used. +:::tip +All processing logic of schemaless will still follow TDengine's underlying restrictions on data structures, such as the total length of each row of data cannot exceed 48 KB and the total length of a tag value cannot exceed 16 KB. See [TDengine SQL Boundary Limits](/taos-sql/limit) for specific constraints in this area. ::: ## Time resolution recognition @@ -114,8 +119,7 @@ In OpenTSDB file and JSON protocol modes, the precision of the timestamp is dete ## Data Model Mapping -This section describes how data in line protocol is mapped to a schema. The data measurement in each line is mapped to a -supertable name. The tag name in tag_set is the tag name in the schema, and the name in field_set is the column name in the schema. The following example shows how data is mapped: +This section describes how data in InfluxDB line protocol is mapped to a schema. The data measurement in each line is mapped to a supertable name. The tag name in tag_set is the tag name in the schema, and the name in field_set is the column name in the schema. The following example shows how data is mapped: ```json st,t1=3,t2=4,t3=t3 c1=3i64,c3="passit",c2=false,c4=4f64 1626006833639000000 @@ -160,7 +164,7 @@ The preceding data includes a new entry, c6, with type binary(6). When this occu TDengine guarantees the idempotency of data writes. This means that you can repeatedly call the API to perform write operations with bad data. However, TDengine does not guarantee the atomicity of multi-row writes. In a multi-row write, some data may be written successfully and other data unsuccessfully. -##: Error Codes +## Error Codes The TSDB_CODE_TSC_LINE_SYNTAX_ERROR indicates an error in the schemaless writing component. This error occurs when writing text. For other errors, schemaless writing uses the standard TDengine error codes diff --git a/docs/en/14-reference/14-taosKeeper.md b/docs/en/14-reference/14-taosKeeper.md index 895bd82e1924a59227bc8193c277b96eccd6cd66..cfc25543873f56f5511abcf3b6c20d556b3da4fc 100644 --- a/docs/en/14-reference/14-taosKeeper.md +++ b/docs/en/14-reference/14-taosKeeper.md @@ -108,7 +108,7 @@ The following `launchctl` commands can help you manage taoskeeper service: #### Launch With Configuration File -You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/keeper.toml` is used by default. If this file does not specify configurations, the default values are used. +You can quickly launch taosKeeper with the following commands. If you do not specify a configuration file, `/etc/taos/taoskeeper.toml` is used by default. If this file does not specify configurations, the default values are used. ```shell $ taoskeeper -c @@ -153,6 +153,10 @@ database = "log" # standard tables to monitor tables = ["normal_table"] + +# database options for db storing metrics data +[metrics.databaseoptions] +cachemodel = "none" ``` ### Obtain Monitoring Metrics @@ -203,7 +207,7 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 ``` -### check_health +### check\_health ``` $ curl -i http://127.0.0.1:6043/check_health @@ -219,3 +223,29 @@ Content-Length: 19 {"version":"1.0.0"} ``` + +### taoskeeper with Prometheus + +There is `/metrics` api in taoskeeper provide TDengine metric data for Prometheus. + +#### scrape config + +Scrape config in Prometheus specifies a set of targets and parameters describing how to scrape metric data from endpoint. For more information, please reference to [Prometheus documents](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config). + +``` +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: "taoskeeper" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["localhost:6043"] +``` + +#### Dashboard + +There is a dashboard named `TaosKeeper Prometheus Dashboard for 3.x`, which provides a monitoring dashboard similar to TInsight. + +In Grafana, click the Dashboard menu and click `import`, enter the dashboard ID `18587` and click the `Load` button. Then finished importing `TaosKeeper Prometheus Dashboard for 3.x` dashboard. + diff --git a/docs/en/28-releases/01-tdengine.md b/docs/en/28-releases/01-tdengine.md index b160058d02c22dc824b0e960ad61cbe596999a70..acfbf6a0baa63d830aed57e7f08c370ea04722c0 100644 --- a/docs/en/28-releases/01-tdengine.md +++ b/docs/en/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ For TDengine 2.x installation packages by version, please visit [here](https://w import Release from "/components/ReleaseV3"; +## 3.0.4.0 + + + ## 3.0.3.2 diff --git a/docs/en/28-releases/02-tools.md b/docs/en/28-releases/02-tools.md index 17581b780a8cba5e0e73ef7d0ff8da8e54f0de56..e8f7a54566128260937cce3a31a5b82f22f1d1b6 100644 --- a/docs/en/28-releases/02-tools.md +++ b/docs/en/28-releases/02-tools.md @@ -10,6 +10,10 @@ For other historical version installers, please visit [here](https://www.taosdat import Release from "/components/ReleaseV3"; +## 2.4.12 + + + ## 2.4.11 diff --git a/docs/examples/python/schemaless_insert.py b/docs/examples/python/schemaless_insert.py new file mode 100644 index 0000000000000000000000000000000000000000..334a4b728f5fb3914bdd346083ba76607ea78ce2 --- /dev/null +++ b/docs/examples/python/schemaless_insert.py @@ -0,0 +1,21 @@ +import taos + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/schemaless_insert_raw.py b/docs/examples/python/schemaless_insert_raw.py new file mode 100644 index 0000000000000000000000000000000000000000..0fda7dc505e8f1ff95912282e85938eef8d8963c --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw.py @@ -0,0 +1,74 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + res = conn.schemaless_insert_raw(lines, 1, 0) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + res = conn.schemaless_insert_raw(lines, 1, 0) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + res = conn.schemaless_insert_raw(lines, 1, 0) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except SchemalessError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_raw_req_id.py b/docs/examples/python/schemaless_insert_raw_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..606e51098665a611d54379368fb3c099f9b6e33b --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw_req_id.py @@ -0,0 +1,76 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + ttl = 1000 + req_id = utils.gen_req_id() + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl, req_id=req_id) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_raw_ttl.py b/docs/examples/python/schemaless_insert_raw_ttl.py new file mode 100644 index 0000000000000000000000000000000000000000..cf57792534cce73317dbdac3a9915adfa73a389c --- /dev/null +++ b/docs/examples/python/schemaless_insert_raw_ttl.py @@ -0,0 +1,73 @@ +import taos +from taos import utils +from taos import TaosConnection +from taos.cinterface import * +from taos.error import OperationalError, SchemalessError + +conn = taos.connect() +dbname = "taos_schemaless_insert" +try: + conn.execute("drop database if exists %s" % dbname) + + if taos.IS_V3: + conn.execute("create database if not exists %s schemaless 1 precision 'ns'" % dbname) + else: + conn.execute("create database if not exists %s update 2 precision 'ns'" % dbname) + + conn.select_db(dbname) + + lines = '''st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000 + st,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin, abc",c2=true,c4=5f64,c5=5f64,c6=7u64 1626006933640000000 + stf,t1=4i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print("affected rows: ", res) + assert (res == 3) + + lines = '''stf,t1=5i64,t3="t4",t2=5f64,t4=5f64 c1=3i64,c3=L"passitagin_stf",c2=false,c5=5f64,c6=7u64 1626006933641000000''' + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print("affected rows: ", res) + assert (res == 1) + + result = conn.query("select * from st") + dict2 = result.fetch_all_into_dict() + print(dict2) + print(result.row_count) + + all = result.rows_iter() + for row in all: + print(row) + result.close() + assert (result.row_count == 2) + + # error test + lines = ''',t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"passit",c2=false,c4=4f64 1626006833639000000''' + try: + ttl = 1000 + res = conn.schemaless_insert_raw(lines, 1, 0, ttl=ttl) + print(res) + # assert(False) + except SchemalessError as err: + print('**** error: ', err) + # assert (err.msg == 'Invalid data format') + + result = conn.query("select * from st") + print(result.row_count) + all = result.rows_iter() + for row in all: + print(row) + result.close() + + conn.execute("drop database if exists %s" % dbname) + conn.close() +except InterfaceError as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) +except Exception as err: + conn.execute("drop database if exists %s" % dbname) + conn.close() + print(err) + raise err diff --git a/docs/examples/python/schemaless_insert_req_id.py b/docs/examples/python/schemaless_insert_req_id.py new file mode 100644 index 0000000000000000000000000000000000000000..ee1472db6939474ccf34c591fe4dcf4891f0d2ab --- /dev/null +++ b/docs/examples/python/schemaless_insert_req_id.py @@ -0,0 +1,22 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, req_id=1) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, req_id=2) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/examples/python/schemaless_insert_ttl.py b/docs/examples/python/schemaless_insert_ttl.py new file mode 100644 index 0000000000000000000000000000000000000000..85050439f2a451b9d12f7fe20ebbae5297adb5a4 --- /dev/null +++ b/docs/examples/python/schemaless_insert_ttl.py @@ -0,0 +1,22 @@ +import taos +from taos import SmlProtocol, SmlPrecision + +conn = taos.connect() +dbname = "pytest_line" +conn.execute("drop database if exists %s" % dbname) +conn.execute("create database if not exists %s precision 'us'" % dbname) +conn.select_db(dbname) + +lines = [ + 'st,t1=3i64,t2=4f64,t3="t3" c1=3i64,c3=L"pass",c2=false,c4=4f64 1626006833639000000', +] +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, ttl=1000) +print("inserted") + +conn.schemaless_insert(lines, taos.SmlProtocol.LINE_PROTOCOL, taos.SmlPrecision.NOT_CONFIGURED, ttl=1000) + +result = conn.query("show tables") +for row in result: + print(row) + +conn.execute("drop database if exists %s" % dbname) diff --git a/docs/zh/05-get-started/03-package.md b/docs/zh/05-get-started/03-package.md index d9c64cf37a1ef204e45e967b924f7abbb023ca73..b327d33350e2396d47c7e7edd90a65ad1ccb35a9 100644 --- a/docs/zh/05-get-started/03-package.md +++ b/docs/zh/05-get-started/03-package.md @@ -179,7 +179,7 @@ Active: inactive (dead) ::: -## TDengine 命令行（CLI） +**TDengine 命令行（CLI）** 为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 @@ -189,7 +189,7 @@ Active: inactive (dead) 安装后，可以在拥有管理员权限的 cmd 窗口执行 `sc start taosd` 或在 `C:\TDengine` 目录下，运行 `taosd.exe` 来启动 TDengine 服务进程。 -## TDengine 命令行（CLI） +**TDengine 命令行（CLI）** 为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在终端执行 `taos` 即可。 @@ -216,7 +216,7 @@ Active: inactive (dead) ::: -## TDengine 命令行（CLI） +**TDengine 命令行（CLI）** 为便于检查 TDengine 的状态，执行数据库（Database）的各种即席（Ad Hoc）查询，TDengine 提供一命令行应用程序（以下简称为 TDengine CLI）taos。要进入 TDengine 命令行，您只要在 Windows 终端的 C:\TDengine 目录下，运行 taos.exe 来启动 TDengine 命令行。 diff --git a/docs/zh/07-develop/07-tmq.mdx b/docs/zh/07-develop/07-tmq.mdx index 036c9aa86e5c91903a74bffd1f9c59a13a97c116..a3833438a2100a5d960cff1e5e6e194ba000004d 100644 --- a/docs/zh/07-develop/07-tmq.mdx +++ b/docs/zh/07-develop/07-tmq.mdx @@ -221,7 +221,7 @@ void Close() ```sql DROP DATABASE IF EXISTS tmqdb; -CREATE DATABASE tmqdb; +CREATE DATABASE tmqdb WAL_RETENTION_PERIOD 3600; CREATE TABLE tmqdb.stb (ts TIMESTAMP, c1 INT, c2 FLOAT, c3 VARCHAR(16)) TAGS(t1 INT, t3 VARCHAR(16)); CREATE TABLE tmqdb.ctb0 USING tmqdb.stb TAGS(0, "subtable0"); CREATE TABLE tmqdb.ctb1 USING tmqdb.stb TAGS(1, "subtable1"); diff --git a/docs/zh/07-develop/09-udf.md b/docs/zh/07-develop/09-udf.md index 2eb4b3dee32d7326476bbca350c951ccd0482d71..8a8ef82009c512b550cd514f3d52716f79d8e0a1 100644 --- a/docs/zh/07-develop/09-udf.md +++ b/docs/zh/07-develop/09-udf.md @@ -231,7 +231,7 @@ bit_add 实现多列的按位与功能。如果只有一列，返回这一列。 -### 聚合函数示例 [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c) +### 聚合函数示例1 返回值为数值类型 [l2norm](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/l2norm.c) l2norm 实现了输入列的所有数据的二阶范数，即对每个数据先平方，再累加求和，最后开方。 @@ -243,3 +243,29 @@ l2norm 实现了输入列的所有数据的二阶范数，即对每个数据先 ``` + +### 聚合函数示例2 返回值为字符串类型 [max_vol](https://github.com/taosdata/TDengine/blob/develop/tests/script/sh/max_vol.c) + +max_vol 实现了从多个输入的电压列中找到最大电压，返回由设备ID + 最大电压所在（行，列）+ 最大电压值 组成的组合字符串值 + +创建表： +```bash +create table battery(ts timestamp, vol1 float, vol2 float, vol3 float, deviceId varchar(16)); +``` +创建自定义函数： +```bash +create aggregate function max_vol as '/root/udf/libmaxvol.so' outputtype binary(64) bufsize 10240 language 'C'; +``` +使用自定义函数： +```bash +select max_vol(vol1,vol2,vol3,deviceid) from battery; +``` + +
+max_vol.c + +```c +{{#include tests/script/sh/max_vol.c}} +``` + +
\ No newline at end of file diff --git a/docs/zh/08-connector/02-rest-api.mdx b/docs/zh/08-connector/02-rest-api.mdx index a081595bca5f66bff3c60808126516633f92614b..f3f1e087d809b7ce98d27721ada13b7a5fdf7159 100644 --- a/docs/zh/08-connector/02-rest-api.mdx +++ b/docs/zh/08-connector/02-rest-api.mdx @@ -383,6 +383,133 @@ curl http://192.168.0.1:6041/rest/login/root/taosdata } ``` +## TDengine 2.x 和 3.0 之间 REST API 的差异 + +### URI + +| URI | TDengine 2.x | TDengine 3.0 | +| :--------------------| :------------------: | :--------------------------------------------------: | +| /rest/sql | 支持 | 支持 （响应代码和消息体不同） | +| /rest/sqlt | 支持 | 不再支持 | +| /rest/sqlutc | 支持 | 不再支持 | + + +### HTTP code + +| HTTP code | TDengine 2.x | TDengine 3.0 | 备注 | +| :--------------------| :------------------: | :----------: | :-----------------------------------: | +| 200 | 支持 | 支持 | 正确返回和 taosc 接口错误返回 | +| 400 | 不支持 | 支持 | 参数错误返回 | +| 401 | 不支持 | 支持 | 鉴权失败 | +| 404 | 支持 | 支持 | 接口不存在 | +| 500 | 不支持 | 支持 | 内部错误 | +| 503 | 支持 | 支持 | 系统资源不足 | + + +### 响应代码和消息体 + +#### TDengine 2.x 响应代码和消息体 + +```JSON +{ + "status": "succ", + "head": [ + "name", + "created_time", + "ntables", + "vgroups", + "replica", + "quorum", + "days", + "keep1,keep2,keep(D)", + "cache(MB)", + "blocks", + "minrows", + "maxrows", + "wallevel", + "fsync", + "comp", + "precision", + "status" + ], + "data": [ + [ + "log", + "2020-09-02 17:23:00.039", + 4, + 1, + 1, + 1, + 10, + "30,30,30", + 1, + 3, + 100, + 4096, + 1, + 3000, + 2, + "us", + "ready" + ] + ], + "rows": 1 +} +``` +``` + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], +``` + +#### TDengine 3.0 响应代码和消息体 + + +```JSON +{ + "code": 0, + "column_meta": [ + [ + "name", + "VARCHAR", + 64 + ], + [ + "ntables", + "BIGINT", + 8 + ], + [ + "status", + "VARCHAR", + 10 + ] + ], + "data": [ + [ + "information_schema", + 16, + "ready" + ], + [ + "performance_schema", + 9, + "ready" + ] + ], + "rows": 2 +} +``` + ## 参考 [taosAdapter](/reference/taosadapter/) diff --git a/docs/zh/08-connector/14-java.mdx b/docs/zh/08-connector/14-java.mdx index d1c12583659e2f6dd2a63357669b92c3b1ff29b1..8a9bba7fe73d5f2bc41095f6f8d5a78371373992 100644 --- a/docs/zh/08-connector/14-java.mdx +++ b/docs/zh/08-connector/14-java.mdx @@ -82,7 +82,7 @@ Maven 项目中，在 pom.xml 中添加以下依赖： com.taosdata.jdbc taos-jdbcdriver - 3.0.0 + 3.1.0 ``` @@ -230,7 +230,7 @@ INSERT INTO test.t1 USING test.weather (ts, temperature) TAGS('California.SanFra **注意**： - 应用中设置的 client parameter 为进程级别的，即如果要更新 client 的参数，需要重启应用。这是因为 client parameter 是全局参数，仅在应用程序的第一次设置生效。 -- 以下示例代码基于 taos-jdbcdriver-3.0.0。 +- 以下示例代码基于 taos-jdbcdriver-3.1.0。 ```java public Connection getConn() throws Exception{ @@ -367,7 +367,7 @@ TDengine 的 JDBC 原生连接实现大幅改进了参数绑定方式对数据 **注意**： - JDBC REST 连接目前不支持参数绑定 -- 以下示例代码基于 taos-jdbcdriver-3.0.0 +- 以下示例代码基于 taos-jdbcdriver-3.1.0 - binary 类型数据需要调用 setString 方法，nchar 类型数据需要调用 setNString 方法 - setString 和 setNString 都要求用户在 size 参数里声明表定义中对应列的列宽 @@ -635,7 +635,7 @@ TDengine 支持无模式写入功能。无模式写入兼容 InfluxDB 的 行协 **注意**： - JDBC REST 连接目前不支持无模式写入 -- 以下示例代码基于 taos-jdbcdriver-3.0.0 +- 以下示例代码基于 taos-jdbcdriver-3.1.0 ```java public class SchemalessInsertTest { diff --git a/docs/zh/08-connector/30-python.mdx b/docs/zh/08-connector/30-python.mdx index 5395610df3ce5535293f8017f5e14c403e9a8279..1cff142e11d39e6afe86fab187697d222f37a9dd 100644 --- a/docs/zh/08-connector/30-python.mdx +++ b/docs/zh/08-connector/30-python.mdx @@ -484,6 +484,56 @@ TaosCursor 类使用原生连接进行写入、查询操作。在客户端多线 +### 无模式写入 + +连接器支持无模式写入功能。 + + + + +简单写入 + +```python +{{#include docs/examples/python/schemaless_insert.py}} +``` + +带有 ttl 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_ttl.py}} +``` + +带有 req_id 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_req_id.py}} +``` + + + + + +简单写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw.py}} +``` + +带有 ttl 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw_ttl.py}} +``` + +带有 req_id 参数的写入 + +```python +{{#include docs/examples/python/schemaless_insert_raw_req_id.py}} +``` + + + + ### 其它示例程序 | 示例程序链接 | 示例程序内容 | diff --git a/docs/zh/08-connector/index.md b/docs/zh/08-connector/index.md index eb1f3a9a9a728f666f9964ef51795b76991d1fcc..bb8c95a15aaa9e46b64e19d8d4b647edd83f5eac 100644 --- a/docs/zh/08-connector/index.md +++ b/docs/zh/08-connector/index.md @@ -61,7 +61,7 @@ TDengine 版本更新往往会增加新的功能特性，列表中的连接器 | **普通查询** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **参数绑定** | 暂不支持 | 暂不支持 | 支持 | 支持 | 暂不支持 | 支持 | | **数据订阅（TMQ）** | 支持 | 支持 | 支持 | 暂不支持 | 暂不支持 | 支持 | -| **Schemaless** | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | +| **Schemaless** | 支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | 暂不支持 | | **批量拉取（基于 WebSocket）** | 支持 | 支持 | 支持 | 支持 | 支持 | 支持 | | **DataFrame** | 不支持 | 支持 | 不支持 | 不支持 | 不支持 | 不支持 | diff --git a/docs/zh/12-taos-sql/02-database.md b/docs/zh/12-taos-sql/02-database.md index b54998e08d33869d113ae883b14f929e5381f168..dd306356605929e521ee4416c7694bc0731af957 100644 --- a/docs/zh/12-taos-sql/02-database.md +++ b/docs/zh/12-taos-sql/02-database.md @@ -36,7 +36,6 @@ database_option: { | TSDB_PAGESIZE value | WAL_RETENTION_PERIOD value | WAL_RETENTION_SIZE value - | WAL_ROLL_PERIOD value | WAL_SEGMENT_SIZE value } ``` diff --git a/docs/zh/12-taos-sql/10-function.md b/docs/zh/12-taos-sql/10-function.md index 94f805205183aca6a7a08d9a836b0df33c7ed1f0..50e82e6b902d0beddbc205a7fde5867389c1cbc3 100644 --- a/docs/zh/12-taos-sql/10-function.md +++ b/docs/zh/12-taos-sql/10-function.md @@ -459,12 +459,17 @@ TO_JSON(str_literal) #### TO_UNIXTIMESTAMP ```sql -TO_UNIXTIMESTAMP(expr) +TO_UNIXTIMESTAMP(expr [, return_timestamp]) + +return_timestamp: { + 0 + | 1 +} ``` **功能说明**：将日期时间格式的字符串转换成为 UNIX 时间戳。 -**返回结果数据类型**：BIGINT。 +**返回结果数据类型**：BIGINT, TIMESTAMP。 **应用字段**：VARCHAR, NCHAR。 @@ -476,6 +481,7 @@ TO_UNIXTIMESTAMP(expr) - 输入的日期时间字符串须符合 ISO8601/RFC3339 标准，无法转换的字符串格式将返回 NULL。 - 返回的时间戳精度与当前 DATABASE 设置的时间精度一致。 +- return_timestamp 指定函数返回值是否为时间戳类型，设置为1时返回 TIMESTAMP 类型，设置为0时返回 BIGINT 类型。如不指定缺省返回 BIGINT 类型。 ### 时间和日期函数 diff --git a/docs/zh/12-taos-sql/14-stream.md b/docs/zh/12-taos-sql/14-stream.md index f10077345f394936d3df4eccf8811fc67f862c15..d90db3cab01e2bca17f00fd8540f4f2dd2e03cdc 100644 --- a/docs/zh/12-taos-sql/14-stream.md +++ b/docs/zh/12-taos-sql/14-stream.md @@ -10,8 +10,12 @@ description: 流式计算的相关 SQL 的详细语法 ```sql CREATE STREAM [IF NOT EXISTS] stream_name [stream_options] INTO stb_name[(field1_name, ...)] [TAGS (create_definition [, create_definition] ...)] SUBTABLE(expression) AS subquery stream_options: { - TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] - WATERMARK time + TRIGGER [AT_ONCE | WINDOW_CLOSE | MAX_DELAY time] + WATERMARK time + IGNORE EXPIRED [0|1] + DELETE_MARK time + FILL_HISTORY [0|1] + IGNORE UPDATE [0|1] } ``` @@ -166,7 +170,7 @@ T3 时刻，最新事件到达，T 向后推移超过了第二个窗口关闭的 在 window_close 或 max_delay 模式下，窗口关闭直接影响推送结果。在 at_once 模式下，窗口关闭只与内存占用有关。 -## 流式计算的过期数据处理策略 +## 流式计算对于过期数据的处理策略 对于已关闭的窗口，再次落入该窗口中的数据被标记为过期数据. @@ -174,11 +178,20 @@ TDengine 对于过期数据提供两种处理方式，由 IGNORE EXPIRED 选项 1. 重新计算，即 IGNORE EXPIRED 0：从 TSDB 中重新查找对应窗口的所有数据并重新计算得到最新结果 -2. 直接丢弃, 即 IGNORE EXPIRED 1：默认配置，忽略过期数据 +2. 直接丢弃，即 IGNORE EXPIRED 1：默认配置，忽略过期数据 无论在哪种模式下，watermark 都应该被妥善设置，来得到正确结果（直接丢弃模式）或避免频繁触发重算带来的性能开销（重新计算模式）。 +## 流式计算对于修改数据的处理策略 + +TDengine 对于修改数据提供两种处理方式，由 IGNORE UPDATE 选项指定： + +1. 检查数据是否被修改，即 IGNORE UPDATE 0：默认配置，如果被修改，则重新计算对应窗口。 + +2. 不检查数据是否被修改，全部按增量数据计算，即 IGNORE UPDATE 1。 + + ## 写入已存在的超级表 ```sql [field1_name,...] @@ -202,3 +215,37 @@ PARTITION 子句中，为 concat("tag-", tbname)定义了一个别名cc, 对应 会对TAG信息进行如下检查 1.检查tag的schema信息是否匹配，对于不匹配的，则自动进行数据类型转换，当前只有数据长度大于4096byte时才报错，其余场景都能进行类型转换。 2.检查tag的个数是否相同，如果不同，需要显示的指定超级表与subquery的tag的对应关系，否则报错；如果相同，可以指定对应关系，也可以不指定，不指定则按位置顺序对应。 + +## 清理中间状态 + +``` +DELETE_MARK time +``` +DELETE_MARK用于删除缓存的窗口状态，也就是删除流计算的中间结果。如果不设置，默认值是10年 +T = 最新事件时间 - DELETE_MARK + +## 流式计算支持的函数 + +1. 所有的 [单行函数](../function/#单行函数) 均可用于流计算。 +2. 以下 19 个聚合/选择函数 不能 应用在创建流计算的 SQL 语句，[系统信息函数](../function/#系统信息函数) 也不能用于流计算中。此外的其他类型的函数均可用于流计算。 + +- [leastsquares](../function/#leastsquares) +- [percentile](../function/#percentile) +- [top](../function/#top) +- [bottom](../function/#bottom) +- [elapsed](../function/#elapsed) +- [interp](../function/#interp) +- [derivative](../function/#derivative) +- [irate](../function/#irate) +- [twa](../function/#twa) +- [histogram](../function/#histogram) +- [diff](../function/#diff) +- [statecount](../function/#statecount) +- [stateduration](../function/#stateduration) +- [csum](../function/#csum) +- [mavg](../function/#mavg) +- [sample](../function/#sample) +- [tail](../function/#tail) +- [unique](../function/#unique) +- [mode](../function/#mode) + diff --git a/docs/zh/12-taos-sql/23-perf.md b/docs/zh/12-taos-sql/23-perf.md index d4ee0e178c02e65eb3f1ceaa73e170893f65cc88..fc0b8072a7ea3722990b76bea2002bae1f12d7d3 100644 --- a/docs/zh/12-taos-sql/23-perf.md +++ b/docs/zh/12-taos-sql/23-perf.md @@ -69,7 +69,7 @@ TDengine 3.0 版本开始提供一个内置数据库 `performance_schema`，其 | 1 | consumer_id | BIGINT | 消费者的唯一 ID | | 2 | consumer_group | BINARY(192) | 消费者组 | | 3 | client_id | BINARY(192) | 用户自定义字符串，通过创建 consumer 时指定 client_id 来展示 | -| 4 | status | BINARY(20) | 消费者当前状态 | +| 4 | status | BINARY(20) | 消费者当前状态。消费者状态包括：ready(正常可用)、 lost(连接已丢失)、 rebalancing(消费者所属 vgroup 正在分配中)、unknown(未知状态)| | 5 | topics | BINARY(204) | 被订阅的 topic。若订阅多个 topic，则展示为多行 | | 6 | up_time | TIMESTAMP | 第一次连接 taosd 的时间 | | 7 | subscribe_time | TIMESTAMP | 上一次发起订阅的时间 | diff --git a/docs/zh/12-taos-sql/29-changes.md b/docs/zh/12-taos-sql/29-changes.md index f1952664c2697e01f8880407a593707dfb25007b..8a5f1b20e3e0847320a0566a53b8438a3b1f1fa9 100644 --- a/docs/zh/12-taos-sql/29-changes.md +++ b/docs/zh/12-taos-sql/29-changes.md @@ -27,7 +27,7 @@ description: "TDengine 3.0 版本的语法变更说明" | - | :------- | :-------- | :------- | | 1 | ALTER ACCOUNT | 废除 | 2.x中为企业版功能，3.0不再支持。语法暂时保留了，执行报“This statement is no longer supported”错误。 | 2 | ALTER ALL DNODES | 新增 | 修改所有DNODE的参数。 -| 3 | ALTER DATABASE | 调整 |

废除

  • QUORUM：写入需要的副本确认数。3.0版本使用STRICT来指定强一致还是弱一致。3.0.0版本STRICT暂不支持修改。
  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • COMP：3.0版本暂不支持修改。

新增

  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。

调整

  • REPLICA：3.0.0版本暂不支持修改。
  • KEEP：3.0版本新增支持带单位的设置方式。
+| 3 | ALTER DATABASE | 调整 |

废除

  • QUORUM：写入需要的副本确认数。3.0 版本默认行为是强一致性，且不支持修改为弱一致性。
  • BLOCKS：VNODE使用的内存块数。3.0版本使用BUFFER来表示VNODE写入内存池的大小。
  • UPDATE：更新操作的支持模式。3.0版本所有数据库都支持部分列更新。
  • CACHELAST：缓存最新一行数据的模式。3.0版本用CACHEMODEL代替。
  • COMP：3.0版本暂不支持修改。

新增

  • CACHEMODEL：表示是否在内存中缓存子表的最近数据。
  • CACHESIZE：表示缓存子表最近数据的内存大小。
  • WAL_FSYNC_PERIOD：代替原FSYNC参数。
  • WAL_LEVEL：代替原WAL参数。
  • WAL_RETENTION_PERIOD：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。
  • WAL_RETENTION_SIZE：3.0.4.0版本新增，wal文件的额外保留策略，用于数据订阅。

调整

  • REPLICA：3.0.0版本暂不支持修改。
  • KEEP：3.0版本新增支持带单位的设置方式。
| 4 | ALTER STABLE | 调整 | 废除
  • CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。
    新增
  • RENAME TAG：代替原CHANGE TAG子句。
  • COMMENT：修改超级表的注释。
| 5 | ALTER TABLE | 调整 | 废除
  • CHANGE TAG：修改标签列的名称。3.0版本使用RENAME TAG代替。
    新增
  • RENAME TAG：代替原CHANGE TAG子句。
  • COMMENT：修改表的注释。
  • TTL：修改表的生命周期。
| 6 | ALTER USER | 调整 | 废除
  • PRIVILEGE：修改用户权限。3.0版本使用GRANT和REVOKE来授予和回收权限。
    新增
  • ENABLE：启用或停用此用户。
  • SYSINFO：修改用户是否可查看系统信息。
diff --git a/docs/zh/14-reference/04-taosadapter.md b/docs/zh/14-reference/04-taosadapter.md index b8c5f9d647fcf920e454fabbf4da9c527b038a76..a10b5b55bc067518056b66346484f3e37bdcdc3c 100644 --- a/docs/zh/14-reference/04-taosadapter.md +++ b/docs/zh/14-reference/04-taosadapter.md @@ -54,94 +54,91 @@ taosAdapter 支持通过命令行参数、环境变量和配置文件来进行 ```shell Usage of taosAdapter: - --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") - --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) - --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") - --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) - --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" - --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") - --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) - -c, --config string config path default /etc/taos/taosadapter.toml - --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) - --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" - --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" - --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" - --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" - --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" - --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) - --help Print this help message and exit - --httpCodeServerError Use a non-200 http status code when taosd returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" - --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) - --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" - --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") - --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) - --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") - --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) - --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) - --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") - --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) - --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") - --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) - --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" - --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" - --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" - --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" - --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") - --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) - --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) - --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") - --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) - --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" - --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" - --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" - --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") - --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" - --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) - --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" - --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" - --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" - --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) - --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" - --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") - --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) - --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" - --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) - --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") - --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) - --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) - --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) - --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" - --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" - --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) - --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") - --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) - --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" - --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" - --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") - --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" - --pool.maxConnect int max connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" - --pool.maxIdle int max idle connections to taosd. Env "TAOS_ADAPTER_POOL_MAX_IDLE" - -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) - --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) - --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) - --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) - --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") - --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) - --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) - --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) - --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) - --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) - --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) - --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) - --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") - --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) - --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") - --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" - --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" - --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") - --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) - --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" - --version Print the version and exit + --collectd.db string collectd db name. Env "TAOS_ADAPTER_COLLECTD_DB" (default "collectd") + --collectd.enable enable collectd. Env "TAOS_ADAPTER_COLLECTD_ENABLE" (default true) + --collectd.password string collectd password. Env "TAOS_ADAPTER_COLLECTD_PASSWORD" (default "taosdata") + --collectd.port int collectd server port. Env "TAOS_ADAPTER_COLLECTD_PORT" (default 6045) + --collectd.ttl int collectd data ttl. Env "TAOS_ADAPTER_COLLECTD_TTL" + --collectd.user string collectd user. Env "TAOS_ADAPTER_COLLECTD_USER" (default "root") + --collectd.worker int collectd write worker. Env "TAOS_ADAPTER_COLLECTD_WORKER" (default 10) + -c, --config string config path default /etc/taos/taosadapter.toml + --cors.allowAllOrigins cors allow all origins. Env "TAOS_ADAPTER_CORS_ALLOW_ALL_ORIGINS" (default true) + --cors.allowCredentials cors allow credentials. Env "TAOS_ADAPTER_CORS_ALLOW_Credentials" + --cors.allowHeaders stringArray cors allow HEADERS. Env "TAOS_ADAPTER_ALLOW_HEADERS" + --cors.allowOrigins stringArray cors allow origins. Env "TAOS_ADAPTER_ALLOW_ORIGINS" + --cors.allowWebSockets cors allow WebSockets. Env "TAOS_ADAPTER_CORS_ALLOW_WebSockets" --cors.exposeHeaders stringArray cors expose headers. Env "TAOS_ADAPTER_Expose_Headers" + --debug enable debug mode. Env "TAOS_ADAPTER_DEBUG" (default true) + --help Print this help message and exit + --httpCodeServerError Use a non-200 http status code when server returns an error. Env "TAOS_ADAPTER_HTTP_CODE_SERVER_ERROR" + --influxdb.enable enable influxdb. Env "TAOS_ADAPTER_INFLUXDB_ENABLE" (default true) + --log.enableRecordHttpSql whether to record http sql. Env "TAOS_ADAPTER_LOG_ENABLE_RECORD_HTTP_SQL" + --log.path string log path. Env "TAOS_ADAPTER_LOG_PATH" (default "/var/log/taos") --log.rotationCount uint log rotation count. Env "TAOS_ADAPTER_LOG_ROTATION_COUNT" (default 30) + --log.rotationSize string log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_ROTATION_SIZE" (default "1GB") + --log.rotationTime duration log rotation time. Env "TAOS_ADAPTER_LOG_ROTATION_TIME" (default 24h0m0s) + --log.sqlRotationCount uint record sql log rotation count. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_COUNT" (default 2) + --log.sqlRotationSize string record sql log rotation size(KB MB GB), must be a positive integer. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_SIZE" (default "1GB") + --log.sqlRotationTime duration record sql log rotation time. Env "TAOS_ADAPTER_LOG_SQL_ROTATION_TIME" (default 24h0m0s) + --logLevel string log level (panic fatal error warn warning info debug trace). Env "TAOS_ADAPTER_LOG_LEVEL" (default "info") + --monitor.collectDuration duration Set monitor duration. Env "TAOS_ADAPTER_MONITOR_COLLECT_DURATION" (default 3s) + --monitor.disable Whether to disable monitoring. Env "TAOS_ADAPTER_MONITOR_DISABLE" + --monitor.disableCollectClientIP Whether to disable collecting clientIP. Env "TAOS_ADAPTER_MONITOR_DISABLE_COLLECT_CLIENT_IP" + --monitor.identity string The identity of the current instance, or 'hostname:port' if it is empty. Env "TAOS_ADAPTER_MONITOR_IDENTITY" + --monitor.incgroup Whether running in cgroup. Env "TAOS_ADAPTER_MONITOR_INCGROUP" + --monitor.password string TDengine password. Env "TAOS_ADAPTER_MONITOR_PASSWORD" (default "taosdata") + --monitor.pauseAllMemoryThreshold float Memory percentage threshold for pause all. Env "TAOS_ADAPTER_MONITOR_PAUSE_ALL_MEMORY_THRESHOLD" (default 80) + --monitor.pauseQueryMemoryThreshold float Memory percentage threshold for pause query. Env "TAOS_ADAPTER_MONITOR_PAUSE_QUERY_MEMORY_THRESHOLD" (default 70) + --monitor.user string TDengine user. Env "TAOS_ADAPTER_MONITOR_USER" (default "root") --monitor.writeInterval duration Set write to TDengine interval. Env "TAOS_ADAPTER_MONITOR_WRITE_INTERVAL" (default 30s) + --monitor.writeToTD Whether write metrics to TDengine. Env "TAOS_ADAPTER_MONITOR_WRITE_TO_TD" + --node_exporter.caCertFile string node_exporter ca cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CA_CERT_FILE" + --node_exporter.certFile string node_exporter cert file path. Env "TAOS_ADAPTER_NODE_EXPORTER_CERT_FILE" + --node_exporter.db string node_exporter db name. Env "TAOS_ADAPTER_NODE_EXPORTER_DB" (default "node_exporter") + --node_exporter.enable enable node_exporter. Env "TAOS_ADAPTER_NODE_EXPORTER_ENABLE" + --node_exporter.gatherDuration duration node_exporter gather duration. Env "TAOS_ADAPTER_NODE_EXPORTER_GATHER_DURATION" (default 5s) + --node_exporter.httpBearerTokenString string node_exporter http bearer token. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_BEARER_TOKEN_STRING" + --node_exporter.httpPassword string node_exporter http password. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_PASSWORD" + --node_exporter.httpUsername string node_exporter http username. Env "TAOS_ADAPTER_NODE_EXPORTER_HTTP_USERNAME" + --node_exporter.insecureSkipVerify node_exporter skip ssl check. Env "TAOS_ADAPTER_NODE_EXPORTER_INSECURE_SKIP_VERIFY" (default true) + --node_exporter.keyFile string node_exporter cert key file path. Env "TAOS_ADAPTER_NODE_EXPORTER_KEY_FILE" + --node_exporter.password string node_exporter password. Env "TAOS_ADAPTER_NODE_EXPORTER_PASSWORD" (default "taosdata") + --node_exporter.responseTimeout duration node_exporter response timeout. Env "TAOS_ADAPTER_NODE_EXPORTER_RESPONSE_TIMEOUT" (default 5s) + --node_exporter.ttl int node_exporter data ttl. Env "TAOS_ADAPTER_NODE_EXPORTER_TTL" + --node_exporter.urls strings node_exporter urls. Env "TAOS_ADAPTER_NODE_EXPORTER_URLS" (default [http://localhost:9100]) + --node_exporter.user string node_exporter user. Env "TAOS_ADAPTER_NODE_EXPORTER_USER" (default "root") + --opentsdb.enable enable opentsdb. Env "TAOS_ADAPTER_OPENTSDB_ENABLE" (default true) + --opentsdb_telnet.batchSize int opentsdb_telnet batch size. Env "TAOS_ADAPTER_OPENTSDB_TELNET_BATCH_SIZE" (default 1) + --opentsdb_telnet.dbs strings opentsdb_telnet db names. Env "TAOS_ADAPTER_OPENTSDB_TELNET_DBS" (default [opentsdb_telnet,collectd_tsdb,icinga2_tsdb,tcollector_tsdb]) + --opentsdb_telnet.enable enable opentsdb telnet,warning: without auth info(default false). Env "TAOS_ADAPTER_OPENTSDB_TELNET_ENABLE" + --opentsdb_telnet.flushInterval duration opentsdb_telnet flush interval (0s means not valid) . Env "TAOS_ADAPTER_OPENTSDB_TELNET_FLUSH_INTERVAL" + --opentsdb_telnet.maxTCPConnections int max tcp connections. Env "TAOS_ADAPTER_OPENTSDB_TELNET_MAX_TCP_CONNECTIONS" (default 250) + --opentsdb_telnet.password string opentsdb_telnet password. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PASSWORD" (default "taosdata") + --opentsdb_telnet.ports ints opentsdb telnet tcp port. Env "TAOS_ADAPTER_OPENTSDB_TELNET_PORTS" (default [6046,6047,6048,6049]) + --opentsdb_telnet.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TCP_KEEP_ALIVE" + --opentsdb_telnet.ttl int opentsdb_telnet data ttl. Env "TAOS_ADAPTER_OPENTSDB_TELNET_TTL" + --opentsdb_telnet.user string opentsdb_telnet user. Env "TAOS_ADAPTER_OPENTSDB_TELNET_USER" (default "root") + --pool.idleTimeout duration Set idle connection timeout. Env "TAOS_ADAPTER_POOL_IDLE_TIMEOUT" + --pool.maxConnect int max connections to server. Env "TAOS_ADAPTER_POOL_MAX_CONNECT" + --pool.maxIdle int max idle connections to server. Env "TAOS_ADAPTER_POOL_MAX_IDLE" + -P, --port int http port. Env "TAOS_ADAPTER_PORT" (default 6041) + --prometheus.enable enable prometheus. Env "TAOS_ADAPTER_PROMETHEUS_ENABLE" (default true) + --restfulRowLimit int restful returns the maximum number of rows (-1 means no limit). Env "TAOS_ADAPTER_RESTFUL_ROW_LIMIT" (default -1) + --smlAutoCreateDB Whether to automatically create db when writing with schemaless. Env "TAOS_ADAPTER_SML_AUTO_CREATE_DB" + --statsd.allowPendingMessages int statsd allow pending messages. Env "TAOS_ADAPTER_STATSD_ALLOW_PENDING_MESSAGES" (default 50000) + --statsd.db string statsd db name. Env "TAOS_ADAPTER_STATSD_DB" (default "statsd") --statsd.deleteCounters statsd delete counter cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_COUNTERS" (default true) + --statsd.deleteGauges statsd delete gauge cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_GAUGES" (default true) + --statsd.deleteSets statsd delete set cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_SETS" (default true) + --statsd.deleteTimings statsd delete timing cache after gather. Env "TAOS_ADAPTER_STATSD_DELETE_TIMINGS" (default true) + --statsd.enable enable statsd. Env "TAOS_ADAPTER_STATSD_ENABLE" (default true) + --statsd.gatherInterval duration statsd gather interval. Env "TAOS_ADAPTER_STATSD_GATHER_INTERVAL" (default 5s) + --statsd.maxTCPConnections int statsd max tcp connections. Env "TAOS_ADAPTER_STATSD_MAX_TCP_CONNECTIONS" (default 250) + --statsd.password string statsd password. Env "TAOS_ADAPTER_STATSD_PASSWORD" (default "taosdata") + --statsd.port int statsd server port. Env "TAOS_ADAPTER_STATSD_PORT" (default 6044) + --statsd.protocol string statsd protocol [tcp or udp]. Env "TAOS_ADAPTER_STATSD_PROTOCOL" (default "udp") + --statsd.tcpKeepAlive enable tcp keep alive. Env "TAOS_ADAPTER_STATSD_TCP_KEEP_ALIVE" --statsd.ttl int statsd data ttl. Env "TAOS_ADAPTER_STATSD_TTL" + --statsd.user string statsd user. Env "TAOS_ADAPTER_STATSD_USER" (default "root") + --statsd.worker int statsd write worker. Env "TAOS_ADAPTER_STATSD_WORKER" (default 10) + --taosConfigDir string load taos client config path. Env "TAOS_ADAPTER_TAOS_CONFIG_FILE" + --tmq.releaseIntervalMultiplierForAutocommit int When set to autocommit, the interval for message release is a multiple of the autocommit interval, with a default value of 2 and a minimum value of 1 and a maximum value of 10. Env "TAOS_ADAPTER_TMQ_RELEASE_INTERVAL_MULTIPLIER_FOR_AUTOCOMMIT" (default 2) + --version Print the version and exit ``` 备注： @@ -331,6 +328,10 @@ taosAdapter 通过参数 `restfulRowLimit` 来控制结果的返回条数，-1 taosAdapter 通过参数 `httpCodeServerError` 来设置当 C 接口返回错误时是否返回非 200 的 http 状态码。当设置为 true 时将根据 C 返回的错误码返回不同 http 状态码。具体见 [HTTP 响应码](../../connector/rest-api/#http-响应码)。 +## 配置 schemaless 写入是否自动创建 DB + +taosAdapter 从 3.0.4.0 版本开始，提供参数 `smlAutoCreateDB` 来控制在 schemaless 协议写入时是否自动创建 DB。默认值为 false 不自动创建 DB，需要用户手动创建 DB 后进行 schemaless 写入。 + ## 故障解决 您可以通过命令 `systemctl status taosadapter` 来检查 taosAdapter 运行状态。 diff --git a/docs/zh/14-reference/12-config/index.md b/docs/zh/14-reference/12-config/index.md index e5efd77f8050ae2305a02a5bd70367d259cb971c..fe23684fde2501ea60cccba589527fdf590d3cef 100644 --- a/docs/zh/14-reference/12-config/index.md +++ b/docs/zh/14-reference/12-config/index.md @@ -626,6 +626,15 @@ charset 的有效值是 UTF-8。 | 缺省值 | 1 | | 补充说明 | 不同的启动方式，生成 core 文件的目录如下：1、systemctl start taosd 启动：生成的 core 在根目录下
2、手动启动，就在 taosd 执行目录下。 | +### enableScience + +| 属性 | 说明 | +| -------- | ------------------------------------------------------------------------------------------------------------------------------------------ | +| 适用范围 | 仅客户端 TAOS-CLI 适用 | +| 含义 | 是否开启科学计数法显示浮点数 | +| 取值范围 | 0：否，1：是 | +| 缺省值 | 0 | + ### udf | 属性 | 说明 | diff --git a/docs/zh/14-reference/14-taosKeeper.md b/docs/zh/14-reference/14-taosKeeper.md index 12b609584a65d729c93e256b3918bd04d03e2010..b4d35fb2400be9407e32182c3d632a52538b95bd 100644 --- a/docs/zh/14-reference/14-taosKeeper.md +++ b/docs/zh/14-reference/14-taosKeeper.md @@ -111,7 +111,7 @@ Active: inactive (dead) #### 配置文件启动 -执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/keeper.toml` 配置，否则将使用默认配置。 +执行以下命令即可快速体验 taosKeeper。当不指定 taosKeeper 配置文件时，优先使用 `/etc/taos/taoskeeper.toml` 配置，否则将使用默认配置。 ```shell $ taoskeeper -c @@ -156,6 +156,10 @@ database = "log" # 指定需要监控的普通表 tables = [] + +# database options for db storing metrics data +[metrics.databaseoptions] +cachemodel = "none" ``` ### 获取监控指标 @@ -206,7 +210,7 @@ taos_cluster_info_dnodes_total{cluster_id="5981392874047724755"} 1 taos_cluster_info_first_ep{cluster_id="5981392874047724755",value="hlb:6030"} 1 ``` -### check_health +### check\_health ``` $ curl -i http://127.0.0.1:6043/check_health @@ -222,3 +226,29 @@ Content-Length: 19 {"version":"1.0.0"} ``` + +### 集成 Prometheus + +taoskeeper 提供了 `/metrics` 接口，返回了 Prometheus 格式的监控数据，Prometheus 可以从 taoskeeper 抽取监控数据，实现通过 Prometheus 监控 TDengine 的目的。 + +#### 抽取配置 + +Prometheus 提供了 `scrape_configs` 配置如何从 endpoint 抽取监控数据，通常只需要修改 `static_configs` 中的 targets 配置为 taoskeeper 的 endpoint 地址，更多配置信息请参考 [Prometheus 配置文档](https://prometheus.io/docs/prometheus/latest/configuration/configuration/#scrape_config)。 + +``` +# A scrape configuration containing exactly one endpoint to scrape: +# Here it's Prometheus itself. +scrape_configs: + - job_name: "taoskeeper" + # metrics_path defaults to '/metrics' + # scheme defaults to 'http'. + static_configs: + - targets: ["localhost:6043"] +``` + +#### Dashboard + +我们提供了 `TaosKeeper Prometheus Dashboard for 3.x` dashboard，提供了和 TDinsight 类似的监控 dashboard。 + +在 Grafana Dashboard 菜单点击 `import`，dashboard ID 填写 `18587`，点击 `Load` 按钮即可导入 `TaosKeeper Prometheus Dashboard for 3.x` dashboard。 + diff --git a/docs/zh/17-operation/10-monitor.md b/docs/zh/17-operation/10-monitor.md index e936f35dcac544ad94035b5e5c9716c4aa50562e..7def90916c9ba5b75c4d1e1be8a2836a5251eec6 100644 --- a/docs/zh/17-operation/10-monitor.md +++ b/docs/zh/17-operation/10-monitor.md @@ -38,3 +38,303 @@ chmod +x TDinsight.sh 运行程序并重启 Grafana 服务，打开面板：`http://localhost:3000/d/tdinsight`。 更多使用场景和限制请参考[TDinsight](/reference/tdinsight/) 文档。 + +## log 库 + +TDinsight dashboard 数据来源于 log 库（存放监控数据的默认db，可以在 taoskeeper 配置文件中修改，具体参考 [taoskeeper 文档](/reference/taosKeeper)）。taoskeeper 启动后会自动创建 log 库，并将监控数据写入到该数据库中。 + +### cluster\_info 表 + +`cluster_info` 表记录集群信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|first\_ep|VARCHAR||集群 first ep| +|first\_ep\_dnode\_id|INT||集群 first ep 的 dnode id| +|version|VARCHAR||tdengine version。例如：3.0.4.0| +|master\_uptime|FLOAT||当前 master 节点的uptime。单位：天| +|monitor\_interval|INT||monitor interval。单位：秒| +|dbs\_total|INT||database 总数| +|tbs\_total|BIGINT||当前集群 table 总数| +|stbs\_total|INT||当前集群 stable 总数| +|dnodes\_total|INT||当前集群 dnode 总数| +|dnodes\_alive|INT||当前集群 dnode 存活总数| +|mnodes\_total|INT||当前集群 mnode 总数| +|mnodes\_alive|INT||当前集群 mnode 存活总数| +|vgroups\_total|INT||当前集群 vgroup 总数| +|vgroups\_alive|INT||当前集群 vgroup 存活总数| +|vnodes\_total|INT||当前集群 vnode 总数| +|vnodes\_alive|INT||当前集群 vnode 存活总数| +|connections\_total|INT||当前集群连接总数| +|topics\_total|INT||当前集群 topic 总数| +|streams\_total|INT||当前集群 stream 总数| +|protocol|INT||协议版本，目前为 1| +|cluster\_id|NCHAR|TAG|cluster id| + +### d\_info 表 + +`d_info` 表记录 dnode 状态信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|status|VARCHAR||dnode 状态| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### m\_info 表 + +`m_info` 表记录 mnode 角色信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|role|VARCHAR||mnode 角色， leader 或 follower| +|mnode\_id|INT|TAG|master node id| +|mnode\_ep|NCHAR|TAG|master node endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### dnodes\_info 表 + +`dnodes_info` 记录 dnode 信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|uptime|FLOAT||dnode uptime| +|cpu\_engine|FLOAT||taosd cpu 使用率，从 `/proc//stat` 读取| +|cpu\_system|FLOAT||服务器 cpu 使用率，从 `/proc/stat` 读取| +|cpu\_cores|FLOAT||服务器 cpu 核数| +|mem\_engine|INT||taosd 内存使用率，从 `/proc//status` 读取| +|mem\_system|INT||服务器可用内存| +|mem\_total|INT||服务器内存总量，单位 KB| +|disk\_engine|INT||| +|disk\_used|BIGINT||data dir 挂载的磁盘使用量，单位 bytes| +|disk\_total|BIGINT||data dir 挂载的磁盘总容量，单位 bytes| +|net\_in|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 received bytes。单位 kb/s| +|net\_out|FLOAT||网络吞吐率，从 `/proc/net/dev` 中读取的 transmit bytes。单位 kb/s| +|io\_read|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 rchar 与上次数值计算之后，计算得到速度。单位 kb/s| +|io\_write|FLOAT||io 吞吐率，从 `/proc//io` 中读取的 wchar 与上次数值计算之后，计算得到速度。单位 kb/s| +|io\_read\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 read_bytes。单位 kb/s| +|io\_write\_disk|FLOAT||磁盘 io 吞吐率，从 `/proc//io` 中读取的 write_bytes。单位 kb/s| +|req\_select|INT||两个间隔内发生的查询请求数目| +|req\_select\_rate|FLOAT||两个间隔内的查询请求速度 = `req_select / monitorInterval`| +|req\_insert|INT||两个间隔内发生的写入请求，包含的单条数据数目| +|req\_insert\_success|INT||两个间隔内发生的处理成功的写入请求，包含的单条数据数目| +|req\_insert\_rate|FLOAT||两个间隔内的单条数据写入速度 = `req_insert / monitorInterval`| +|req\_insert\_batch|INT||两个间隔内发生的写入请求数目| +|req\_insert\_batch\_success|INT||两个间隔内发生的成功的批量写入请求数目| +|req\_insert\_batch\_rate|FLOAT||两个间隔内的写入请求数目的速度 = `req_insert_batch / monitorInterval`| +|errors|INT||两个间隔内的出错的写入请求的数目| +|vnodes\_num|INT||dnode 上 vnodes 数量| +|masters|INT||dnode 上 master node 数量| +|has\_mnode|INT||dnode 是否包含 mnode| +|has\_qnode|INT||dnode 是否包含 qnode| +|has\_snode|INT||dnode 是否包含 snode| +|has\_bnode|INT||dnode 是否包含 bnode| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### data\_dir 表 + +`data_dir` 表记录 data 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||data 目录，一般为 `/var/lib/taos`| +|level|INT||0、1、2 多级存储级别| +|avail|BIGINT||data 目录可用空间| +|used|BIGINT||data 目录已使用空间| +|total|BIGINT||data 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_dir 表 + +`log_dir` 表记录 log 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||log 目录名，一般为 `/var/log/taos/`| +|avail|BIGINT||log 目录可用空间| +|used|BIGINT||log 目录已使用空间| +|total|BIGINT||log 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### temp\_dir 表 + +`temp_dir` 表记录 temp 目录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|name|NCHAR||temp 目录名，一般为 `/tmp/`| +|avail|BIGINT||temp 目录可用空间| +|used|BIGINT||temp 目录已使用空间| +|total|BIGINT||temp 目录空间| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vgroups\_info 表 + +`vgroups_info` 表记录虚拟节点组信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vgroup\_id|INT||vgroup id| +|database\_name|VARCHAR||vgroup 所属的 database 名字| +|tables\_num|BIGINT||vgroup 中 table 数量| +|status|VARCHAR||vgroup 状态| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### vnodes\_role 表 + +`vnodes_role` 表记录虚拟节点角色信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|vnode\_role|VARCHAR||vnode 角色，leader 或 follower| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### logs 表 + +`logs` 表记录登录信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|level|VARCHAR||log level| +|content|NCHAR||log content，长度不超过1024字节| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### log\_summary 表 + +`log_summary` 记录日志统计信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|error|INT||error 总数| +|info|INT||info 总数| +|debug|INT||debug 总数| +|trace|INT||trace 总数| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### grants\_info 表 + +`grants_info` 记录授权信息。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|expire\_time|BIGINT||认证过期时间，企业版有效，社区版为 bigint 最大值| +|timeseries\_used|BIGINT||已用测点数| +|timeseries\_total|BIGINT||总测点数，开源版本为 bigint 最大值| +|dnode\_id|INT|TAG|dnode id| +|dnode\_ep|NCHAR|TAG|dnode endpoint| +|cluster\_id|NCHAR|TAG|cluster id| + +### keeper\_monitor 表 + +`keeper_monitor` 记录 taoskeeper 监控数据。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|ts|TIMESTAMP||timestamp| +|cpu|FLOAT||cpu 使用率| +|mem|FLOAT||内存使用率| +|identify|NCHAR|TAG|| + +### taosadapter\_restful\_http\_request\_total 表 + +`taosadapter_restful_http_request_total` 记录 taosadapter rest 请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_fail 表 + +`taosadapter_restful_http_request_fail` 记录 taosadapter rest 请求失败信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|client\_ip|NCHAR|TAG|client ip| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| +|status\_code|NCHAR|TAG|status code| + +### taosadapter\_restful\_http\_request\_in\_flight 表 + +`taosadapter_restful_http_request_in_flight` 记录 taosadapter rest 实时请求信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_restful\_http\_request\_summary\_milliseconds 表 + +`taosadapter_restful_http_request_summary_milliseconds` 记录 taosadapter rest 请求汇总信息，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|count|DOUBLE||| +|sum|DOUBLE||| +|0.5|DOUBLE||| +|0.9|DOUBLE||| +|0.99|DOUBLE||| +|0.1|DOUBLE||| +|0.2|DOUBLE||| +|endpoint|NCHAR|TAG|taosadpater endpoint| +|request\_method|NCHAR|TAG|request method| +|request\_uri|NCHAR|TAG|request uri| + +### taosadapter\_system\_mem\_percent 表 + +`taosadapter_system_mem_percent` 表记录 taosadapter 内存使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| + +### taosadapter\_system\_cpu\_percent 表 + +`taosadapter_system_cpu_percent` 表记录 taosadapter cpu 使用情况，该表为 schemaless 方式创建的表，时间戳字段名为 `_ts`。 + +|field|type|is\_tag|comment| +|:----|:---|:-----|:------| +|\_ts|TIMESTAMP||timestamp| +|guage|DOUBLE||监控指标值| +|endpoint|NCHAR|TAG|taosadpater endpoint| diff --git a/docs/zh/28-releases/01-tdengine.md b/docs/zh/28-releases/01-tdengine.md index b4441ad078059f51d8632e0db9c8007db9eb804e..0974289c1fa5a1e220589af3a5d43e237b47acb6 100644 --- a/docs/zh/28-releases/01-tdengine.md +++ b/docs/zh/28-releases/01-tdengine.md @@ -10,6 +10,10 @@ TDengine 2.x 各版本安装包请访问[这里](https://www.taosdata.com/all-do import Release from "/components/ReleaseV3"; +## 3.0.4.0 + + + ## 3.0.3.2 diff --git a/docs/zh/28-releases/02-tools.md b/docs/zh/28-releases/02-tools.md index cce6834f12af02568632949bde044b81ea718808..78926555f1f4e1d579e8b2593575c84f67d18f6a 100644 --- a/docs/zh/28-releases/02-tools.md +++ b/docs/zh/28-releases/02-tools.md @@ -10,6 +10,10 @@ taosTools 各版本安装包下载链接如下： import Release from "/components/ReleaseV3"; +## 2.4.12 + + + ## 2.4.11 diff --git a/examples/c/tmq.c b/examples/c/tmq.c index 83b40c4f20a332b94abe45dd5c4fb7afd820861a..94545dfaad943a3ec5e35ae6859213835d8c4540 100644 --- a/examples/c/tmq.c +++ b/examples/c/tmq.c @@ -20,7 +20,8 @@ #include #include "taos.h" -static int running = 1; +static int running = 1; +const char* topic_name = "topicname"; static int32_t msg_process(TAOS_RES* msg) { char buf[1024]; @@ -243,7 +244,7 @@ _end: tmq_list_t* build_topic_list() { tmq_list_t* topicList = tmq_list_new(); - int32_t code = tmq_list_append(topicList, "topicname"); + int32_t code = tmq_list_append(topicList, topic_name); if (code) { tmq_list_destroy(topicList); return NULL; @@ -269,6 +270,31 @@ void basic_consume_loop(tmq_t* tmq) { fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); } +void consume_repeatly(tmq_t* tmq) { + int32_t numOfAssignment = 0; + tmq_topic_assignment* pAssign = NULL; + + int32_t code = tmq_get_topic_assignment(tmq, topic_name, &pAssign, &numOfAssignment); + if (code != 0) { + fprintf(stderr, "failed to get assignment, reason:%s", tmq_err2str(code)); + } + + // seek to the earliest offset + for(int32_t i = 0; i < numOfAssignment; ++i) { + tmq_topic_assignment* p = &pAssign[i]; + + code = tmq_offset_seek(tmq, topic_name, p->vgId, p->begin); + if (code != 0) { + fprintf(stderr, "failed to seek to %ld, reason:%s", p->begin, tmq_err2str(code)); + } + } + + free(pAssign); + + // let's do it again + basic_consume_loop(tmq); +} + int main(int argc, char* argv[]) { int32_t code; @@ -294,10 +320,13 @@ int main(int argc, char* argv[]) { if ((code = tmq_subscribe(tmq, topic_list))) { fprintf(stderr, "Failed to tmq_subscribe(): %s\n", tmq_err2str(code)); } + tmq_list_destroy(topic_list); basic_consume_loop(tmq); + consume_repeatly(tmq); + code = tmq_consumer_close(tmq); if (code) { fprintf(stderr, "Failed to close consumer: %s\n", tmq_err2str(code)); diff --git a/include/client/taos.h b/include/client/taos.h index cf410a42daf1e9c401af767497a603aa12c7a536..44703c2797d3e39bbc05f817a0cfd65acd6af2cf 100644 --- a/include/client/taos.h +++ b/include/client/taos.h @@ -101,6 +101,7 @@ typedef struct TAOS_FIELD_E { #endif typedef void (*__taos_async_fn_t)(void *param, TAOS_RES *res, int code); +typedef void (*__taos_notify_fn_t)(void *param, void *ext, int type); typedef struct TAOS_MULTI_BIND { int buffer_type; @@ -121,6 +122,10 @@ typedef enum { SET_CONF_RET_ERR_TOO_LONG = -6 } SET_CONF_RET_CODE; +typedef enum { + TAOS_NOTIFY_PASSVER = 0, +} TAOS_NOTIFY_TYPE; + #define RET_MSG_LENGTH 1024 typedef struct setConfRet { SET_CONF_RET_CODE retCode; @@ -225,6 +230,8 @@ DLL_EXPORT int taos_get_tables_vgId(TAOS *taos, const char *db, const char *tabl DLL_EXPORT int taos_load_table_info(TAOS *taos, const char *tableNameList); +DLL_EXPORT int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type); + /* --------------------------schemaless INTERFACE------------------------------- */ DLL_EXPORT TAOS_RES *taos_schemaless_insert(TAOS *taos, char *lines[], int numLines, int protocol, int precision); @@ -262,6 +269,12 @@ DLL_EXPORT tmq_t *tmq_consumer_new(tmq_conf_t *conf, char *errstr, int32_t errst DLL_EXPORT const char *tmq_err2str(int32_t code); /* ------------------------TMQ CONSUMER INTERFACE------------------------ */ +typedef struct tmq_topic_assignment { + int32_t vgId; + int64_t currentOffset; + int64_t begin; + int64_t end; +} tmq_topic_assignment; DLL_EXPORT int32_t tmq_subscribe(tmq_t *tmq, const tmq_list_t *topic_list); DLL_EXPORT int32_t tmq_unsubscribe(tmq_t *tmq); @@ -270,6 +283,8 @@ DLL_EXPORT TAOS_RES *tmq_consumer_poll(tmq_t *tmq, int64_t timeout); DLL_EXPORT int32_t tmq_consumer_close(tmq_t *tmq); DLL_EXPORT int32_t tmq_commit_sync(tmq_t *tmq, const TAOS_RES *msg); DLL_EXPORT void tmq_commit_async(tmq_t *tmq, const TAOS_RES *msg, tmq_commit_cb *cb, void *param); +DLL_EXPORT int32_t tmq_get_topic_assignment(tmq_t *tmq, const char* pTopicName, tmq_topic_assignment **assignment, int32_t *numOfAssignment); +DLL_EXPORT int32_t tmq_offset_seek(tmq_t *tmq, const char* pTopicName, int32_t vgId, int64_t offset); /* ----------------------TMQ CONFIGURATION INTERFACE---------------------- */ diff --git a/include/common/tcommon.h b/include/common/tcommon.h index 51a714c7921c4d3d3bebddc8e5a76b6f075bcba4..3d0dac7c3de095a60cb88c66bb82f70d66d2afdc 100644 --- a/include/common/tcommon.h +++ b/include/common/tcommon.h @@ -128,6 +128,7 @@ enum { TMQ_MSG_TYPE__POLL_META_RSP, TMQ_MSG_TYPE__EP_RSP, TMQ_MSG_TYPE__TAOSX_RSP, + TMQ_MSG_TYPE__WALINFO_RSP, TMQ_MSG_TYPE__END_RSP, }; @@ -208,15 +209,12 @@ typedef struct SSDataBlock { } SSDataBlock; enum { - FETCH_TYPE__DATA = 1, - FETCH_TYPE__META, - FETCH_TYPE__SEP, + FETCH_TYPE__DATA = 0, FETCH_TYPE__NONE, }; typedef struct { int8_t fetchType; - STqOffsetVal offset; union { SSDataBlock data; void* meta; diff --git a/include/common/tglobal.h b/include/common/tglobal.h index 13e8454ac32f9d14ac0fa09aa9f7c3caf5f931ef..25a386b23dd2183c2af2161cc713a912046adda8 100644 --- a/include/common/tglobal.h +++ b/include/common/tglobal.h @@ -24,6 +24,12 @@ extern "C" { #endif +#define SLOW_LOG_TYPE_QUERY 0x1 +#define SLOW_LOG_TYPE_INSERT 0x2 +#define SLOW_LOG_TYPE_OTHERS 0x4 +#define SLOW_LOG_TYPE_ALL 0xFFFFFFFF + + // cluster extern char tsFirst[]; extern char tsSecond[]; @@ -104,6 +110,7 @@ extern int32_t tsCacheLazyLoadThreshold; // cost threshold for last/last_row lo // query client extern int32_t tsQueryPolicy; extern int32_t tsQueryRspPolicy; +extern int64_t tsQueryMaxConcurrentTables; extern int32_t tsQuerySmaOptimize; extern int32_t tsQueryRsmaTolerance; extern bool tsQueryPlannerTrace; @@ -111,11 +118,14 @@ extern int32_t tsQueryNodeChunkSize; extern bool tsQueryUseNodeAllocator; extern bool tsKeepColumnName; extern bool tsEnableQueryHb; +extern bool tsEnableScience; extern int32_t tsRedirectPeriod; extern int32_t tsRedirectFactor; extern int32_t tsRedirectMaxPeriod; extern int32_t tsMaxRetryWaitTime; extern bool tsUseAdapter; +extern int32_t tsSlowLogThreshold; +extern int32_t tsSlowLogScope; // client extern int32_t tsMinSlidingTime; diff --git a/include/common/tmsg.h b/include/common/tmsg.h index 999cc17767cbb6de851fad7b817fdcde70ddf146..f377ad0d63e140a8e5e4e7c46130b473d8bd684e 100644 --- a/include/common/tmsg.h +++ b/include/common/tmsg.h @@ -77,7 +77,7 @@ static inline bool tmsgIsValid(tmsg_t type) { } static inline bool vnodeIsMsgBlock(tmsg_t type) { return (type == TDMT_VND_CREATE_TABLE) || (type == TDMT_VND_ALTER_TABLE) || (type == TDMT_VND_DROP_TABLE) || - (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM); + (type == TDMT_VND_UPDATE_TAG_VAL) || (type == TDMT_VND_ALTER_CONFIRM) || (type == TDMT_VND_COMMIT); } static inline bool syncUtilUserCommit(tmsg_t msgType) { @@ -106,6 +106,7 @@ enum { HEARTBEAT_KEY_DBINFO, HEARTBEAT_KEY_STBINFO, HEARTBEAT_KEY_TMQ, + HEARTBEAT_KEY_USER_PASSINFO, }; typedef enum _mgmt_table { @@ -177,6 +178,12 @@ typedef enum _mgmt_table { #define TSDB_ALTER_USER_SYSINFO 0xA #define TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC 0xB #define TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC 0xC +#define TSDB_ALTER_USER_ADD_READ_TABLE 0xD +#define TSDB_ALTER_USER_REMOVE_READ_TABLE 0xE +#define TSDB_ALTER_USER_ADD_WRITE_TABLE 0xF +#define TSDB_ALTER_USER_REMOVE_WRITE_TABLE 0x10 +#define TSDB_ALTER_USER_ADD_ALL_TABLE 0x11 +#define TSDB_ALTER_USER_REMOVE_ALL_TABLE 0x12 #define TSDB_ALTER_USER_PRIVILEGES 0x2 @@ -628,6 +635,7 @@ typedef struct { int8_t connType; SEpSet epSet; int32_t svrTimestamp; + int32_t passVer; char sVer[TSDB_VERSION_LEN]; char sDetailVer[128]; } SConnectRsp; @@ -669,13 +677,16 @@ int32_t tSerializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq int32_t tDeserializeSCreateUserReq(void* buf, int32_t bufLen, SCreateUserReq* pReq); typedef struct { - int8_t alterType; - int8_t superUser; - int8_t sysInfo; - int8_t enable; - char user[TSDB_USER_LEN]; - char pass[TSDB_USET_PASSWORD_LEN]; - char objname[TSDB_DB_FNAME_LEN]; // db or topic + int8_t alterType; + int8_t superUser; + int8_t sysInfo; + int8_t enable; + char user[TSDB_USER_LEN]; + char pass[TSDB_USET_PASSWORD_LEN]; + char objname[TSDB_DB_FNAME_LEN]; // db or topic + char tabName[TSDB_TABLE_NAME_LEN]; + char* tagCond; + int32_t tagCondLen; } SAlterUserReq; int32_t tSerializeSAlterUserReq(void* buf, int32_t bufLen, SAlterUserReq* pReq); @@ -698,12 +709,23 @@ typedef struct { SHashObj* createdDbs; SHashObj* readDbs; SHashObj* writeDbs; + SHashObj* readTbs; + SHashObj* writeTbs; + SHashObj* useDbs; } SGetUserAuthRsp; int32_t tSerializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); int32_t tDeserializeSGetUserAuthRsp(void* buf, int32_t bufLen, SGetUserAuthRsp* pRsp); void tFreeSGetUserAuthRsp(SGetUserAuthRsp* pRsp); +typedef struct SUserPassVersion { + char user[TSDB_USER_LEN]; + int32_t version; +} SUserPassVersion; + +typedef SGetUserAuthReq SGetUserPassReq; +typedef SUserPassVersion SGetUserPassRsp; + /* * for client side struct, only column id, type, bytes are necessary * But for data in vnode side, we need all the following information. @@ -1034,6 +1056,14 @@ int32_t tSerializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp int32_t tDeserializeSUserAuthBatchRsp(void* buf, int32_t bufLen, SUserAuthBatchRsp* pRsp); void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp* pRsp); +typedef struct { + SArray* pArray; // Array of SGetUserPassRsp +} SUserPassBatchRsp; + +int32_t tSerializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); +int32_t tDeserializeSUserPassBatchRsp(void* buf, int32_t bufLen, SUserPassBatchRsp* pRsp); +void tFreeSUserPassBatchRsp(SUserPassBatchRsp* pRsp); + typedef struct { char db[TSDB_DB_FNAME_LEN]; STimeWindow timeRange; @@ -1093,9 +1123,15 @@ typedef struct { char* pCode; } SFuncInfo; +typedef struct { + int32_t funcVersion; + int64_t funcCreatedTime; +} SFuncExtraInfo; + typedef struct { int32_t numOfFuncs; SArray* pFuncInfos; + SArray* pFuncExtraInfos; } SRetrieveFuncRsp; int32_t tSerializeSRetrieveFuncRsp(void* buf, int32_t bufLen, SRetrieveFuncRsp* pRsp); @@ -1268,6 +1304,9 @@ typedef struct { int16_t hashSuffix; int32_t tsdbPageSize; int64_t reserved[8]; + int8_t learnerReplica; + int8_t learnerSelfIndex; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; } SCreateVnodeReq; int32_t tSerializeSCreateVnodeReq(void* buf, int32_t bufLen, SCreateVnodeReq* pReq); @@ -1339,7 +1378,10 @@ typedef struct { int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; int64_t reserved[8]; -} SAlterVnodeReplicaReq; + int8_t learnerSelfIndex; + int8_t learnerReplica; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; +} SAlterVnodeReplicaReq, SAlterVnodeTypeReq; int32_t tSerializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); int32_t tDeserializeSAlterVnodeReplicaReq(void* buf, int32_t bufLen, SAlterVnodeReplicaReq* pReq); @@ -1611,7 +1653,10 @@ int32_t tDeserializeSCreateDropMQSNodeReq(void* buf, int32_t bufLen, SMCreateQno typedef struct { int8_t replica; SReplica replicas[TSDB_MAX_REPLICA]; -} SDCreateMnodeReq, SDAlterMnodeReq; + int8_t learnerReplica; + SReplica learnerReplicas[TSDB_MAX_LEARNER_REPLICA]; + int64_t lastIndex; +} SDCreateMnodeReq, SDAlterMnodeReq, SDAlterMnodeTypeReq; int32_t tSerializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); int32_t tDeserializeSDCreateMnodeReq(void* buf, int32_t bufLen, SDCreateMnodeReq* pReq); @@ -2787,6 +2832,7 @@ typedef struct { } SMqOffset; typedef struct { + int64_t consumerId; int32_t num; SMqOffset* offsets; } SMqCMCommitOffsetReq; @@ -2854,6 +2900,14 @@ typedef struct { int32_t tEncodeSTqOffset(SEncoder* pEncoder, const STqOffset* pOffset); int32_t tDecodeSTqOffset(SDecoder* pDecoder, STqOffset* pOffset); +typedef struct SMqVgOffset { + int64_t consumerId; + STqOffset offset; +} SMqVgOffset; + +int32_t tEncodeMqVgOffset(SEncoder* pEncoder, const SMqVgOffset* pOffset); +int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset); + typedef struct { char name[TSDB_TABLE_FNAME_LEN]; char stb[TSDB_TABLE_FNAME_LEN]; @@ -3083,18 +3137,19 @@ typedef struct { int32_t code; int32_t epoch; int64_t consumerId; + int64_t walsver; + int64_t walever; } SMqRspHead; typedef struct { - SMsgHead head; - char subKey[TSDB_SUBSCRIBE_KEY_LEN]; - int8_t withTbName; - int8_t useSnapshot; - int32_t epoch; - uint64_t reqId; - int64_t consumerId; - int64_t timeout; - // int64_t currentOffset; + SMsgHead head; + char subKey[TSDB_SUBSCRIBE_KEY_LEN]; + int8_t withTbName; + int8_t useSnapshot; + int32_t epoch; + uint64_t reqId; + int64_t consumerId; + int64_t timeout; STqOffsetVal reqOffset; } SMqPollReq; @@ -3129,43 +3184,9 @@ typedef struct { SSchemaWrapper schema; } SMqSubTopicEp; -static FORCE_INLINE int32_t tEncodeSMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp) { - int32_t tlen = 0; - tlen += taosEncodeString(buf, pTopicEp->topic); - tlen += taosEncodeString(buf, pTopicEp->db); - int32_t sz = taosArrayGetSize(pTopicEp->vgs); - tlen += taosEncodeFixedI32(buf, sz); - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp* pVgEp = (SMqSubVgEp*)taosArrayGet(pTopicEp->vgs, i); - tlen += tEncodeSMqSubVgEp(buf, pVgEp); - } - tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); - return tlen; -} - -static FORCE_INLINE void* tDecodeSMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp) { - buf = taosDecodeStringTo(buf, pTopicEp->topic); - buf = taosDecodeStringTo(buf, pTopicEp->db); - int32_t sz; - buf = taosDecodeFixedI32(buf, &sz); - pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); - if (pTopicEp->vgs == NULL) { - return NULL; - } - for (int32_t i = 0; i < sz; i++) { - SMqSubVgEp vgEp; - buf = tDecodeSMqSubVgEp(buf, &vgEp); - taosArrayPush(pTopicEp->vgs, &vgEp); - } - buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); - return buf; -} - -static FORCE_INLINE void tDeleteSMqSubTopicEp(SMqSubTopicEp* pSubTopicEp) { - taosMemoryFreeClear(pSubTopicEp->schema.pSchema); - pSubTopicEp->schema.nCols = 0; - taosArrayDestroy(pSubTopicEp->vgs); -} +int32_t tEncodeMqSubTopicEp(void** buf, const SMqSubTopicEp* pTopicEp); +void* tDecodeMqSubTopicEp(void* buf, SMqSubTopicEp* pTopicEp); +void tDeleteMqSubTopicEp(SMqSubTopicEp* pSubTopicEp); typedef struct { SMqRspHead head; @@ -3175,8 +3196,8 @@ typedef struct { void* metaRsp; } SMqMetaRsp; -int32_t tEncodeSMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp); -int32_t tDecodeSMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); +int32_t tEncodeMqMetaRsp(SEncoder* pEncoder, const SMqMetaRsp* pRsp); +int32_t tDecodeMqMetaRsp(SDecoder* pDecoder, SMqMetaRsp* pRsp); typedef struct { SMqRspHead head; @@ -3191,9 +3212,9 @@ typedef struct { SArray* blockSchema; } SMqDataRsp; -int32_t tEncodeSMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); -int32_t tDecodeSMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); -void tDeleteSMqDataRsp(SMqDataRsp* pRsp); +int32_t tEncodeMqDataRsp(SEncoder* pEncoder, const SMqDataRsp* pRsp); +int32_t tDecodeMqDataRsp(SDecoder* pDecoder, SMqDataRsp* pRsp); +void tDeleteMqDataRsp(SMqDataRsp* pRsp); typedef struct { SMqRspHead head; @@ -3229,7 +3250,7 @@ static FORCE_INLINE int32_t tEncodeSMqAskEpRsp(void** buf, const SMqAskEpRsp* pR tlen += taosEncodeFixedI32(buf, sz); for (int32_t i = 0; i < sz; i++) { SMqSubTopicEp* pVgEp = (SMqSubTopicEp*)taosArrayGet(pRsp->topics, i); - tlen += tEncodeSMqSubTopicEp(buf, pVgEp); + tlen += tEncodeMqSubTopicEp(buf, pVgEp); } return tlen; } @@ -3244,14 +3265,14 @@ static FORCE_INLINE void* tDecodeSMqAskEpRsp(void* buf, SMqAskEpRsp* pRsp) { } for (int32_t i = 0; i < sz; i++) { SMqSubTopicEp topicEp; - buf = tDecodeSMqSubTopicEp(buf, &topicEp); + buf = tDecodeMqSubTopicEp(buf, &topicEp); taosArrayPush(pRsp->topics, &topicEp); } return buf; } static FORCE_INLINE void tDeleteSMqAskEpRsp(SMqAskEpRsp* pRsp) { - taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteSMqSubTopicEp); + taosArrayDestroyEx(pRsp->topics, (FDelete)tDeleteMqSubTopicEp); } #define TD_AUTO_CREATE_TABLE 0x1 diff --git a/include/common/tmsgdef.h b/include/common/tmsgdef.h index 6cf6140815cf1663392950b0ee119b0ae65db126..ffad6f66467fbdc90dcca4185a34d6a29d83e74b 100644 --- a/include/common/tmsgdef.h +++ b/include/common/tmsgdef.h @@ -83,6 +83,8 @@ enum { TD_DEF_MSG_TYPE(TDMT_DND_CONFIG_DNODE, "config-dnode", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_SYSTABLE_RETRIEVE, "dnode-retrieve", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_DND_MAX_MSG, "dnd-max", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_MNODE_TYPE, "dnode-alter-mnode-type", NULL, NULL) + TD_DEF_MSG_TYPE(TDMT_DND_ALTER_VNODE_TYPE, "dnode-alter-vnode-type", NULL, NULL) TD_NEW_MSG_SEG(TDMT_MND_MSG) TD_DEF_MSG_TYPE(TDMT_MND_CONNECT, "connect", NULL, NULL) @@ -300,9 +302,11 @@ enum { TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SUBSCRIBE, "vnode-tmq-subscribe", SMqRebVgReq, SMqRebVgRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DELETE_SUB, "vnode-tmq-delete-sub", SMqVDeleteReq, SMqVDeleteRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_COMMIT_OFFSET, "vnode-tmq-commit-offset", STqOffset, STqOffset) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_SEEK_TO_OFFSET, "vnode-tmq-seekto-offset", STqOffset, STqOffset) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_ADD_CHECKINFO, "vnode-tmq-add-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_DEL_CHECKINFO, "vnode-del-checkinfo", NULL, NULL) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_CONSUME, "vnode-tmq-consume", SMqPollReq, SMqDataBlkRsp) + TD_DEF_MSG_TYPE(TDMT_VND_TMQ_VG_WALINFO, "vnode-tmq-vg-walinfo", SMqPollReq, SMqDataBlkRsp) TD_DEF_MSG_TYPE(TDMT_VND_TMQ_MAX_MSG, "vnd-tmq-max", NULL, NULL) diff --git a/include/common/ttokendef.h b/include/common/ttokendef.h index fc66363603fb177b98b805bd84daefeaafaaff75..641cbbb58885259502fd71ecf57dbf4690069afb 100644 --- a/include/common/ttokendef.h +++ b/include/common/ttokendef.h @@ -63,55 +63,55 @@ #define TK_READ 45 #define TK_WRITE 46 #define TK_NK_DOT 47 -#define TK_DNODE 48 -#define TK_PORT 49 -#define TK_DNODES 50 -#define TK_NK_IPTOKEN 51 -#define TK_FORCE 52 -#define TK_LOCAL 53 -#define TK_QNODE 54 -#define TK_BNODE 55 -#define TK_SNODE 56 -#define TK_MNODE 57 -#define TK_DATABASE 58 -#define TK_USE 59 -#define TK_FLUSH 60 -#define TK_TRIM 61 -#define TK_COMPACT 62 -#define TK_IF 63 -#define TK_NOT 64 -#define TK_EXISTS 65 -#define TK_BUFFER 66 -#define TK_CACHEMODEL 67 -#define TK_CACHESIZE 68 -#define TK_COMP 69 -#define TK_DURATION 70 -#define TK_NK_VARIABLE 71 -#define TK_MAXROWS 72 -#define TK_MINROWS 73 -#define TK_KEEP 74 -#define TK_PAGES 75 -#define TK_PAGESIZE 76 -#define TK_TSDB_PAGESIZE 77 -#define TK_PRECISION 78 -#define TK_REPLICA 79 -#define TK_VGROUPS 80 -#define TK_SINGLE_STABLE 81 -#define TK_RETENTIONS 82 -#define TK_SCHEMALESS 83 -#define TK_WAL_LEVEL 84 -#define TK_WAL_FSYNC_PERIOD 85 -#define TK_WAL_RETENTION_PERIOD 86 -#define TK_WAL_RETENTION_SIZE 87 -#define TK_WAL_ROLL_PERIOD 88 -#define TK_WAL_SEGMENT_SIZE 89 -#define TK_STT_TRIGGER 90 -#define TK_TABLE_PREFIX 91 -#define TK_TABLE_SUFFIX 92 -#define TK_NK_COLON 93 -#define TK_MAX_SPEED 94 -#define TK_START 95 -#define TK_WITH 96 +#define TK_WITH 48 +#define TK_DNODE 49 +#define TK_PORT 50 +#define TK_DNODES 51 +#define TK_NK_IPTOKEN 52 +#define TK_FORCE 53 +#define TK_LOCAL 54 +#define TK_QNODE 55 +#define TK_BNODE 56 +#define TK_SNODE 57 +#define TK_MNODE 58 +#define TK_DATABASE 59 +#define TK_USE 60 +#define TK_FLUSH 61 +#define TK_TRIM 62 +#define TK_COMPACT 63 +#define TK_IF 64 +#define TK_NOT 65 +#define TK_EXISTS 66 +#define TK_BUFFER 67 +#define TK_CACHEMODEL 68 +#define TK_CACHESIZE 69 +#define TK_COMP 70 +#define TK_DURATION 71 +#define TK_NK_VARIABLE 72 +#define TK_MAXROWS 73 +#define TK_MINROWS 74 +#define TK_KEEP 75 +#define TK_PAGES 76 +#define TK_PAGESIZE 77 +#define TK_TSDB_PAGESIZE 78 +#define TK_PRECISION 79 +#define TK_REPLICA 80 +#define TK_VGROUPS 81 +#define TK_SINGLE_STABLE 82 +#define TK_RETENTIONS 83 +#define TK_SCHEMALESS 84 +#define TK_WAL_LEVEL 85 +#define TK_WAL_FSYNC_PERIOD 86 +#define TK_WAL_RETENTION_PERIOD 87 +#define TK_WAL_RETENTION_SIZE 88 +#define TK_WAL_ROLL_PERIOD 89 +#define TK_WAL_SEGMENT_SIZE 90 +#define TK_STT_TRIGGER 91 +#define TK_TABLE_PREFIX 92 +#define TK_TABLE_SUFFIX 93 +#define TK_NK_COLON 94 +#define TK_MAX_SPEED 95 +#define TK_START 96 #define TK_TIMESTAMP 97 #define TK_END 98 #define TK_TABLE 99 @@ -127,24 +127,24 @@ #define TK_NK_EQ 109 #define TK_USING 110 #define TK_TAGS 111 -#define TK_COMMENT 112 -#define TK_BOOL 113 -#define TK_TINYINT 114 -#define TK_SMALLINT 115 -#define TK_INT 116 -#define TK_INTEGER 117 -#define TK_BIGINT 118 -#define TK_FLOAT 119 -#define TK_DOUBLE 120 -#define TK_BINARY 121 -#define TK_NCHAR 122 -#define TK_UNSIGNED 123 -#define TK_JSON 124 -#define TK_VARCHAR 125 -#define TK_MEDIUMBLOB 126 -#define TK_BLOB 127 -#define TK_VARBINARY 128 -#define TK_DECIMAL 129 +#define TK_BOOL 112 +#define TK_TINYINT 113 +#define TK_SMALLINT 114 +#define TK_INT 115 +#define TK_INTEGER 116 +#define TK_BIGINT 117 +#define TK_FLOAT 118 +#define TK_DOUBLE 119 +#define TK_BINARY 120 +#define TK_NCHAR 121 +#define TK_UNSIGNED 122 +#define TK_JSON 123 +#define TK_VARCHAR 124 +#define TK_MEDIUMBLOB 125 +#define TK_BLOB 126 +#define TK_VARBINARY 127 +#define TK_DECIMAL 128 +#define TK_COMMENT 129 #define TK_MAX_DELAY 130 #define TK_WATERMARK 131 #define TK_ROLLUP 132 diff --git a/include/dnode/mnode/mnode.h b/include/dnode/mnode/mnode.h index cdb1642a5c313767fe6a6d5ad53d6202f8c677a6..6c3c7497b1f5b123143e99283657f55a49f2eb63 100644 --- a/include/dnode/mnode/mnode.h +++ b/include/dnode/mnode/mnode.h @@ -20,6 +20,7 @@ #include "tmsg.h" #include "tmsgcb.h" #include "trpc.h" +#include "sync.h" #ifdef __cplusplus extern "C" { @@ -33,8 +34,11 @@ typedef struct { bool deploy; int8_t selfIndex; int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + int8_t numOfTotalReplicas; + SReplica replicas[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + int32_t nodeRoles[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; SMsgCb msgCb; + int64_t lastIndex; } SMnodeOpt; /* ------------------------ SMnode ------------------------ */ @@ -69,6 +73,9 @@ int32_t mndStart(SMnode *pMnode); */ void mndStop(SMnode *pMnode); +int32_t mndIsCatchUp(SMnode *pMnode); +ESyncRole mndGetRole(SMnode *pMnode); + /** * @brief Get mnode monitor info. * diff --git a/include/libs/catalog/catalog.h b/include/libs/catalog/catalog.h index fbb24d28623d99932966e625d6499ebbc836ac0a..2c684f8f76a30b9700956c470de49687daedaeb9 100644 --- a/include/libs/catalog/catalog.h +++ b/include/libs/catalog/catalog.h @@ -29,6 +29,7 @@ extern "C" { #include "tmsg.h" #include "tname.h" #include "transport.h" +#include "nodes.h" typedef struct SCatalog SCatalog; @@ -49,10 +50,15 @@ typedef enum { typedef struct SUserAuthInfo { char user[TSDB_USER_LEN]; - char dbFName[TSDB_DB_FNAME_LEN]; + SName tbName; AUTH_TYPE type; } SUserAuthInfo; +typedef struct SUserAuthRes { + bool pass; + SNode* pCond; +} SUserAuthRes; + typedef struct SDbInfo { int32_t vgVer; int32_t tbNum; @@ -96,7 +102,7 @@ typedef struct SMetaData { SArray* pTableIndex; // pRes = SArray* SArray* pUdfList; // pRes = SFuncInfo* SArray* pIndex; // pRes = SIndexInfo* - SArray* pUser; // pRes = bool* + SArray* pUser; // pRes = SUserAuthRes* SArray* pQnodeList; // pRes = SArray* SArray* pTableCfg; // pRes = STableCfg* SArray* pDnodeList; // pRes = SArray* @@ -312,11 +318,9 @@ int32_t catalogUpdateTableIndex(SCatalog* pCtg, STableIndexRsp* pRsp); int32_t catalogGetUdfInfo(SCatalog* pCtg, SRequestConnInfo* pConn, const char* funcName, SFuncInfo* pInfo); -int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass); +int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes); -int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, - bool* exists); +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists); int32_t catalogUpdateUserAuthInfo(SCatalog* pCtg, SGetUserAuthRsp* pAuth); diff --git a/include/libs/executor/dataSinkMgt.h b/include/libs/executor/dataSinkMgt.h index ed7cbc812504ae09099abd79020c6b9d761d6674..ce7d038d42c8f37bec730111d6d26d946c47a168 100644 --- a/include/libs/executor/dataSinkMgt.h +++ b/include/libs/executor/dataSinkMgt.h @@ -29,7 +29,6 @@ extern "C" { #define DS_BUF_FULL 2 #define DS_BUF_EMPTY 3 -struct SDataSink; struct SSDataBlock; typedef struct SDeleterRes { diff --git a/include/libs/executor/executor.h b/include/libs/executor/executor.h index 91b6fa51a2682296a8da3d935d7c34d098f6b1ab..34372dc2ff23bacf0e21ee459025112c28b813bb 100644 --- a/include/libs/executor/executor.h +++ b/include/libs/executor/executor.h @@ -26,6 +26,7 @@ extern "C" { typedef void* qTaskInfo_t; typedef void* DataSinkHandle; + struct SRpcMsg; struct SSubplan; @@ -91,6 +92,9 @@ void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId); int32_t qSetStreamOpOpen(qTaskInfo_t tinfo); +// todo refactor +void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId); + /** * Set multiple input data blocks for the stream scan. * @param tinfo @@ -119,7 +123,7 @@ int32_t qSetSMAInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, * @param isAdd * @return */ -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); +int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd); /** * Create the exec task object according to task json @@ -163,6 +167,7 @@ void qCleanExecTaskBlockBuf(qTaskInfo_t tinfo); * @return */ int32_t qAsyncKillTask(qTaskInfo_t tinfo, int32_t rspCode); + int32_t qKillTask(qTaskInfo_t tinfo, int32_t rspCode); bool qTaskIsExecuting(qTaskInfo_t qinfo); @@ -182,28 +187,18 @@ int32_t qSerializeTaskStatus(qTaskInfo_t tinfo, char** pOutput, int32_t* len); int32_t qDeserializeTaskStatus(qTaskInfo_t tinfo, const char* pInput, int32_t len); STimeWindow getAlignQueryTimeWindow(SInterval* pInterval, int32_t precision, int64_t key); -/** - * return the scan info, in the form of tuple of two items, including table uid and current timestamp - * @param tinfo - * @param uid - * @param ts - * @return - */ -int32_t qGetStreamScanStatus(qTaskInfo_t tinfo, uint64_t* uid, int64_t* ts); -int32_t qStreamPrepareTsdbScan(qTaskInfo_t tinfo, uint64_t uid, int64_t ts); +SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo); int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType); -// int32_t qStreamScanMemData(qTaskInfo_t tinfo, const SSubmitReq* pReq, int64_t ver); -// int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit); -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); +void qStreamSetOpen(qTaskInfo_t tinfo); -SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); +void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset); -int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo); +SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo); const SSchemaWrapper* qExtractSchemaFromTask(qTaskInfo_t tinfo); diff --git a/include/libs/function/functionMgt.h b/include/libs/function/functionMgt.h index 60a4a8c6056763c6670cc209eb406c79e5ad47a6..42bc89f0b71aeba488e0e27f899c754caf8f1df9 100644 --- a/include/libs/function/functionMgt.h +++ b/include/libs/function/functionMgt.h @@ -189,7 +189,7 @@ bool fmIsScalarFunc(int32_t funcId); bool fmIsVectorFunc(int32_t funcId); bool fmIsIndefiniteRowsFunc(int32_t funcId); bool fmIsStringFunc(int32_t funcId); -bool fmIsDatetimeFunc(int32_t funcId); +bool fmIsDateTimeFunc(int32_t funcId); bool fmIsSelectFunc(int32_t funcId); bool fmIsTimelineFunc(int32_t funcId); bool fmIsTimeorderFunc(int32_t funcId); diff --git a/include/libs/function/taosudf.h b/include/libs/function/taosudf.h index b4daa895fd1a66ae01db72704c37c7a8707a8ace..c825574fa6e3aea8936a3a9e5cd16224956a94a6 100644 --- a/include/libs/function/taosudf.h +++ b/include/libs/function/taosudf.h @@ -165,6 +165,8 @@ static FORCE_INLINE int32_t udfColEnsureCapacity(SUdfColumn *pColumn, int32_t ne if (tmp == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } + uint32_t extend = BitmapLen(allocCapacity) - BitmapLen(data->rowsAlloc); + memset(tmp + BitmapLen(data->rowsAlloc), 0, extend); data->fixLenCol.nullBitmap = tmp; data->fixLenCol.nullBitmapLen = BitmapLen(allocCapacity); int32_t oldLen = BitmapLen(existedRows); @@ -276,6 +278,8 @@ typedef enum EUdfFuncType { UDF_FUNC_TYPE_SCALAR = 1, UDF_FUNC_TYPE_AGG = 2 } EU typedef struct SScriptUdfInfo { const char *name; + int32_t version; + int64_t createdTime; EUdfFuncType funcType; int8_t scriptType; diff --git a/include/libs/nodes/cmdnodes.h b/include/libs/nodes/cmdnodes.h index 3ae3900a5c501944676ed34de35c92da1cd8670a..2323d044ec4d2c79dad571682026f62f09631588 100644 --- a/include/libs/nodes/cmdnodes.h +++ b/include/libs/nodes/cmdnodes.h @@ -458,7 +458,9 @@ typedef struct SGrantStmt { ENodeType type; char userName[TSDB_USER_LEN]; char objName[TSDB_DB_NAME_LEN]; // db or topic + char tabName[TSDB_TABLE_NAME_LEN]; int64_t privileges; + SNode* pTagCond; } SGrantStmt; typedef SGrantStmt SRevokeStmt; diff --git a/include/libs/nodes/querynodes.h b/include/libs/nodes/querynodes.h index 1a9700907effa002ce4b2acf69196eb08fd7e6d9..480912a8cfd043eeeb985d88d034f2f342150766 100644 --- a/include/libs/nodes/querynodes.h +++ b/include/libs/nodes/querynodes.h @@ -298,6 +298,7 @@ typedef struct SSelectStmt { bool hasUniqueFunc; bool hasTailFunc; bool hasInterpFunc; + bool hasInterpPseudoColFunc; bool hasLastRowFunc; bool hasLastFunc; bool hasTimeLineFunc; diff --git a/include/libs/qcom/query.h b/include/libs/qcom/query.h index b6ada5a0c7c22bbbe1bd938faa1080464c98ae5a..cfc6ef2025d87e5dfc8e025886c62b81cf9f7cbc 100644 --- a/include/libs/qcom/query.h +++ b/include/libs/qcom/query.h @@ -194,6 +194,7 @@ typedef struct SRequestConnInfo { typedef void (*__freeFunc)(void* param); +// todo add creator/destroyer function typedef struct SMsgSendInfo { __async_send_cb_fn_t fp; // async callback function STargetInfo target; // for update epset diff --git a/include/libs/stream/streamState.h b/include/libs/stream/streamState.h index fd5cec293167604795961f49ca7b048102d068e8..42a7261f3889f41befa4545471bbffd40d355f4e 100644 --- a/include/libs/stream/streamState.h +++ b/include/libs/stream/streamState.h @@ -42,6 +42,7 @@ typedef struct STdbState { typedef struct { STdbState* pTdbState; int32_t number; + int64_t checkPointId; } SStreamState; SStreamState* streamStateOpen(char* path, struct SStreamTask* pTask, bool specPath, int32_t szPage, int32_t pages); diff --git a/include/libs/stream/tstream.h b/include/libs/stream/tstream.h index cdcf54bc61fb80dabcbef14da5cae403acf8cd9a..aade34e965a705bd201fcca59d341d152d989a7e 100644 --- a/include/libs/stream/tstream.h +++ b/include/libs/stream/tstream.h @@ -13,8 +13,8 @@ * along with this program. If not, see . */ -#include "executor.h" #include "os.h" +#include "executor.h" #include "query.h" #include "streamState.h" #include "tdatablock.h" @@ -50,6 +50,7 @@ enum { TASK_STATUS__RECOVER_PREPARE, TASK_STATUS__RECOVER1, TASK_STATUS__RECOVER2, + TASK_STATUS__RESTORE, // only available for source task to replay WAL from the checkpoint }; enum { @@ -103,21 +104,8 @@ typedef struct { int8_t type; } SStreamQueueItem; -#if 0 -typedef struct { - int8_t type; - int64_t ver; - int32_t* dataRef; - SSubmitReq* data; -} SStreamDataSubmit; - -typedef struct { - int8_t type; - int64_t ver; - SArray* dataRefs; // SArray - SArray* reqs; // SArray -} SStreamMergedSubmit; -#endif +typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); +typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); typedef struct { int8_t type; @@ -219,36 +207,20 @@ static FORCE_INLINE void streamQueueProcessFail(SStreamQueue* queue) { } static FORCE_INLINE void* streamQueueCurItem(SStreamQueue* queue) { - // return queue->qItem; } -static FORCE_INLINE void* streamQueueNextItem(SStreamQueue* queue) { - int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); - if (dequeueFlag == STREAM_QUEUE__FAILED) { - ASSERT(queue->qItem != NULL); - return streamQueueCurItem(queue); - } else { - queue->qItem = NULL; - taosGetQitem(queue->qall, &queue->qItem); - if (queue->qItem == NULL) { - taosReadAllQitems(queue->queue, queue->qall); - taosGetQitem(queue->qall, &queue->qItem); - } - return streamQueueCurItem(queue); - } -} - -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit); +void* streamQueueNextItem(SStreamQueue* queue); -void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit); +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type); +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit); -SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit); +SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit); typedef struct { - char* qmsg; - // followings are not applicable to encoder and decoder - void* executor; + char* qmsg; + void* pExecutor; // not applicable to encoder and decoder + struct SWalReader* pWalReader; // not applicable to encoder and decoder } STaskExec; typedef struct { @@ -263,16 +235,13 @@ typedef struct { SUseDbRsp dbInfo; } STaskDispatcherShuffle; -typedef void FTbSink(SStreamTask* pTask, void* vnode, int64_t ver, void* data); - typedef struct { int64_t stbUid; char stbFullName[TSDB_TABLE_FNAME_LEN]; SSchemaWrapper* pSchemaWrapper; - // not applicable to encoder and decoder - void* vnode; - FTbSink* tbSinkFunc; - STSchema* pTSchema; + void* vnode; // not available to encoder and decoder + FTbSink* tbSinkFunc; + STSchema* pTSchema; } STaskSinkTb; typedef void FSmaSink(void* vnode, int64_t smaId, const SArray* data); @@ -295,24 +264,34 @@ typedef struct { SEpSet epSet; } SStreamChildEpInfo; -struct SStreamTask { - int64_t streamId; - int32_t taskId; - int32_t totalLevel; - int8_t taskLevel; - int8_t outputType; - int16_t dispatchMsgType; +typedef struct SStreamId { + int64_t streamId; + int32_t taskId; + const char* idStr; +} SStreamId; + +typedef struct SCheckpointInfo { + int64_t id; + int64_t version; // offset in WAL +} SCheckpointInfo; +typedef struct SStreamStatus { int8_t taskStatus; int8_t schedStatus; +} SStreamStatus; - // node info - int32_t selfChildId; - int32_t nodeId; - SEpSet epSet; - - int64_t recoverSnapVer; - int64_t startVer; +struct SStreamTask { + SStreamId id; + int32_t totalLevel; + int8_t taskLevel; + int8_t outputType; + int16_t dispatchMsgType; + SStreamStatus status; + int32_t selfChildId; + int32_t nodeId; + SEpSet epSet; + SCheckpointInfo chkInfo; + STaskExec exec; // fill history int8_t fillHistory; @@ -322,9 +301,6 @@ struct SStreamTask { int32_t nextCheckId; SArray* checkpointInfo; // SArray - // exec - STaskExec exec; - // output union { STaskDispatcherFixedEp fixedEpDispatcher; @@ -334,84 +310,53 @@ struct SStreamTask { STaskSinkFetch fetchSink; }; - int8_t inputStatus; - int8_t outputStatus; - - // STaosQueue* inputQueue1; - // STaosQall* inputQall; + int8_t inputStatus; + int8_t outputStatus; SStreamQueue* inputQueue; SStreamQueue* outputQueue; // trigger - int8_t triggerStatus; - int64_t triggerParam; - void* timer; - - // msg handle - SMsgCb* pMsgCb; - - // state backend - SStreamState* pState; - - // do not serialize - int32_t recoverTryingDownstream; - int32_t recoverWaitingUpstream; - int64_t checkReqId; - SArray* checkReqIds; // shuffle - int32_t refCnt; - - int64_t checkpointingId; - int32_t checkpointAlignCnt; + int8_t triggerStatus; + int64_t triggerParam; + void* timer; + SMsgCb* pMsgCb; // msg handle + SStreamState* pState; // state backend + + // the followings attributes don't be serialized + int32_t recoverTryingDownstream; + int32_t recoverWaitingUpstream; + int64_t checkReqId; + SArray* checkReqIds; // shuffle + int32_t refCnt; + int64_t checkpointingId; + int32_t checkpointAlignCnt; + struct SStreamMeta* pMeta; }; +// meta +typedef struct SStreamMeta { + char* path; + TDB* db; + TTB* pTaskDb; + TTB* pCheckpointDb; + SHashObj* pTasks; + void* ahandle; + TXN* txn; + FTaskExpand* expandFunc; + int32_t vgId; + SRWLatch lock; + int8_t walScan; +} SStreamMeta; + int32_t tEncodeStreamEpInfo(SEncoder* pEncoder, const SStreamChildEpInfo* pInfo); int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo); -SStreamTask* tNewSStreamTask(int64_t streamId); -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask); -void tFreeSStreamTask(SStreamTask* pTask); - -static FORCE_INLINE int32_t streamTaskInput(SStreamTask* pTask, SStreamQueueItem* pItem) { - int8_t type = pItem->type; - if (type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmitClone = streamSubmitRefClone((SStreamDataSubmit2*)pItem); - if (pSubmitClone == NULL) { - qDebug("task %d %p submit enqueue failed since out of memory", pTask->taskId, pTask); - terrno = TSDB_CODE_OUT_OF_MEMORY; - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); - return -1; - } - qDebug("task %d %p submit enqueue %p %p %p %d %" PRId64, pTask->taskId, pTask, pItem, pSubmitClone, - pSubmitClone->submit.msgStr, pSubmitClone->submit.msgLen, pSubmitClone->submit.ver); - taosWriteQitem(pTask->inputQueue->queue, pSubmitClone); - // qStreamInput(pTask->exec.executor, pSubmitClone); - } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || - type == STREAM_INPUT__REF_DATA_BLOCK) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } else if (type == STREAM_INPUT__CHECKPOINT) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } else if (type == STREAM_INPUT__GET_RES) { - taosWriteQitem(pTask->inputQueue->queue, pItem); - // qStreamInput(pTask->exec.executor, pItem); - } - - if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { - atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); - } - -#if 0 - // TODO: back pressure - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); -#endif - return 0; -} - -static FORCE_INLINE void streamTaskInputFail(SStreamTask* pTask) { - atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); -} +SStreamTask* tNewStreamTask(int64_t streamId); +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask); +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask); +void tFreeStreamTask(SStreamTask* pTask); +int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem); +bool tInputQueueIsFull(const SStreamTask* pTask); typedef struct { SMsgHead head; @@ -588,6 +533,7 @@ int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, i int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pMsg); // int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp); +void streamTaskInputFail(SStreamTask* pTask); int32_t streamTryExec(SStreamTask* pTask); int32_t streamSchedExec(SStreamTask* pTask); int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock); @@ -615,40 +561,22 @@ int32_t streamAggRecoverPrepare(SStreamTask* pTask); // int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask); int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId); -// expand and deploy -typedef int32_t FTaskExpand(void* ahandle, SStreamTask* pTask, int64_t ver); - -// meta -typedef struct SStreamMeta { - char* path; - TDB* db; - TTB* pTaskDb; - TTB* pCheckpointDb; - SHashObj* pTasks; - SHashObj* pRecoverStatus; - void* ahandle; - TXN* txn; - FTaskExpand* expandFunc; - int32_t vgId; - SRWLatch lock; -} SStreamMeta; - SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandFunc, int32_t vgId); void streamMetaClose(SStreamMeta* streamMeta); int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask); -int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); -int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t startVer, char* msg, int32_t msgLen); -// SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId); +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask); +int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t checkpointVer, char* msg, int32_t msgLen); +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta); SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId); void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask); void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId); -int32_t streamMetaBegin(SStreamMeta* pMeta); -int32_t streamMetaCommit(SStreamMeta* pMeta); -int32_t streamMetaRollBack(SStreamMeta* pMeta); -int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); +int32_t streamMetaBegin(SStreamMeta* pMeta); +int32_t streamMetaCommit(SStreamMeta* pMeta); +int32_t streamMetaRollBack(SStreamMeta* pMeta); +int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver); // checkpoint int32_t streamProcessCheckpointSourceReq(SStreamMeta* pMeta, SStreamTask* pTask, SStreamCheckpointSourceReq* pReq); diff --git a/include/libs/sync/sync.h b/include/libs/sync/sync.h index fdb4506cf3c00d04b6f5981601eb6915d3cd381b..e86a4f96905512dd08af6acc4b9563206b93f985 100644 --- a/include/libs/sync/sync.h +++ b/include/libs/sync/sync.h @@ -55,6 +55,8 @@ extern "C" { #define SYNC_INDEX_INVALID -1 #define SYNC_TERM_INVALID -1 +#define SYNC_LEARNER_CATCHUP 10 + typedef enum { SYNC_STRATEGY_NO_SNAPSHOT = 0, SYNC_STRATEGY_STANDARD_SNAPSHOT = 1, @@ -76,19 +78,29 @@ typedef enum { TAOS_SYNC_STATE_CANDIDATE = 101, TAOS_SYNC_STATE_LEADER = 102, TAOS_SYNC_STATE_ERROR = 103, + TAOS_SYNC_STATE_LEARNER = 104, } ESyncState; +typedef enum { + TAOS_SYNC_ROLE_VOTER = 0, + TAOS_SYNC_ROLE_LEARNER = 1, + TAOS_SYNC_ROLE_ERROR = 2, +} ESyncRole; + typedef struct SNodeInfo { - int64_t clusterId; - int32_t nodeId; - uint16_t nodePort; - char nodeFqdn[TSDB_FQDN_LEN]; + int64_t clusterId; + int32_t nodeId; + uint16_t nodePort; + char nodeFqdn[TSDB_FQDN_LEN]; + ESyncRole nodeRole; } SNodeInfo; typedef struct SSyncCfg { + int32_t totalReplicaNum; int32_t replicaNum; int32_t myIndex; - SNodeInfo nodeInfo[TSDB_MAX_REPLICA]; + SNodeInfo nodeInfo[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex lastIndex; } SSyncCfg; typedef struct SFsmCbMeta { @@ -155,6 +167,7 @@ typedef struct SSyncFSM { void (*FpBecomeLeaderCb)(const struct SSyncFSM* pFsm); void (*FpBecomeFollowerCb)(const struct SSyncFSM* pFsm); + void (*FpBecomeLearnerCb)(const struct SSyncFSM* pFsm); int32_t (*FpGetSnapshot)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot, void* pReaderParam, void** ppReader); void (*FpGetSnapshotInfo)(const struct SSyncFSM* pFsm, SSnapshot* pSnapshot); @@ -236,6 +249,8 @@ void syncStop(int64_t rid); void syncPreStop(int64_t rid); void syncPostStop(int64_t rid); int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq); +int32_t syncIsCatchUp(int64_t rid); +ESyncRole syncGetRole(int64_t rid); int32_t syncProcessMsg(int64_t rid, SRpcMsg* pMsg); int32_t syncReconfig(int64_t rid, SSyncCfg* pCfg); int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex); diff --git a/include/libs/wal/wal.h b/include/libs/wal/wal.h index 014ed518a397e97424f189527ac4ac2304288bc2..1e3974f5cc93bfef99a54173e92b14d6a1d8c689 100644 --- a/include/libs/wal/wal.h +++ b/include/libs/wal/wal.h @@ -138,7 +138,8 @@ typedef struct { int8_t enableRef; } SWalFilterCond; -typedef struct { +// todo hide this struct +typedef struct SWalReader { SWal *pWal; int64_t readerId; TdFilePtr pLogFile; @@ -146,8 +147,8 @@ typedef struct { int64_t curFileFirstVer; int64_t curVersion; int64_t capacity; - int8_t curInvalid; - int8_t curStopped; +// int8_t curInvalid; +// int8_t curStopped; TdThreadMutex mutex; SWalFilterCond cond; // TODO remove it @@ -189,13 +190,15 @@ int32_t walApplyVer(SWal *, int64_t ver); // int32_t walDataCorrupted(SWal*); -// read +// wal reader SWalReader *walOpenReader(SWal *, SWalFilterCond *pCond); void walCloseReader(SWalReader *pRead); void walReadReset(SWalReader *pReader); int32_t walReadVer(SWalReader *pRead, int64_t ver); int32_t walReadSeekVer(SWalReader *pRead, int64_t ver); int32_t walNextValidMsg(SWalReader *pRead); +int64_t walReaderGetCurrentVer(const SWalReader* pReader); +void walReaderValidVersionRange(SWalReader* pReader, int64_t *sver, int64_t *ever); // only for tq usage void walSetReaderCapacity(SWalReader *pRead, int32_t capacity); diff --git a/include/util/taoserror.h b/include/util/taoserror.h index 80e17ccfd6dd97bdb4c15af545c0c18dac8ec4d1..a709ccf10c98f2753b11e77946306c9e9a750106 100644 --- a/include/util/taoserror.h +++ b/include/util/taoserror.h @@ -103,7 +103,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_CHECKSUM_ERROR TAOS_DEF_ERROR_CODE(0, 0x011F) // internal #define TSDB_CODE_COMPRESS_ERROR TAOS_DEF_ERROR_CODE(0, 0x0120) -#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) // +#define TSDB_CODE_MSG_NOT_PROCESSED TAOS_DEF_ERROR_CODE(0, 0x0121) #define TSDB_CODE_CFG_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0122) #define TSDB_CODE_REPEAT_INIT TAOS_DEF_ERROR_CODE(0, 0x0123) #define TSDB_CODE_DUP_KEY TAOS_DEF_ERROR_CODE(0, 0x0124) @@ -118,9 +118,10 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MSG_ENCODE_ERROR TAOS_DEF_ERROR_CODE(0, 0x012D) #define TSDB_CODE_NO_ENOUGH_DISKSPACE TAOS_DEF_ERROR_CODE(0, 0x012E) -#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) // -#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) // -#define TSDB_CODE_IVLD_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132) // +#define TSDB_CODE_APP_IS_STARTING TAOS_DEF_ERROR_CODE(0, 0x0130) +#define TSDB_CODE_APP_IS_STOPPING TAOS_DEF_ERROR_CODE(0, 0x0131) +#define TSDB_CODE_INVALID_DATA_FMT TAOS_DEF_ERROR_CODE(0, 0x0132) +#define TSDB_CODE_INVALID_CFG_VALUE TAOS_DEF_ERROR_CODE(0, 0x0133) //client #define TSDB_CODE_TSC_INVALID_OPERATION TAOS_DEF_ERROR_CODE(0, 0x0200) @@ -242,6 +243,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_MND_INVALID_ALTER_OPER TAOS_DEF_ERROR_CODE(0, 0x0356) #define TSDB_CODE_MND_AUTH_FAILURE TAOS_DEF_ERROR_CODE(0, 0x0357) #define TSDB_CODE_MND_USER_NOT_AVAILABLE TAOS_DEF_ERROR_CODE(0, 0x0358) +#define TSDB_CODE_MND_PRIVILEDGE_EXIST TAOS_DEF_ERROR_CODE(0, 0x0359) // mnode-stable-part1 #define TSDB_CODE_MND_STB_ALREADY_EXIST TAOS_DEF_ERROR_CODE(0, 0x0360) @@ -402,6 +404,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_SNODE_ALREADY_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x040F) #define TSDB_CODE_SNODE_NOT_FOUND TAOS_DEF_ERROR_CODE(0, 0x0410) #define TSDB_CODE_SNODE_NOT_DEPLOYED TAOS_DEF_ERROR_CODE(0, 0x0411) +#define TSDB_CODE_MNODE_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0412) // internal +#define TSDB_CODE_MNODE_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0413) // internal // vnode // #define TSDB_CODE_VND_ACTION_IN_PROGRESS TAOS_DEF_ERROR_CODE(0, 0x0500) // 2.x @@ -436,6 +440,8 @@ int32_t* taosGetErrno(); #define TSDB_CODE_VND_STOPPED TAOS_DEF_ERROR_CODE(0, 0x0529) #define TSDB_CODE_VND_DUP_REQUEST TAOS_DEF_ERROR_CODE(0, 0x0530) #define TSDB_CODE_VND_QUERY_BUSY TAOS_DEF_ERROR_CODE(0, 0x0531) +#define TSDB_CODE_VND_NOT_CATCH_UP TAOS_DEF_ERROR_CODE(0, 0x0532) // internal +#define TSDB_CODE_VND_ALREADY_IS_VOTER TAOS_DEF_ERROR_CODE(0, 0x0533) // internal // tsdb #define TSDB_CODE_TDB_INVALID_TABLE_ID TAOS_DEF_ERROR_CODE(0, 0x0600) @@ -533,19 +539,20 @@ int32_t* taosGetErrno(); // #define TSDB_CODE_SYN_INVALID_CHECKSUM TAOS_DEF_ERROR_CODE(0, 0x0908) // 2.x // #define TSDB_CODE_SYN_INVALID_MSGLEN TAOS_DEF_ERROR_CODE(0, 0x0909) // 2.x // #define TSDB_CODE_SYN_INVALID_MSGTYPE TAOS_DEF_ERROR_CODE(0, 0x090A) // 2.x -#define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) +// #define TSDB_CODE_SYN_IS_LEADER TAOS_DEF_ERROR_CODE(0, 0x090B) // unused #define TSDB_CODE_SYN_NOT_LEADER TAOS_DEF_ERROR_CODE(0, 0x090C) -#define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) -#define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x090E) -#define TSDB_CODE_SYN_NEW_CONFIG_ERROR TAOS_DEF_ERROR_CODE(0, 0x090F) // internal -#define TSDB_CODE_SYN_RECONFIG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0910) +// #define TSDB_CODE_SYN_ONE_REPLICA TAOS_DEF_ERROR_CODE(0, 0x090D) // unused +// #define TSDB_CODE_SYN_NOT_IN_NEW_CONFIG TAOS_DEF_ERROR_CODE(0, 0x090E) // unused +#define TSDB_CODE_SYN_NEW_CONFIG_ERROR TAOS_DEF_ERROR_CODE(0, 0x090F) // internal +// #define TSDB_CODE_SYN_RECONFIG_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0910) // unused #define TSDB_CODE_SYN_PROPOSE_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0911) -#define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) -#define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) +// #define TSDB_CODE_SYN_STANDBY_NOT_READY TAOS_DEF_ERROR_CODE(0, 0x0912) // unused +// #define TSDB_CODE_SYN_BATCH_ERROR TAOS_DEF_ERROR_CODE(0, 0x0913) // unused #define TSDB_CODE_SYN_RESTORING TAOS_DEF_ERROR_CODE(0, 0x0914) -#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal +#define TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG TAOS_DEF_ERROR_CODE(0, 0x0915) // internal #define TSDB_CODE_SYN_BUFFER_FULL TAOS_DEF_ERROR_CODE(0, 0x0916) #define TSDB_CODE_SYN_WRITE_STALL TAOS_DEF_ERROR_CODE(0, 0x0917) +#define TSDB_CODE_SYN_NEGOTIATION_WIN_FULL TAOS_DEF_ERROR_CODE(0, 0x0918) #define TSDB_CODE_SYN_INTERNAL_ERROR TAOS_DEF_ERROR_CODE(0, 0x09FF) // tq @@ -566,7 +573,7 @@ int32_t* taosGetErrno(); // wal // #define TSDB_CODE_WAL_APP_ERROR TAOS_DEF_ERROR_CODE(0, 0x1000) // 2.x #define TSDB_CODE_WAL_FILE_CORRUPTED TAOS_DEF_ERROR_CODE(0, 0x1001) -#define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) +// #define TSDB_CODE_WAL_SIZE_LIMIT TAOS_DEF_ERROR_CODE(0, 0x1002) // unused #define TSDB_CODE_WAL_INVALID_VER TAOS_DEF_ERROR_CODE(0, 0x1003) // #define TSDB_CODE_WAL_OUT_OF_MEMORY TAOS_DEF_ERROR_CODE(0, 0x1004) // 2.x #define TSDB_CODE_WAL_LOG_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x1005) @@ -710,15 +717,13 @@ int32_t* taosGetErrno(); #define TSDB_CODE_UDF_STOPPING TAOS_DEF_ERROR_CODE(0, 0x2901) #define TSDB_CODE_UDF_PIPE_READ_ERR TAOS_DEF_ERROR_CODE(0, 0x2902) #define TSDB_CODE_UDF_PIPE_CONNECT_ERR TAOS_DEF_ERROR_CODE(0, 0x2903) -#define TSDB_CODE_UDF_PIPE_NO_PIPE TAOS_DEF_ERROR_CODE(0, 0x2904) +#define TSDB_CODE_UDF_PIPE_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x2904) #define TSDB_CODE_UDF_LOAD_UDF_FAILURE TAOS_DEF_ERROR_CODE(0, 0x2905) -#define TSDB_CODE_UDF_INVALID_STATE TAOS_DEF_ERROR_CODE(0, 0x2906) -#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2907) -#define TSDB_CODE_UDF_NO_FUNC_HANDLE TAOS_DEF_ERROR_CODE(0, 0x2908) -#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2909) -#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x290A) -#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x290B) -#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290C) +#define TSDB_CODE_UDF_INVALID_INPUT TAOS_DEF_ERROR_CODE(0, 0x2906) +#define TSDB_CODE_UDF_INVALID_BUFSIZE TAOS_DEF_ERROR_CODE(0, 0x2907) +#define TSDB_CODE_UDF_INVALID_OUTPUT_TYPE TAOS_DEF_ERROR_CODE(0, 0x2908) +#define TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED TAOS_DEF_ERROR_CODE(0, 0x2909) +#define TSDB_CODE_UDF_FUNC_EXEC_FAILURE TAOS_DEF_ERROR_CODE(0, 0x290A) // sml #define TSDB_CODE_SML_INVALID_PROTOCOL_TYPE TAOS_DEF_ERROR_CODE(0, 0x3000) @@ -762,6 +767,7 @@ int32_t* taosGetErrno(); #define TSDB_CODE_TMQ_INVALID_MSG TAOS_DEF_ERROR_CODE(0, 0x4000) #define TSDB_CODE_TMQ_CONSUMER_MISMATCH TAOS_DEF_ERROR_CODE(0, 0x4001) #define TSDB_CODE_TMQ_CONSUMER_CLOSED TAOS_DEF_ERROR_CODE(0, 0x4002) +#define TSDB_CODE_TMQ_CONSUMER_ERROR TAOS_DEF_ERROR_CODE(0, 0x4003) // stream #define TSDB_CODE_STREAM_TASK_NOT_EXIST TAOS_DEF_ERROR_CODE(0, 0x4100) diff --git a/include/util/tdef.h b/include/util/tdef.h index e5000891c95e3a2b9722c86434b1010490377e2a..664f22eeed7a2b167a2053739d78676afb71d9ac 100644 --- a/include/util/tdef.h +++ b/include/util/tdef.h @@ -198,6 +198,7 @@ typedef enum ELogicConditionType { #define TSDB_STREAM_NAME_LEN 193 // it is a null-terminated string #define TSDB_DB_NAME_LEN 65 #define TSDB_DB_FNAME_LEN (TSDB_ACCT_ID_LEN + TSDB_DB_NAME_LEN + TSDB_NAME_DELIMITER_LEN) +#define TSDB_PRIVILEDGE_CONDITION_LEN 200 #define TSDB_FUNC_NAME_LEN 65 #define TSDB_FUNC_COMMENT_LEN 1024 * 1024 @@ -284,8 +285,11 @@ typedef enum ELogicConditionType { #define TSDB_DNODE_ROLE_VNODE 2 #define TSDB_MAX_REPLICA 5 +#define TSDB_MAX_LEARNER_REPLICA 10 #define TSDB_SYNC_LOG_BUFFER_SIZE 4096 -#define TSDB_SYNC_LOG_BUFFER_RETENTION (TSDB_SYNC_LOG_BUFFER_SIZE >> 4) +#define TSDB_SYNC_LOG_BUFFER_RETENTION 256 +#define TSDB_SYNC_APPLYQ_SIZE_LIMIT 512 +#define TSDB_SYNC_NEGOTIATION_WIN 512 #define TSDB_TBNAME_COLUMN_INDEX (-1) #define TSDB_MULTI_TABLEMETA_MAX_NUM 100000 // maximum batch size allowed to load table meta diff --git a/include/util/tlog.h b/include/util/tlog.h index 0071b3d32cd72be79ab5adf8bdba20b92c18f4f7..d267d5876a265d0aea8fae911f0ff0187ff5f9e3 100644 --- a/include/util/tlog.h +++ b/include/util/tlog.h @@ -83,6 +83,12 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons #endif ; +void taosPrintSlowLog(const char *format, ...) +#ifdef __GNUC__ + __attribute__((format(printf, 1, 2))) +#endif + ; + bool taosAssertDebug(bool condition, const char *file, int32_t line, const char *format, ...); bool taosAssertRelease(bool condition); diff --git a/include/util/tqueue.h b/include/util/tqueue.h index 1f6b205cdf8a2faa045a7a08b9008cb271a741a1..d05b5418b37d1506d52f7b211c3c92db53dabc85 100644 --- a/include/util/tqueue.h +++ b/include/util/tqueue.h @@ -61,7 +61,7 @@ typedef void (*FItems)(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfItems); typedef struct STaosQnode STaosQnode; -typedef struct STaosQnode { +struct STaosQnode { STaosQnode *next; STaosQueue *queue; int64_t timestamp; @@ -70,9 +70,9 @@ typedef struct STaosQnode { int8_t itype; int8_t reserved[3]; char item[]; -} STaosQnode; +}; -typedef struct STaosQueue { +struct STaosQueue { STaosQnode *head; STaosQnode *tail; STaosQueue *next; // for queue set @@ -84,22 +84,22 @@ typedef struct STaosQueue { int64_t memOfItems; int32_t numOfItems; int64_t threadId; -} STaosQueue; +}; -typedef struct STaosQset { +struct STaosQset { STaosQueue *head; STaosQueue *current; TdThreadMutex mutex; tsem_t sem; int32_t numOfQueues; int32_t numOfItems; -} STaosQset; +}; -typedef struct STaosQall { +struct STaosQall { STaosQnode *current; STaosQnode *start; int32_t numOfItems; -} STaosQall; +}; STaosQueue *taosOpenQueue(); void taosCloseQueue(STaosQueue *queue); diff --git a/packaging/debRpmAutoInstall.sh b/packaging/debRpmAutoInstall.sh index 2fe18fd7a939ddf614e7f8a565ad9448d46f4122..8fadffe4c68d05f9bb0e6f3ed160ead7d8bdb00d 100755 --- a/packaging/debRpmAutoInstall.sh +++ b/packaging/debRpmAutoInstall.sh @@ -1,7 +1,7 @@ #!/usr/bin/expect set packageName [lindex $argv 0] set packageSuffix [lindex $argv 1] -set timeout 3 +set timeout 30 if { ${packageSuffix} == "deb" } { spawn dpkg -i ${packageName} } elseif { ${packageSuffix} == "rpm"} { diff --git a/packaging/docker/Dockerfile b/packaging/docker/Dockerfile index 35bea0e65ccc5070fe9d4e82adadc7132ae7cc81..7d90beac1c9e12792a4d9a1de76c06b91157049f 100644 --- a/packaging/docker/Dockerfile +++ b/packaging/docker/Dockerfile @@ -12,7 +12,7 @@ ENV TINI_VERSION v0.19.0 ADD https://github.com/krallin/tini/releases/download/${TINI_VERSION}/tini-${cpuType} /tini ENV DEBIAN_FRONTEND=noninteractive WORKDIR /root/ -RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini +RUN tar -zxf ${pkgFile} && cd /root/${dirName}/ && /bin/bash install.sh -e no && cd /root && rm /root/${pkgFile} && rm -rf /root/${dirName} && apt-get update && apt-get install -y locales tzdata netcat curl gdb vim tmux less net-tools valgrind && locale-gen en_US.UTF-8 && apt-get clean && rm -rf /var/lib/apt/lists/ && chmod +x /tini ENV LD_LIBRARY_PATH="$LD_LIBRARY_PATH:/usr/lib" \ LC_CTYPE=en_US.UTF-8 \ diff --git a/packaging/testpackage.sh b/packaging/testpackage.sh index 97226a86b5a4cdb9eac50abee4da54dfdfa46996..081383f89b358325f55cc1a7641015efcb0a4eed 100755 --- a/packaging/testpackage.sh +++ b/packaging/testpackage.sh @@ -246,7 +246,7 @@ if [ ! -f debRpmAutoInstall.sh ];then echo '#!/usr/bin/expect ' > debRpmAutoInstall.sh echo 'set packageName [lindex $argv 0]' >> debRpmAutoInstall.sh echo 'set packageSuffix [lindex $argv 1]' >> debRpmAutoInstall.sh - echo 'set timeout 3 ' >> debRpmAutoInstall.sh + echo 'set timeout 30 ' >> debRpmAutoInstall.sh echo 'if { ${packageSuffix} == "deb" } {' >> debRpmAutoInstall.sh echo ' spawn dpkg -i ${packageName} ' >> debRpmAutoInstall.sh echo '} elseif { ${packageSuffix} == "rpm"} {' >> debRpmAutoInstall.sh diff --git a/packaging/tools/install.sh b/packaging/tools/install.sh index 0a4f3ff6227b9710a8a533e0af55d1abc0a594c7..1b47b10520147664a3c2f3a558fed1208f84a2ca 100755 --- a/packaging/tools/install.sh +++ b/packaging/tools/install.sh @@ -572,6 +572,22 @@ function install_config() { done } +function install_share_etc() { + [ ! -d ${script_dir}/share/etc ] && return + for c in `ls ${script_dir}/share/etc/`; do + if [ -e /etc/${clientName2}/$c ]; then + out=/etc/${clientName2}/$c.new.`date +%F` + ${csudo}cp -f ${script_dir}/share/etc/$c $out ||: + else + ${csudo}mkdir -p /etc/${clientName2} >/dev/null 2>/dev/null ||: + ${csudo}cp -f ${script_dir}/share/etc/$c /etc/${clientName2}/$c ||: + fi + done + + [ ! -d ${script_dir}/share/srv ] && return + ${csudo} cp ${script_dir}/share/srv/* ${service_config_dir} ||: +} + function install_log() { ${csudo}rm -rf ${log_dir} || : ${csudo}mkdir -p ${log_dir} && ${csudo}chmod 777 ${log_dir} @@ -599,7 +615,7 @@ function install_examples() { function install_web() { if [ -d "${script_dir}/share" ]; then - ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share + ${csudo}cp -rf ${script_dir}/share/* ${install_main_dir}/share > /dev/null 2>&1 ||: fi } @@ -687,11 +703,33 @@ function clean_service_on_systemd() { # if [ "$verMode" == "cluster" ] && [ "$clientName" != "$clientName2" ]; then # ${csudo}rm -f ${service_config_dir}/${serverName2}.service # fi + x_service_config="${service_config_dir}/${xName2}.service" + if [ -e "$x_service_config" ]; then + if systemctl is-active --quiet ${xName2}; then + echo "${productName2} ${xName2} is running, stopping it..." + ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${x_service_config} + fi + + explorer_service_config="${service_config_dir}/${explorerName2}.service" + if [ -e "$explorer_service_config" ]; then + if systemctl is-active --quiet ${explorerName2}; then + echo "${productName2} ${explorerName2} is running, stopping it..." + ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${explorer_service_config} + ${csudo}rm -f /etc/${clientName2}/explorer.toml + fi } function install_service_on_systemd() { clean_service_on_systemd + install_share_etc + [ -f ${script_dir}/cfg/${serverName2}.service ] && ${csudo}cp ${script_dir}/cfg/${serverName2}.service \ ${service_config_dir}/ || : diff --git a/packaging/tools/mac_before_install.txt b/packaging/tools/mac_before_install.txt index a428c612b2607ea482f0a6ddc5b26636470b72b6..4ce2374b7fed8e9cdaf50c62231df8ae6a2032e4 100644 --- a/packaging/tools/mac_before_install.txt +++ b/packaging/tools/mac_before_install.txt @@ -1,9 +1,9 @@ TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. -To configure TDengine : edit /etc/taos/taos.cfg -To start service : launchctl start com.tdengine.taosd -To start Taos Adapter : launchctl start com.tdengine.taosadapter -To access TDengine : use taos in shell +• To configure TDengine, edit /etc/taos/taos.cfg +• To start service, run launchctl start com.tdengine.taosd +• To start Taos Adapter, run launchctl start com.tdengine.taosadapter +• To access TDengine from your local machine, run taos If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. diff --git a/packaging/tools/mac_before_install_client.txt b/packaging/tools/mac_before_install_client.txt index 0457d73c49a0bdba49864477f32d07aaa512c375..cce8191667f7570591612944f611729e9e399c31 100644 --- a/packaging/tools/mac_before_install_client.txt +++ b/packaging/tools/mac_before_install_client.txt @@ -1,9 +1,9 @@ TDengine is an open-source, cloud-native time-series database optimized for Internet of Things (IoT), Connected Cars, and Industrial IoT. With its built-in caching, stream processing, and data subscription capabilities, TDengine offers a simplified solution for time-series data processing. -Once it's installed, please take the steps below: -1: open a terminal/shell in Mac -2: if connecting to Cloud Service, follow the instructions on your cloud service account and configure the environment variable -3: if connecting to another TDengine Service, you can also view help information via "taos --help" -4: execute command taos +After the installation process is complete, perform the following steps to start using TDengine: +1: Open Terminal on your Mac. +2: To connect to a TDengine server using the default settings and credentials, run the taos command. +3: To connect to a TDengine server using custom settings or credentials, run taos --help for more information. +4: To connect to TDengine Cloud, follow the instructions on the Tools - TDengine CLI page in your TDengine Cloud account. -If you're experiencing problems installing TDengine, check the file /var/log/taos/tdengine_install.log to help troubleshoot the installation. +If any issues occur during installation, check the /var/log/taos/tdengine_install.log file to troubleshoot. \ No newline at end of file diff --git a/packaging/tools/makeclient.sh b/packaging/tools/makeclient.sh index b473f3b52728714276b8493ea0bc96094e880e34..1dbfc897bc7f11a5e5c302fbf08d73bc539808be 100755 --- a/packaging/tools/makeclient.sh +++ b/packaging/tools/makeclient.sh @@ -197,7 +197,8 @@ if [[ $productName == "TDengine" ]]; then mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then if [ "$osType" != "Darwin" ]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : fi git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/makepkg.sh b/packaging/tools/makepkg.sh index 1a1622cb9368c00a40105daa653639b94e46fb94..3da005c405cd61aa77ebf89240497ffa485b989f 100755 --- a/packaging/tools/makepkg.sh +++ b/packaging/tools/makepkg.sh @@ -51,11 +51,9 @@ fi if [ -d ${top_dir}/tools/taos-tools/packaging/deb ]; then cd ${top_dir}/tools/taos-tools/packaging/deb - - taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) - [ -z "$taos_tools_ver" ] && taos_tools_ver="0.1.0" taostools_ver=$(git for-each-ref --sort=taggerdate --format '%(tag)' refs/tags|grep -v taos | tail -1) + [ -z "$taostools_ver" ] && taostools_ver="0.1.0" taostools_install_dir="${release_dir}/${clientName2}Tools-${taostools_ver}" cd ${curr_dir} @@ -152,6 +150,7 @@ fi mkdir -p ${install_dir}/bin && cp ${bin_files} ${install_dir}/bin && chmod a+x ${install_dir}/bin/* || : mkdir -p ${install_dir}/init.d && cp ${init_file_deb} ${install_dir}/init.d/${serverName}.deb mkdir -p ${install_dir}/init.d && cp ${init_file_rpm} ${install_dir}/init.d/${serverName}.rpm +# mkdir -p ${install_dir}/share && cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: if [ $adapterName != "taosadapter" ]; then mv ${install_dir}/cfg/${clientName2}adapter.toml ${install_dir}/cfg/$adapterName.toml @@ -323,6 +322,7 @@ if [[ $dbName == "taos" ]]; then mkdir -p ${install_dir}/share/ cp -Rfap ${web_dir}/admin ${install_dir}/share/ cp ${web_dir}/png/taos.png ${install_dir}/share/admin/images/taos.png + cp -rf ${build_dir}/share/{etc,srv} ${install_dir}/share ||: else echo "directory not found for enterprise release: ${web_dir}/admin" fi @@ -338,7 +338,20 @@ if [ "$verMode" == "cluster" ] || [ "$verMode" == "cloud" ]; then connector_dir="${code_dir}/connector" mkdir -p ${install_dir}/connector if [[ "$pagMode" != "lite" ]] && [[ "$cpuType" != "aarch32" ]]; then - [ -f ${build_dir}/lib/*.jar ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : + tmp_pwd=`pwd` + cd ${install_dir}/connector + if [ ! -d taos-connector-jdbc ];then + git clone -b 3.1.0 --depth=1 https://github.com/taosdata/taos-connector-jdbc.git ||: + fi + cd taos-connector-jdbc + mvn clean package -Dmaven.test.skip=true + echo ${build_dir}/lib/ + cp target/*.jar ${build_dir}/lib/ + cd ${install_dir}/connector + rm -rf taos-connector-jdbc + cd ${tmp_pwd} + jars=$(ls ${build_dir}/lib/*.jar 2>/dev/null|wc -l) + [ "${jars}" != "0" ] && cp ${build_dir}/lib/*.jar ${install_dir}/connector || : git clone --depth 1 https://github.com/taosdata/driver-go ${install_dir}/connector/go rm -rf ${install_dir}/connector/go/.git ||: diff --git a/packaging/tools/remove.sh b/packaging/tools/remove.sh index 8ed3bd74b9f6b822c15bf568256f5f80f2b8b618..6c671473bfb5046ac18e7b813088ec6d6d3843d2 100755 --- a/packaging/tools/remove.sh +++ b/packaging/tools/remove.sh @@ -192,7 +192,27 @@ function clean_service_on_systemd() { ${csudo}systemctl stop ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null fi ${csudo}systemctl disable ${tarbitrator_service_name} &>/dev/null || echo &>/dev/null - ${csudo}rm -f ${tarbitratord_service_config} + + x_service_config="${service_config_dir}/${xName2}.service" + if [ -e "$x_service_config" ]; then + if systemctl is-active --quiet ${xName2}; then + echo "${productName2} ${xName2} is running, stopping it..." + ${csudo}systemctl stop ${xName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${xName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${x_service_config} + fi + + explorer_service_config="${service_config_dir}/${explorerName2}.service" + if [ -e "$explorer_service_config" ]; then + if systemctl is-active --quiet ${explorerName2}; then + echo "${productName2} ${explorerName2} is running, stopping it..." + ${csudo}systemctl stop ${explorerName2} &>/dev/null || echo &>/dev/null + fi + ${csudo}systemctl disable ${explorerName2} &>/dev/null || echo &>/dev/null + ${csudo}rm -f ${explorer_service_config} + ${csudo}rm -f /etc/${clientName2}/explorer.toml + fi } function clean_service_on_sysvinit() { diff --git a/source/client/inc/clientInt.h b/source/client/inc/clientInt.h index 86db35b4123c87c3d3f6d546caa15e70d09884e4..8e20d7d2752d143d59766ac6eca7887e90f08333 100644 --- a/source/client/inc/clientInt.h +++ b/source/client/inc/clientInt.h @@ -36,14 +36,6 @@ extern "C" { #include "tconfig.h" -#define CHECK_CODE_GOTO(expr, label) \ - do { \ - code = expr; \ - if (TSDB_CODE_SUCCESS != code) { \ - goto label; \ - } \ - } while (0) - #define ERROR_MSG_BUF_DEFAULT_SIZE 512 #define HEARTBEAT_INTERVAL 1500 // ms @@ -71,6 +63,7 @@ typedef struct { // statistics int32_t reportCnt; int32_t connKeyCnt; + int32_t passKeyCnt; // with passVer call back int64_t reportBytes; // not implemented int64_t startTime; // ctl @@ -133,6 +126,12 @@ typedef struct SAppInfo { TdThreadMutex mutex; } SAppInfo; +typedef struct { + int32_t ver; + void* param; + __taos_notify_fn_t fp; +} SPassInfo; + typedef struct STscObj { char user[TSDB_USER_LEN]; char pass[TSDB_PASSWORD_LEN]; @@ -148,6 +147,7 @@ typedef struct STscObj { int32_t numOfReqs; // number of sqlObj bound to this connection SAppInstInfo* pAppInfo; SHashObj* pRequests; + SPassInfo passInfo; } STscObj; typedef struct STscDbg { @@ -286,28 +286,7 @@ static FORCE_INLINE SReqResultInfo* tmqGetCurResInfo(TAOS_RES* res) { return (SReqResultInfo*)&msg->resInfo; } -static FORCE_INLINE SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { - SMqRspObj* pRspObj = (SMqRspObj*)res; - pRspObj->resIter++; - - if (pRspObj->resIter < pRspObj->rsp.blockNum) { - SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); - if (pRspObj->rsp.withSchema) { - SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); - setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); - taosMemoryFreeClear(pRspObj->resInfo.row); - taosMemoryFreeClear(pRspObj->resInfo.pCol); - taosMemoryFreeClear(pRspObj->resInfo.length); - taosMemoryFreeClear(pRspObj->resInfo.convertBuf); - taosMemoryFreeClear(pRspObj->resInfo.convertJson); - } - - setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); - return &pRspObj->resInfo; - } - - return NULL; -} +SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4); static FORCE_INLINE SReqResultInfo* tscGetCurResInfo(TAOS_RES* res) { if (TD_RES_QUERY(res)) return &(((SRequestObj*)res)->body.resInfo); @@ -320,7 +299,6 @@ extern int32_t clientConnRefPool; extern int32_t timestampDeltaLimit; extern int64_t lastClusterId; - __async_send_cb_fn_t getMsgRspHandle(int32_t msgType); SMsgSendInfo* buildMsgInfoImpl(SRequestObj* pReqObj); @@ -373,7 +351,6 @@ void taos_close_internal(void* taos); // global, called by mgmt int hbMgrInit(); void hbMgrCleanUp(); -int hbHandleRsp(SClientHbBatchRsp* hbRsp); // cluster level SAppHbMgr* appHbMgrInit(SAppInstInfo* pAppInstInfo, char* key); @@ -384,10 +361,7 @@ void stopAllRequests(SHashObj* pRequests); // conn level int hbRegisterConn(SAppHbMgr* pAppHbMgr, int64_t tscRefId, int64_t clusterId, int8_t connType); -void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey); - -// --- mq -void hbMgrInitMqHbRspHandle(); +void hbDeregisterConn(SAppHbMgr* pAppHbMgr, SClientHbKey connKey, void* param); typedef struct SSqlCallbackWrapper { SParseContext* pParseCtx; diff --git a/source/client/inc/clientLog.h b/source/client/inc/clientLog.h index c29f495201d7b9ec0360abf6a5414798739b3da8..908e47083058b25bc0b6439e3e8888e1d2585328 100644 --- a/source/client/inc/clientLog.h +++ b/source/client/inc/clientLog.h @@ -26,6 +26,7 @@ extern "C" { #define tscFatal(...) do { if (cDebugFlag & DEBUG_FATAL) { taosPrintLog("TSC FATAL ", DEBUG_FATAL, cDebugFlag, __VA_ARGS__); }} while(0) #define tscError(...) do { if (cDebugFlag & DEBUG_ERROR) { taosPrintLog("TSC ERROR ", DEBUG_ERROR, cDebugFlag, __VA_ARGS__); }} while(0) #define tscWarn(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLog("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) +#define tscWarnL(...) do { if (cDebugFlag & DEBUG_WARN) { taosPrintLongString("TSC WARN ", DEBUG_WARN, cDebugFlag, __VA_ARGS__); }} while(0) #define tscInfo(...) do { if (cDebugFlag & DEBUG_INFO) { taosPrintLog("TSC ", DEBUG_INFO, cDebugFlag, __VA_ARGS__); }} while(0) #define tscDebug(...) do { if (cDebugFlag & DEBUG_DEBUG) { taosPrintLog("TSC ", DEBUG_DEBUG, cDebugFlag, __VA_ARGS__); }} while(0) #define tscTrace(...) do { if (cDebugFlag & DEBUG_TRACE) { taosPrintLog("TSC ", DEBUG_TRACE, cDebugFlag, __VA_ARGS__); }} while(0) diff --git a/source/client/inc/clientStmt.h b/source/client/inc/clientStmt.h index 2b42de93e3f45ccb95c085c31825d4e0a117538c..cbef80b6daa63fd3192aae98cfb6170893381894 100644 --- a/source/client/inc/clientStmt.h +++ b/source/client/inc/clientStmt.h @@ -102,6 +102,7 @@ typedef struct STscStmt { SStmtBindInfo bInfo; int64_t reqid; + int32_t errCode; } STscStmt; extern char *gStmtStatusStr[]; @@ -121,6 +122,7 @@ extern char *gStmtStatusStr[]; int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ + pStmt->errCode = _code; \ return _code; \ } \ } while (0) @@ -129,6 +131,7 @@ extern char *gStmtStatusStr[]; int32_t _code = c; \ if (_code != TSDB_CODE_SUCCESS) { \ terrno = _code; \ + pStmt->errCode = _code; \ } \ return _code; \ } while (0) @@ -137,9 +140,19 @@ extern char *gStmtStatusStr[]; code = c; \ if (code != TSDB_CODE_SUCCESS) { \ terrno = code; \ + pStmt->errCode = code; \ goto _return; \ } \ } while (0) +#define STMT_ERRI_JRET(c) \ + do { \ + code = c; \ + if (code != TSDB_CODE_SUCCESS) { \ + terrno = code; \ + goto _return; \ + } \ + } while (0) + #define STMT_ELOG(param, ...) qError("stmt:%p " param, pStmt, __VA_ARGS__) #define STMT_DLOG(param, ...) qDebug("stmt:%p " param, pStmt, __VA_ARGS__) diff --git a/source/client/src/clientEnv.c b/source/client/src/clientEnv.c index 874ac12f5c97befe568e6c6488ca52f1fb32f34c..07ca3d89093c721aca6b51bfe0f107e01ca3240f 100644 --- a/source/client/src/clientEnv.c +++ b/source/client/src/clientEnv.c @@ -42,7 +42,7 @@ SAppInfo appInfo; int64_t lastClusterId = 0; int32_t clientReqRefPool = -1; int32_t clientConnRefPool = -1; -int32_t clientStop = 0; +int32_t clientStop = -1; int32_t timestampDeltaLimit = 900; // s @@ -69,7 +69,6 @@ static int32_t registerRequest(SRequestObj *pRequest, STscObj *pTscObj) { } static void deregisterRequest(SRequestObj *pRequest) { - const static int64_t SLOW_QUERY_INTERVAL = 3000000L; // todo configurable if (pRequest == NULL) { tscError("pRequest == NULL"); return; @@ -80,6 +79,7 @@ static void deregisterRequest(SRequestObj *pRequest) { int32_t currentInst = atomic_sub_fetch_64((int64_t *)&pActivity->currentRequests, 1); int32_t num = atomic_sub_fetch_32(&pTscObj->numOfReqs, 1); + int32_t reqType = SLOW_LOG_TYPE_OTHERS; int64_t duration = taosGetTimestampUs() - pRequest->metric.start; tscDebug("0x%" PRIx64 " free Request from connObj: 0x%" PRIx64 ", reqId:0x%" PRIx64 @@ -95,6 +95,7 @@ static void deregisterRequest(SRequestObj *pRequest) { duration, pRequest->metric.parseCostUs, pRequest->metric.ctgCostUs, pRequest->metric.analyseCostUs, pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->insertElapsedTime, duration); + reqType = SLOW_LOG_TYPE_INSERT; } else if (QUERY_NODE_SELECT_STMT == pRequest->stmtType) { tscDebug("query duration %" PRId64 "us: parseCost:%" PRId64 "us, ctgCost:%" PRId64 "us, analyseCost:%" PRId64 "us, planCost:%" PRId64 "us, exec:%" PRId64 "us", @@ -102,11 +103,16 @@ static void deregisterRequest(SRequestObj *pRequest) { pRequest->metric.planCostUs, pRequest->metric.execCostUs); atomic_add_fetch_64((int64_t *)&pActivity->queryElapsedTime, duration); + reqType = SLOW_LOG_TYPE_QUERY; } } - if (duration >= SLOW_QUERY_INTERVAL) { + if (duration >= (tsSlowLogThreshold * 1000000UL)) { atomic_add_fetch_64((int64_t *)&pActivity->numOfSlowQueries, 1); + if (tsSlowLogScope & reqType) { + taosPrintSlowLog("PID:%d, Conn:%u, QID:0x%" PRIx64 ", Start:%" PRId64 ", Duration:%" PRId64 "us, SQL:%s", + taosGetPId(), pTscObj->connId, pRequest->requestId, pRequest->metric.start, duration, pRequest->sqlstr); + } } releaseTscObj(pTscObj->id); @@ -238,7 +244,7 @@ void destroyTscObj(void *pObj) { tscTrace("begin to destroy tscObj %" PRIx64 " p:%p", tscId, pTscObj); SClientHbKey connKey = {.tscRid = pTscObj->id, .connType = pTscObj->connType}; - hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey); + hbDeregisterConn(pTscObj->pAppInfo->pAppHbMgr, connKey, pTscObj->passInfo.fp); destroyAllRequests(pTscObj->pRequests); taosHashCleanup(pTscObj->pRequests); @@ -426,8 +432,12 @@ static void *tscCrashReportThreadFp(void *param) { } #endif + if (-1 != atomic_val_compare_exchange_32(&clientStop, -1, 0)) { + return NULL; + } + while (1) { - if (clientStop) break; + if (clientStop > 0) break; if (loopTimes++ < reportPeriodNum) { taosMsleep(sleepTime); continue; @@ -465,7 +475,7 @@ static void *tscCrashReportThreadFp(void *param) { loopTimes = 0; } - clientStop = -1; + clientStop = -2; return NULL; } diff --git a/source/client/src/clientHb.c b/source/client/src/clientHb.c index c9c2e7a5f82453035100f2e4a67d806c70a7ff57..79435da89fb5f85fa0fdace11e573e7c85c716b9 100644 --- a/source/client/src/clientHb.c +++ b/source/client/src/clientHb.c @@ -19,6 +19,15 @@ #include "scheduler.h" #include "trpc.h" +typedef struct { + union { + struct { + int64_t clusterId; + int32_t passKeyCnt; + }; + }; +} SHbParam; + static SClientHbMgr clientHbMgr = {0}; static int32_t hbCreateThread(); @@ -49,6 +58,52 @@ static int32_t hbProcessUserAuthInfoRsp(void *value, int32_t valueLen, struct SC return TSDB_CODE_SUCCESS; } +static int32_t hbProcessUserPassInfoRsp(void *value, int32_t valueLen, SClientHbKey *connKey, SAppHbMgr *pAppHbMgr) { + int32_t code = 0; + int32_t numOfBatchs = 0; + SUserPassBatchRsp batchRsp = {0}; + if (tDeserializeSUserPassBatchRsp(value, valueLen, &batchRsp) != 0) { + code = TSDB_CODE_INVALID_MSG; + return code; + } + + numOfBatchs = taosArrayGetSize(batchRsp.pArray); + + SClientHbReq *pReq = NULL; + while ((pReq = taosHashIterate(pAppHbMgr->activeInfo, pReq))) { + STscObj *pTscObj = (STscObj *)acquireTscObj(pReq->connKey.tscRid); + if (!pTscObj) { + continue; + } + SPassInfo *passInfo = &pTscObj->passInfo; + if (!passInfo->fp) { + releaseTscObj(pReq->connKey.tscRid); + continue; + } + + for (int32_t i = 0; i < numOfBatchs; ++i) { + SGetUserPassRsp *rsp = taosArrayGet(batchRsp.pArray, i); + if (0 == strncmp(rsp->user, pTscObj->user, TSDB_USER_LEN)) { + int32_t oldVer = atomic_load_32(&passInfo->ver); + if (oldVer < rsp->version) { + atomic_store_32(&passInfo->ver, rsp->version); + if (passInfo->fp) { + (*passInfo->fp)(passInfo->param, &passInfo->ver, TAOS_NOTIFY_PASSVER); + } + tscDebug("update passVer of user %s from %d to %d, tscRid:%" PRIi64, rsp->user, oldVer, + atomic_load_32(&passInfo->ver), pTscObj->id); + } + break; + } + } + releaseTscObj(pReq->connKey.tscRid); + } + + taosArrayDestroy(batchRsp.pArray); + + return code; +} + static int32_t hbGenerateVgInfoFromRsp(SDBVgInfo **pInfo, SUseDbRsp *rsp) { int32_t code = 0; SDBVgInfo *vgInfo = taosMemoryCalloc(1, sizeof(SDBVgInfo)); @@ -291,6 +346,15 @@ static int32_t hbQueryHbRspHandle(SAppHbMgr *pAppHbMgr, SClientHbRsp *pRsp) { hbProcessStbInfoRsp(kv->value, kv->valueLen, pCatalog); break; } + case HEARTBEAT_KEY_USER_PASSINFO: { + if (kv->valueLen <= 0 || NULL == kv->value) { + tscError("invalid hb user pass info, len:%d, value:%p", kv->valueLen, kv->value); + break; + } + + hbProcessUserPassInfoRsp(kv->value, kv->valueLen, &pRsp->connKey, pAppHbMgr); + break; + } default: tscError("invalid hb key type:%d", kv->key); break; @@ -472,6 +536,49 @@ int32_t hbGetQueryBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { return TSDB_CODE_SUCCESS; } +static int32_t hbGetUserBasicInfo(SClientHbKey *connKey, SClientHbReq *req) { + STscObj *pTscObj = (STscObj *)acquireTscObj(connKey->tscRid); + if (!pTscObj) { + tscWarn("tscObj rid %" PRIx64 " not exist", connKey->tscRid); + return TSDB_CODE_APP_ERROR; + } + + int32_t code = 0; + SUserPassVersion *user = taosMemoryMalloc(sizeof(SUserPassVersion)); + if (!user) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _return; + } + strncpy(user->user, pTscObj->user, TSDB_USER_LEN); + user->version = htonl(pTscObj->passInfo.ver); + + SKv kv = { + .key = HEARTBEAT_KEY_USER_PASSINFO, + .valueLen = sizeof(SUserPassVersion), + .value = user, + }; + + tscDebug("hb got user basic info, valueLen:%d, user:%s, passVer:%d, tscRid:%" PRIi64, kv.valueLen, user->user, + pTscObj->passInfo.ver, connKey->tscRid); + + if (!req->info) { + req->info = taosHashInit(64, hbKeyHashFunc, 1, HASH_ENTRY_LOCK); + } + + if (taosHashPut(req->info, &kv.key, sizeof(kv.key), &kv, sizeof(kv)) < 0) { + code = terrno ? terrno : TSDB_CODE_APP_ERROR; + goto _return; + } + +_return: + releaseTscObj(connKey->tscRid); + if (code) { + tscError("hb got user basic info failed since %s", terrstr(code)); + } + + return code; +} + int32_t hbGetExpiredUserInfo(SClientHbKey *connKey, struct SCatalog *pCatalog, SClientHbReq *req) { SUserAuthVersion *users = NULL; uint32_t userNum = 0; @@ -607,19 +714,23 @@ int32_t hbGetAppInfo(int64_t clusterId, SClientHbReq *req) { } int32_t hbQueryHbReqHandle(SClientHbKey *connKey, void *param, SClientHbReq *req) { - int64_t *clusterId = (int64_t *)param; + SHbParam *hbParam = (SHbParam *)param; struct SCatalog *pCatalog = NULL; - int32_t code = catalogGetHandle(*clusterId, &pCatalog); + int32_t code = catalogGetHandle(hbParam->clusterId, &pCatalog); if (code != TSDB_CODE_SUCCESS) { - tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", *clusterId, tstrerror(code)); + tscWarn("catalogGetHandle failed, clusterId:%" PRIx64 ", error:%s", hbParam->clusterId, tstrerror(code)); return code; } - hbGetAppInfo(*clusterId, req); + hbGetAppInfo(hbParam->clusterId, req); hbGetQueryBasicInfo(connKey, req); + if (hbParam->passKeyCnt > 0) { + hbGetUserBasicInfo(connKey, req); + } + code = hbGetExpiredUserInfo(connKey, pCatalog, req); if (TSDB_CODE_SUCCESS != code) { return code; @@ -673,7 +784,26 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { while (pIter != NULL) { pOneReq = taosArrayPush(pBatchReq->reqs, pOneReq); - code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, &pOneReq->clusterId, pOneReq); + SHbParam param; + switch (pOneReq->connKey.connType) { + case CONN_TYPE__QUERY: { + param.clusterId = pOneReq->clusterId; + param.passKeyCnt = atomic_load_32(&pAppHbMgr->passKeyCnt); + break; + } + default: + break; + } + if (clientHbMgr.reqHandle[pOneReq->connKey.connType]) { + code = (*clientHbMgr.reqHandle[pOneReq->connKey.connType])(&pOneReq->connKey, ¶m, pOneReq); + if (code) { + tscWarn("hbGatherAllInfo failed since %s, tscRid:%" PRIi64 ", connType:%" PRIi8, tstrerror(code), + pOneReq->connKey.tscRid, pOneReq->connKey.connType); + } + } + break; + +#if 0 if (code) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; @@ -682,6 +812,7 @@ SClientHbBatchReq *hbGatherAllInfo(SAppHbMgr *pAppHbMgr) { pIter = taosHashIterate(pAppHbMgr->activeInfo, pIter); pOneReq = pIter; +#endif } releaseTscObj(rid); @@ -1023,7 +1154,7 @@ int hbRegisterConn(SAppHbMgr *pAppHbMgr, int64_t tscRefId, int64_t clusterId, in } } -void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { +void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey, void *param) { SClientHbReq *pReq = taosHashAcquire(pAppHbMgr->activeInfo, &connKey, sizeof(SClientHbKey)); if (pReq) { tFreeClientHbReq(pReq); @@ -1036,4 +1167,7 @@ void hbDeregisterConn(SAppHbMgr *pAppHbMgr, SClientHbKey connKey) { } atomic_sub_fetch_32(&pAppHbMgr->connKeyCnt, 1); + if (param) { + atomic_sub_fetch_32(&pAppHbMgr->passKeyCnt, 1); + } } diff --git a/source/client/src/clientImpl.c b/source/client/src/clientImpl.c index dac44bd9c4511261124d43d19db214cac3300b49..f8eade1d7c62f3da22d765aea5b98622b34a0ba7 100644 --- a/source/client/src/clientImpl.c +++ b/source/client/src/clientImpl.c @@ -1039,8 +1039,7 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat .sysInfo = pRequest->pTscObj->sysInfo, .allocatorId = pRequest->allocatorRefId}; - SAppInstInfo* pAppInfo = getAppInfo(pRequest); - SQueryPlan* pDag = NULL; + SQueryPlan* pDag = NULL; int64_t st = taosGetTimestampUs(); int32_t code = qCreateQueryPlan(&cxt, &pDag, pMnodeList); @@ -1052,7 +1051,6 @@ static int32_t asyncExecSchQuery(SRequestObj* pRequest, SQuery* pQuery, SMetaDat } pRequest->metric.execStart = taosGetTimestampUs(); - pRequest->metric.planCostUs = pRequest->metric.execStart - st; if (TSDB_CODE_SUCCESS == code && !pRequest->validateOnly) { @@ -1250,6 +1248,11 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t return NULL; } + pRequest->sqlstr = taosStrdup("taos_connect"); + if (pRequest->sqlstr) { + pRequest->sqlLen = strlen(pRequest->sqlstr); + } + SMsgSendInfo* body = buildConnectMsg(pRequest); int64_t transporterId = 0; @@ -1259,7 +1262,7 @@ STscObj* taosConnectImpl(const char* user, const char* auth, const char* db, __t if (pRequest->code != TSDB_CODE_SUCCESS) { const char* errorMsg = (pRequest->code == TSDB_CODE_RPC_FQDN_ERROR) ? taos_errstr(pRequest) : tstrerror(pRequest->code); - fprintf(stderr, "failed to connect to server, reason: %s\n\n", errorMsg); + tscError("failed to connect to server, reason: %s", errorMsg); terrno = pRequest->code; destroyRequest(pRequest); diff --git a/source/client/src/clientJniConnector.c b/source/client/src/clientJniConnector.c index 2f4bafe26aa29a48e7c51c82275f14dba15a400d..b61335475125955b801b5b697992e79c98010431 100644 --- a/source/client/src/clientJniConnector.c +++ b/source/client/src/clientJniConnector.c @@ -816,7 +816,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setBindTableNameI (*env)->ReleaseStringUTFChars(env, jname, name); jniError("bindTableName jobj:%p, conn:%p, code: 0x%x", jobj, tsconn, code); - return JNI_TDENGINE_ERROR; + return code; } jniDebug("jobj:%p, conn:%p, set stmt bind table name:%s", jobj, tsconn, name); @@ -891,7 +891,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_setTableNameTagsI if (code != TSDB_CODE_SUCCESS) { jniError("tableNameTags jobj:%p, conn:%p, code: 0x%x", jobj, tsconn, code); - return JNI_TDENGINE_ERROR; + return code; } return JNI_SUCCESS; } @@ -957,7 +957,7 @@ JNIEXPORT jlong JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_bindColDataImp( if (code != TSDB_CODE_SUCCESS) { jniError("bindColData jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); - return JNI_TDENGINE_ERROR; + return code; } return JNI_SUCCESS; @@ -980,7 +980,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_addBatchImp(JNIEn int32_t code = taos_stmt_add_batch(pStmt); if (code != TSDB_CODE_SUCCESS) { jniError("add batch jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); - return JNI_TDENGINE_ERROR; + return code; } jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); @@ -1004,7 +1004,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_executeBatchImp(J int32_t code = taos_stmt_execute(pStmt); if (code != TSDB_CODE_SUCCESS) { jniError("excute batch jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); - return JNI_TDENGINE_ERROR; + return code; } jniDebug("jobj:%p, conn:%p, batch execute", jobj, tscon); @@ -1028,7 +1028,7 @@ JNIEXPORT jint JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_closeStmt(JNIEnv int32_t code = taos_stmt_close(pStmt); if (code != TSDB_CODE_SUCCESS) { jniError("close stmt jobj:%p, conn:%p, code: 0x%x", jobj, tscon, code); - return JNI_TDENGINE_ERROR; + return code; } jniDebug("jobj:%p, conn:%p, stmt closed", jobj, tscon); @@ -1259,8 +1259,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1286,8 +1287,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1315,8 +1317,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); @@ -1343,8 +1346,9 @@ JNIEXPORT jobject JNICALL Java_com_taosdata_jdbc_TSDBJNIConnector_schemalessInse int code = taos_errno(tres); if (code != TSDB_CODE_SUCCESS) { jniError("jobj:%p, conn:%p, code: 0x%x, msg:%s", jobj, taos, code, taos_errstr(tres)); + jobject jobject = createSchemalessResp(env, 0, code, taos_errstr(tres)); taos_free_result(tres); - return createSchemalessResp(env, 0, code, taos_errstr(tres)); + return jobject; } taos_free_result(tres); diff --git a/source/client/src/clientMain.c b/source/client/src/clientMain.c index 60c7b44b3d078e4102c6a6a5920f4fce61f8afbc..aed11b4fb153983e02d05d2d2d59f5d4a0b7a94d 100644 --- a/source/client/src/clientMain.c +++ b/source/client/src/clientMain.c @@ -119,6 +119,39 @@ TAOS *taos_connect(const char *ip, const char *user, const char *pass, const cha return NULL; } +int taos_set_notify_cb(TAOS *taos, __taos_notify_fn_t fp, void *param, int type) { + if (taos == NULL) { + terrno = TSDB_CODE_INVALID_PARA; + return terrno; + } + + STscObj *pObj = acquireTscObj(*(int64_t *)taos); + if (NULL == pObj) { + terrno = TSDB_CODE_TSC_DISCONNECTED; + tscError("invalid parameter for %s", __func__); + return terrno; + } + + switch (type) { + case TAOS_NOTIFY_PASSVER: { + pObj->passInfo.fp = fp; + pObj->passInfo.param = param; + if (fp) { + atomic_add_fetch_32(&pObj->pAppInfo->pAppHbMgr->passKeyCnt, 1); + } + break; + } + default: { + terrno = TSDB_CODE_INVALID_PARA; + releaseTscObj(*(int64_t *)taos); + return terrno; + } + } + + releaseTscObj(*(int64_t *)taos); + return 0; +} + void taos_close_internal(void *taos) { if (taos == NULL) { return; diff --git a/source/client/src/clientMsgHandler.c b/source/client/src/clientMsgHandler.c index a0146cfa39fee78fee7287526a3a1c7ea7a8c2da..52c5fc79408a05cf499d33059c6e7c07d176b615 100644 --- a/source/client/src/clientMsgHandler.c +++ b/source/client/src/clientMsgHandler.c @@ -130,6 +130,7 @@ int32_t processConnectRsp(void* param, SDataBuf* pMsg, int32_t code) { lastClusterId = connectRsp.clusterId; pTscObj->connType = connectRsp.connType; + pTscObj->passInfo.ver = connectRsp.passVer; hbRegisterConn(pTscObj->pAppInfo->pAppHbMgr, pTscObj->id, connectRsp.clusterId, connectRsp.connType); diff --git a/source/client/src/clientRawBlockWrite.c b/source/client/src/clientRawBlockWrite.c index a09780dc153db0512d3c2678646397a68cafb53e..32f28e456319cd67a4808fa9f0ffefaedc1d3dbf 100644 --- a/source/client/src/clientRawBlockWrite.c +++ b/source/client/src/clientRawBlockWrite.c @@ -1511,7 +1511,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { rspObj.resType = RES_TYPE__TMQ; tDecoderInit(&decoder, data, dataLen); - code = tDecodeSMqDataRsp(&decoder, &rspObj.rsp); + code = tDecodeMqDataRsp(&decoder, &rspObj.rsp); if (code != 0) { uError("WriteRaw:decode smqDataRsp error"); code = TSDB_CODE_INVALID_MSG; @@ -1615,7 +1615,7 @@ static int32_t tmqWriteRawDataImpl(TAOS* taos, void* data, int32_t dataLen) { code = pRequest->code; end: - tDeleteSMqDataRsp(&rspObj.rsp); + tDeleteMqDataRsp(&rspObj.rsp); tDecoderClear(&decoder); qDestroyQuery(pQuery); destroyRequest(pRequest); @@ -1858,7 +1858,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { int32_t len = 0; int32_t code = 0; - tEncodeSize(tEncodeSMqDataRsp, &rspObj->rsp, len, code); + tEncodeSize(tEncodeMqDataRsp, &rspObj->rsp, len, code); if (code < 0) { return -1; } @@ -1866,7 +1866,7 @@ int32_t tmq_get_raw(TAOS_RES* res, tmq_raw_data* raw) { void* buf = taosMemoryCalloc(1, len); SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - tEncodeSMqDataRsp(&encoder, &rspObj->rsp); + tEncodeMqDataRsp(&encoder, &rspObj->rsp); tEncoderClear(&encoder); raw->raw = buf; diff --git a/source/client/src/clientSml.c b/source/client/src/clientSml.c index ef7bd546c1a911200e1fa0e878ba21b43e0b19e0..011a1b31119c2a69c50dd83d85a948388d2e25c0 100644 --- a/source/client/src/clientSml.c +++ b/source/client/src/clientSml.c @@ -174,6 +174,7 @@ static int32_t smlParseTableName(SArray *tags, char *childTableName) { if (childTableNameLen == tag->keyLen && strncmp(tag->key, tsSmlChildTableName, tag->keyLen) == 0) { memset(childTableName, 0, TSDB_TABLE_NAME_LEN); strncpy(childTableName, tag->value, (tag->length < TSDB_TABLE_NAME_LEN ? tag->length : TSDB_TABLE_NAME_LEN)); + taosArrayRemove(tags, i); break; } } @@ -533,8 +534,7 @@ static int32_t smlGenerateSchemaAction(SSchema *colField, SHashObj *colHash, SSm uint16_t *index = colHash ? (uint16_t *)taosHashGet(colHash, kv->key, kv->keyLen) : NULL; if (index) { if (colField[*index].type != kv->type) { - uError("SML:0x%" PRIx64 " point type and db type mismatch. key: %s. point type: %d, db type: %d", info->id, - kv->key, colField[*index].type, kv->type); + uError("SML:0x%" PRIx64 " point type and db type mismatch. point type: %d, db type: %d, key: %s", info->id, colField[*index].type, kv->type, kv->key); return TSDB_CODE_TSC_INVALID_VALUE; } @@ -660,6 +660,7 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, SMCreateStbReq pReq = {0}; int32_t code = TSDB_CODE_SUCCESS; SCmdMsgInfo pCmdMsg = {0}; + char *pSql = NULL; // put front for free pReq.numOfColumns = taosArrayGetSize(pColumns); @@ -667,39 +668,42 @@ static int32_t smlSendMetaMsg(SSmlHandle *info, SName *pName, SArray *pColumns, pReq.numOfTags = taosArrayGetSize(pTags); pReq.pTags = pTags; - code = buildRequest(info->taos->id, "", 0, NULL, false, &pRequest, 0); - if (code != TSDB_CODE_SUCCESS) { - goto end; - } - - pRequest->syncQuery = true; - if (!pRequest->pDb) { - code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; - goto end; - } - if (action == SCHEMA_ACTION_CREATE_STABLE) { pReq.colVer = 1; pReq.tagVer = 1; pReq.suid = 0; pReq.source = TD_REQ_FROM_APP; + pSql = "sml_create_stable"; } else if (action == SCHEMA_ACTION_ADD_TAG || action == SCHEMA_ACTION_CHANGE_TAG_SIZE) { pReq.colVer = pTableMeta->sversion; pReq.tagVer = pTableMeta->tversion + 1; pReq.suid = pTableMeta->uid; pReq.source = TD_REQ_FROM_TAOX; + pSql = (action == SCHEMA_ACTION_ADD_TAG) ? "sml_add_tag" : "sml_modify_tag_size"; } else if (action == SCHEMA_ACTION_ADD_COLUMN || action == SCHEMA_ACTION_CHANGE_COLUMN_SIZE) { pReq.colVer = pTableMeta->sversion + 1; pReq.tagVer = pTableMeta->tversion; pReq.suid = pTableMeta->uid; pReq.source = TD_REQ_FROM_TAOX; + pSql = (action == SCHEMA_ACTION_ADD_COLUMN) ? "sml_add_column" : "sml_modify_column_size"; + } + + code = buildRequest(info->taos->id, pSql, strlen(pSql), NULL, false, &pRequest, 0); + if (code != TSDB_CODE_SUCCESS) { + goto end; + } + + pRequest->syncQuery = true; + if (!pRequest->pDb) { + code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; + goto end; } if (pReq.numOfTags == 0) { pReq.numOfTags = 1; SField field = {0}; field.type = TSDB_DATA_TYPE_NCHAR; - field.bytes = 1; + field.bytes = TSDB_NCHAR_SIZE + VARSTR_HEADER_SIZE; strcpy(field.name, tsSmlTagName); taosArrayPush(pReq.pTags, &field); } @@ -1514,6 +1518,44 @@ static int smlProcess(SSmlHandle *info, char *lines[], char *rawLine, char *rawL return code; } +void smlSetReqSQL(SRequestObj *request, char *lines[], char *rawLine, char *rawLineEnd) { + if (tsSlowLogScope & SLOW_LOG_TYPE_INSERT) { + int32_t len = 0; + int32_t rlen = 0; + char* p = NULL; + + if (lines && lines[0]) { + len = strlen(lines[0]); + p = lines[0]; + } else if (rawLine) { + if (rawLineEnd) { + len = rawLineEnd - rawLine; + } else { + len = strlen(rawLine); + } + p = rawLine; + } + + if (NULL == p) { + return; + } + + rlen = TMIN(len, TSDB_MAX_ALLOWED_SQL_LEN); + rlen = TMAX(rlen, 0); + + char *sql = taosMemoryMalloc(rlen + 1); + if (NULL == sql) { + uError("malloc %d for sml sql failed", rlen + 1); + return; + } + memcpy(sql, p, rlen); + sql[rlen] = 0; + + request->sqlstr = sql; + request->sqlLen = rlen; + } +} + TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, char *rawLineEnd, int numLines, int protocol, int precision, int32_t ttl, int64_t reqid) { int32_t code = TSDB_CODE_SUCCESS; @@ -1546,6 +1588,8 @@ TAOS_RES *taos_schemaless_insert_inner(TAOS *taos, char *lines[], char *rawLine, info->msgBuf.len = ERROR_MSG_BUF_DEFAULT_SIZE; info->lineNum = numLines; + smlSetReqSQL(request, lines, rawLine, rawLineEnd); + SSmlMsgBuf msg = {ERROR_MSG_BUF_DEFAULT_SIZE, request->msgBuf}; if (request->pDb == NULL) { request->code = TSDB_CODE_PAR_DB_NOT_SPECIFIED; diff --git a/source/client/src/clientStmt.c b/source/client/src/clientStmt.c index 3ed157efef1fc84b58cfe4d566965123108047cd..6e529f1a0b44556d5f07e750f050f83711fdc0b2 100644 --- a/source/client/src/clientStmt.c +++ b/source/client/src/clientStmt.c @@ -32,8 +32,14 @@ int32_t stmtSwitchStatus(STscStmt* pStmt, STMT_STATUS newStatus) { STMT_LOG_SEQ(newStatus); } + if (pStmt->errCode && newStatus != STMT_PREPARE) { + STMT_DLOG("stmt already failed with err: %s", tstrerror(pStmt->errCode)); + return pStmt->errCode; + } + switch (newStatus) { case STMT_PREPARE: + pStmt->errCode = 0; break; case STMT_SETTBNAME: if (STMT_STATUS_EQ(INIT) || STMT_STATUS_EQ(BIND) || STMT_STATUS_EQ(BIND_COL)) { @@ -197,7 +203,10 @@ int32_t stmtGetExecInfo(TAOS_STMT* stmt, SHashObj** pVgHash, SHashObj** pBlockHa STscStmt* pStmt = (STscStmt*)stmt; *pVgHash = pStmt->sql.pVgHash; + pStmt->sql.pVgHash = NULL; + *pBlockHash = pStmt->exec.pBlockHash; + pStmt->exec.pBlockHash = NULL; return TSDB_CODE_SUCCESS; } @@ -325,6 +334,8 @@ int32_t stmtCleanExecInfo(STscStmt* pStmt, bool keepTable, bool deepClean) { } int32_t stmtCleanSQLInfo(STscStmt* pStmt) { + STMT_DLOG_E("start to free SQL info"); + taosMemoryFree(pStmt->sql.queryRes.fields); taosMemoryFree(pStmt->sql.queryRes.userFields); taosMemoryFree(pStmt->sql.sqlStr); @@ -351,6 +362,8 @@ int32_t stmtCleanSQLInfo(STscStmt* pStmt) { memset(&pStmt->sql, 0, sizeof(pStmt->sql)); + STMT_DLOG_E("end to free SQL info"); + return TSDB_CODE_SUCCESS; } @@ -441,11 +454,10 @@ int32_t stmtGetFromCache(STscStmt* pStmt) { .mgmtEps = getEpSet_s(&pStmt->taos->pAppInfo->mgmtEp)}; int32_t code = catalogGetTableMeta(pStmt->pCatalog, &conn, &pStmt->bInfo.sname, &pTableMeta); if (TSDB_CODE_PAR_TABLE_NOT_EXIST == code) { - STMT_ERR_RET(stmtCleanBindInfo(pStmt)); - tscDebug("tb %s not exist", pStmt->bInfo.tbFName); + stmtCleanBindInfo(pStmt); - return TSDB_CODE_SUCCESS; + STMT_ERR_RET(code); } STMT_ERR_RET(code); @@ -922,9 +934,13 @@ _return: int stmtClose(TAOS_STMT* stmt) { STscStmt* pStmt = (STscStmt*)stmt; + STMT_DLOG_E("start to free stmt"); + stmtCleanSQLInfo(pStmt); taosMemoryFree(stmt); + STMT_DLOG_E("stmt freed"); + return TSDB_CODE_SUCCESS; } @@ -959,15 +975,17 @@ int stmtIsInsert(TAOS_STMT* stmt, int* insert) { } int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get tag fields"); if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { @@ -979,27 +997,33 @@ int stmtGetTagFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - STMT_ERR_RET(stmtCreateRequest(pStmt)); + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtFetchTagFields(stmt, nums, fields)); + STMT_ERRI_JRET(stmtFetchTagFields(stmt, nums, fields)); - return TSDB_CODE_SUCCESS; +_return: + + pStmt->errCode = preCode; + + return code; } int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { + int32_t code = 0; STscStmt* pStmt = (STscStmt*)stmt; + int32_t preCode = pStmt->errCode; STMT_DLOG_E("start to get col fields"); if (STMT_TYPE_QUERY == pStmt->sql.type) { - STMT_RET(TSDB_CODE_TSC_STMT_API_ERROR); + STMT_ERRI_JRET(TSDB_CODE_TSC_STMT_API_ERROR); } - STMT_ERR_RET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); + STMT_ERRI_JRET(stmtSwitchStatus(pStmt, STMT_FETCH_FIELDS)); if (pStmt->bInfo.needParse && pStmt->sql.runTimes && pStmt->sql.type > 0 && STMT_TYPE_MULTI_INSERT != pStmt->sql.type) { @@ -1011,15 +1035,19 @@ int stmtGetColFields(TAOS_STMT* stmt, int* nums, TAOS_FIELD_E** fields) { pStmt->exec.pRequest = NULL; } - STMT_ERR_RET(stmtCreateRequest(pStmt)); + STMT_ERRI_JRET(stmtCreateRequest(pStmt)); if (pStmt->bInfo.needParse) { - STMT_ERR_RET(stmtParseSql(pStmt)); + STMT_ERRI_JRET(stmtParseSql(pStmt)); } - STMT_ERR_RET(stmtFetchColFields(stmt, nums, fields)); + STMT_ERRI_JRET(stmtFetchColFields(stmt, nums, fields)); - return TSDB_CODE_SUCCESS; +_return: + + pStmt->errCode = preCode; + + return code; } int stmtGetParamNum(TAOS_STMT* stmt, int* nums) { diff --git a/source/client/src/clientTmq.c b/source/client/src/clientTmq.c index 9e43765b5b93cac7ef90ac6a11684305d9647c19..76fd1d84d0bec43ddea33172aecdcbdfa3ab68a3 100644 --- a/source/client/src/clientTmq.c +++ b/source/client/src/clientTmq.c @@ -98,17 +98,16 @@ struct tmq_t { int64_t totalRows; // timer - tmr_h hbLiveTimer; - tmr_h epTimer; - tmr_h reportTimer; - tmr_h commitTimer; - STscObj* pTscObj; // connection - SArray* clientTopics; // SArray - STaosQueue* mqueue; // queue of rsp - STaosQall* qall; - STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit - TdThreadMutex lock; // used to protect the operation on each topic, when updating the epsets. - tsem_t rspSem; + tmr_h hbLiveTimer; + tmr_h epTimer; + tmr_h reportTimer; + tmr_h commitTimer; + STscObj* pTscObj; // connection + SArray* clientTopics; // SArray + STaosQueue* mqueue; // queue of rsp + STaosQall* qall; + STaosQueue* delayedTask; // delayed task queue for heartbeat and auto commit + tsem_t rspSem; }; typedef struct SAskEpInfo { @@ -134,16 +133,23 @@ enum { TMQ_DELAYED_TASK__COMMIT, }; -typedef struct { - int64_t pollCnt; - int64_t numOfRows; +typedef struct SVgOffsetInfo { STqOffsetVal committedOffset; STqOffsetVal currentOffset; - int32_t vgId; - int32_t vgStatus; - int32_t vgSkipCnt; - int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data - SEpSet epSet; + int64_t walVerBegin; + int64_t walVerEnd; +} SVgOffsetInfo; + +typedef struct { + int64_t pollCnt; + int64_t numOfRows; + SVgOffsetInfo offsetInfo; + int32_t vgId; + int32_t vgStatus; + int32_t vgSkipCnt; // here used to mark the slow vgroups + bool receiveInfo; + int64_t emptyBlockReceiveTs; // once empty block is received, idle for ignoreCnt then start to poll data + SEpSet epSet; } SMqClientVg; typedef struct { @@ -188,10 +194,26 @@ typedef struct { SMqClientVg* pVg; SMqClientTopic* pTopic; int32_t vgId; - tsem_t rspSem; uint64_t requestId; // request id for debug purpose } SMqPollCbParam; +typedef struct SMqVgCommon { + tsem_t rsp; + int32_t numOfRsp; + SArray* pList; + TdThreadMutex mutex; + int64_t consumerId; + char* pTopicName; + int32_t code; +} SMqVgCommon; + +typedef struct SMqVgWalInfoParam { + int32_t vgId; + int32_t epoch; + int32_t totalReq; + SMqVgCommon* pCommon; +} SMqVgWalInfoParam; + typedef struct { int64_t refId; int32_t epoch; @@ -206,17 +228,22 @@ typedef struct { typedef struct { SMqCommitCbParamSet* params; - STqOffset* pOffset; + SMqVgOffset* pOffset; char topicName[TSDB_TOPIC_FNAME_LEN]; int32_t vgId; tmq_t* pTmq; } SMqCommitCbParam; +typedef struct SSyncCommitInfo { + tsem_t sem; + int32_t code; +} SSyncCommitInfo; + static int32_t doAskEp(tmq_t* tmq); static int32_t makeTopicVgroupKey(char* dst, const char* topicName, int32_t vg); static int32_t tmqCommitDone(SMqCommitCbParamSet* pParamSet); static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, - int32_t index, int32_t totalVgroups); + int32_t index, int32_t totalVgroups, int32_t type); static void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, const char* pTopic, int32_t vgId); static void asyncAskEp(tmq_t* pTmq, __tmq_askep_fn_t askEpFn, void* param); static void addToQueueCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* param); @@ -438,7 +465,7 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { // if (code1 != TSDB_CODE_SUCCESS) { // retry failed. // tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 // " retry failed, ignore this commit. code:%s ordinal:%d/%d", - // pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->committedOffset.version, + // pParam->pTmq->consumerId, pParam->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, // tstrerror(terrno), index + 1, numOfVgroups); // } // } @@ -464,22 +491,23 @@ static int32_t tmqCommitCb(void* param, SDataBuf* pBuf, int32_t code) { } static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicName, SMqCommitCbParamSet* pParamSet, - int32_t index, int32_t totalVgroups) { - STqOffset* pOffset = taosMemoryCalloc(1, sizeof(STqOffset)); + int32_t index, int32_t totalVgroups, int32_t type) { + SMqVgOffset* pOffset = taosMemoryCalloc(1, sizeof(SMqVgOffset)); if (pOffset == NULL) { return TSDB_CODE_OUT_OF_MEMORY; } - pOffset->val = pVg->currentOffset; + pOffset->consumerId = tmq->consumerId; + pOffset->offset.val = pVg->offsetInfo.currentOffset; int32_t groupLen = strlen(tmq->groupId); - memcpy(pOffset->subKey, tmq->groupId, groupLen); - pOffset->subKey[groupLen] = TMQ_SEPARATOR; - strcpy(pOffset->subKey + groupLen + 1, pTopicName); + memcpy(pOffset->offset.subKey, tmq->groupId, groupLen); + pOffset->offset.subKey[groupLen] = TMQ_SEPARATOR; + strcpy(pOffset->offset.subKey + groupLen + 1, pTopicName); int32_t len = 0; int32_t code = 0; - tEncodeSize(tEncodeSTqOffset, pOffset, len, code); + tEncodeSize(tEncodeMqVgOffset, pOffset, len, code); if (code < 0) { return TSDB_CODE_INVALID_PARA; } @@ -496,7 +524,7 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN SEncoder encoder; tEncoderInit(&encoder, abuf, len); - tEncodeSTqOffset(&encoder, pOffset); + tEncodeMqVgOffset(&encoder, pOffset); tEncoderClear(&encoder); // build param @@ -523,30 +551,26 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN return TSDB_CODE_OUT_OF_MEMORY; } - pMsgSendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = sizeof(SMsgHead) + len, - .handle = NULL, - }; + pMsgSendInfo->msgInfo = (SDataBuf) { .pData = buf, .len = sizeof(SMsgHead) + len, .handle = NULL }; pMsgSendInfo->requestId = generateRequestId(); pMsgSendInfo->requestObjRefId = 0; pMsgSendInfo->param = pParam; pMsgSendInfo->paramFreeFp = taosMemoryFree; pMsgSendInfo->fp = tmqCommitCb; - pMsgSendInfo->msgType = TDMT_VND_TMQ_COMMIT_OFFSET; + pMsgSendInfo->msgType = type; atomic_add_fetch_32(&pParamSet->waitingRspNum, 1); atomic_add_fetch_32(&pParamSet->totalRspNum, 1); SEp* pEp = GET_ACTIVE_EP(&pVg->epSet); char offsetBuf[80] = {0}; - tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->val); + tFormatOffset(offsetBuf, tListLen(offsetBuf), &pOffset->offset.val); char commitBuf[80] = {0}; - tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->committedOffset); + tFormatOffset(commitBuf, tListLen(commitBuf), &pVg->offsetInfo.committedOffset); tscDebug("consumer:0x%" PRIx64 " topic:%s on vgId:%d send offset:%s prev:%s, ep:%s:%d, ordinal:%d/%d, req:0x%" PRIx64, - tmq->consumerId, pOffset->subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, + tmq->consumerId, pOffset->offset.subKey, pVg->vgId, offsetBuf, commitBuf, pEp->fqdn, pEp->port, index + 1, totalVgroups, pMsgSendInfo->requestId); int64_t transporterId = 0; @@ -555,7 +579,22 @@ static int32_t doSendCommitMsg(tmq_t* tmq, SMqClientVg* pVg, const char* pTopicN return TSDB_CODE_SUCCESS; } -static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* pCommitFp, void* userParam) { +static SMqClientTopic* getTopicByName(tmq_t* tmq, const char* pTopicName) { + int32_t numOfTopics = taosArrayGetSize(tmq->clientTopics); + for (int32_t i = 0; i < numOfTopics; ++i) { + SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); + if (strcmp(pTopic->topicName, pTopicName) != 0) { + continue; + } + + return pTopic; + } + + tscError("consumer:0x%" PRIx64 ", total:%d, failed to find topic:%s", tmq->consumerId, numOfTopics, pTopicName); + return NULL; +} + +static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, int32_t type, tmq_commit_cb* pCommitFp, void* userParam) { char* pTopicName = NULL; int32_t vgId = 0; int32_t code = 0; @@ -597,15 +636,8 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p tscDebug("consumer:0x%" PRIx64 " do manual commit offset for %s, vgId:%d", tmq->consumerId, pTopicName, vgId); - int32_t i = 0; - for (; i < numOfTopics; i++) { - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - if (strcmp(pTopic->topicName, pTopicName) == 0) { - break; - } - } - - if (i == numOfTopics) { + SMqClientTopic* pTopic = getTopicByName(tmq, pTopicName); + if (pTopic == NULL) { tscWarn("consumer:0x%" PRIx64 " failed to find the specified topic:%s, total topics:%d", tmq->consumerId, pTopicName, numOfTopics); taosMemoryFree(pParamSet); @@ -613,8 +645,6 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p return; } - SMqClientTopic* pTopic = taosArrayGet(tmq->clientTopics, i); - int32_t j = 0; int32_t numOfVgroups = taosArrayGetSize(pTopic->vgs); for (j = 0; j < numOfVgroups; j++) { @@ -633,8 +663,8 @@ static void asyncCommitOffset(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* p } SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups); + if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { + code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, type); // failed to commit, callback user function directly. if (code != TSDB_CODE_SUCCESS) { @@ -674,20 +704,20 @@ static void asyncCommitAllOffsets(tmq_t* tmq, tmq_commit_cb* pCommitFp, void* us for (int32_t j = 0; j < numOfVgroups; j++) { SMqClientVg* pVg = taosArrayGet(pTopic->vgs, j); - if (pVg->currentOffset.type > 0 && !tOffsetEqual(&pVg->currentOffset, &pVg->committedOffset)) { - int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups); + if (pVg->offsetInfo.currentOffset.type > 0 && !tOffsetEqual(&pVg->offsetInfo.currentOffset, &pVg->offsetInfo.committedOffset)) { + int32_t code = doSendCommitMsg(tmq, pVg, pTopic->topicName, pParamSet, j, numOfVgroups, TDMT_VND_TMQ_COMMIT_OFFSET); if (code != TSDB_CODE_SUCCESS) { tscError("consumer:0x%" PRIx64 " topic:%s vgId:%d offset:%" PRId64 " failed, code:%s ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->committedOffset.version, tstrerror(terrno), + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.committedOffset.version, tstrerror(terrno), j + 1, numOfVgroups); continue; } // update the offset value. - pVg->committedOffset = pVg->currentOffset; + pVg->offsetInfo.committedOffset = pVg->offsetInfo.currentOffset; } else { tscDebug("consumer:0x%" PRIx64 " topic:%s vgId:%d, no commit, current:%" PRId64 ", ordinal:%d/%d", - tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->currentOffset.version, j + 1, numOfVgroups); + tmq->consumerId, pTopic->topicName, pVg->vgId, pVg->offsetInfo.currentOffset.version, j + 1, numOfVgroups); } } } @@ -788,11 +818,7 @@ void tmqSendHbReq(void* param, void* tmrId) { goto OVER; } - sendInfo->msgInfo = (SDataBuf){ - .pData = pReq, - .len = tlen, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){ .pData = pReq, .len = tlen, .handle = NULL }; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; @@ -979,7 +1005,6 @@ void tmqFreeImpl(void* handle) { taosFreeQall(tmq->qall); tsem_destroy(&tmq->rspSem); - taosThreadMutexDestroy(&tmq->lock); taosArrayDestroyEx(tmq->clientTopics, freeClientVgImpl); taos_close_internal(tmq->pTscObj); @@ -1024,7 +1049,6 @@ tmq_t* tmq_consumer_new(tmq_conf_t* conf, char* errstr, int32_t errstrLen) { pTmq->delayedTask = taosOpenQueue(); pTmq->qall = taosAllocateQall(); - taosThreadMutexInit(&pTmq->lock, NULL); if (pTmq->clientTopics == NULL || pTmq->mqueue == NULL || pTmq->qall == NULL || pTmq->delayedTask == NULL || conf->groupId[0] == 0) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -1095,7 +1119,7 @@ _failed: } int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { - const int32_t MAX_RETRY_COUNT = 120 * 2; // let's wait for 2 mins at most + const int32_t MAX_RETRY_COUNT = 120 * 4; // let's wait for 4 mins at most const SArray* container = &topic_list->container; int32_t sz = taosArrayGetSize(container); void* buf = NULL; @@ -1148,21 +1172,13 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { goto FAIL; } - SMqSubscribeCbParam param = { - .rspErr = 0, - .refId = tmq->refId, - .epoch = tmq->epoch, - }; - + SMqSubscribeCbParam param = { .rspErr = 0, .refId = tmq->refId, .epoch = tmq->epoch }; if (tsem_init(¶m.rspSem, 0, 0) != 0) { + code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; } - sendInfo->msgInfo = (SDataBuf){ - .pData = buf, - .len = tlen, - .handle = NULL, - }; + sendInfo->msgInfo = (SDataBuf){.pData = buf, .len = tlen, .handle = NULL}; sendInfo->requestId = generateRequestId(); sendInfo->requestObjRefId = 0; @@ -1190,6 +1206,8 @@ int32_t tmq_subscribe(tmq_t* tmq, const tmq_list_t* topic_list) { int32_t retryCnt = 0; while (TSDB_CODE_MND_CONSUMER_NOT_READY == doAskEp(tmq)) { if (retryCnt++ > MAX_RETRY_COUNT) { + tscError("consumer:0x%" PRIx64 ", mnd not ready for subscribe, max retry reached:%d", tmq->consumerId, retryCnt); + code = TSDB_CODE_TSC_INTERNAL_ERROR; goto FAIL; } @@ -1224,7 +1242,7 @@ void tmq_conf_set_auto_commit_cb(tmq_conf_t* conf, tmq_commit_cb* cb, void* para conf->commitCbUserParam = param; } -int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { +static int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { SMqPollCbParam* pParam = (SMqPollCbParam*)param; int64_t refId = pParam->refId; @@ -1233,7 +1251,6 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { tmq_t* tmq = taosAcquireRef(tmqMgmt.rsetId, refId); if (tmq == NULL) { - tsem_destroy(&pParam->rspSem); taosMemoryFree(pParam); taosMemoryFree(pMsg->pData); taosMemoryFree(pMsg->pEpSet); @@ -1270,18 +1287,20 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { pRspWrapper->tmqRspType = TMQ_MSG_TYPE__END_RSP; taosWriteQitem(tmq->mqueue, pRspWrapper); + } else if (code == TSDB_CODE_WAL_LOG_NOT_EXIST) { // poll data while insert + taosMsleep(500); } goto CREATE_MSG_FAIL; } int32_t msgEpoch = ((SMqRspHead*)pMsg->pData)->epoch; - int32_t tmqEpoch = atomic_load_32(&tmq->epoch); - if (msgEpoch < tmqEpoch) { + int32_t clientEpoch = atomic_load_32(&tmq->epoch); + if (msgEpoch < clientEpoch) { // do not write into queue since updating epoch reset tscWarn("consumer:0x%" PRIx64 " msg discard from vgId:%d since from earlier epoch, rsp epoch %d, current epoch %d, reqId:0x%" PRIx64, - tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId); + tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); @@ -1291,9 +1310,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { return 0; } - if (msgEpoch != tmqEpoch) { + if (msgEpoch != clientEpoch) { tscWarn("consumer:0x%" PRIx64 " mismatch rsp from vgId:%d, epoch %d, current epoch %d, reqId:0x%" PRIx64, - tmq->consumerId, vgId, msgEpoch, tmqEpoch, requestId); + tmq->consumerId, vgId, msgEpoch, clientEpoch, requestId); } // handle meta rsp @@ -1319,7 +1338,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { if (rspType == TMQ_MSG_TYPE__POLL_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeSMqDataRsp(&decoder, &pRspWrapper->dataRsp); + tDecodeMqDataRsp(&decoder, &pRspWrapper->dataRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->dataRsp, pMsg->pData, sizeof(SMqRspHead)); @@ -1330,7 +1349,7 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { } else if (rspType == TMQ_MSG_TYPE__POLL_META_RSP) { SDecoder decoder; tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); - tDecodeSMqMetaRsp(&decoder, &pRspWrapper->metaRsp); + tDecodeMqMetaRsp(&decoder, &pRspWrapper->metaRsp); tDecoderClear(&decoder); memcpy(&pRspWrapper->metaRsp, pMsg->pData, sizeof(SMqRspHead)); } else if (rspType == TMQ_MSG_TYPE__TAOSX_RSP) { @@ -1346,8 +1365,9 @@ int32_t tmqPollCb(void* param, SDataBuf* pMsg, int32_t code) { taosMemoryFree(pMsg->pData); taosWriteQitem(tmq->mqueue, pRspWrapper); + int32_t total = taosQueueItemSize(tmq->mqueue); tscDebug("consumer:0x%" PRIx64 " put poll res into mqueue, type:%d, vgId:%d, total in queue:%d, reqId:0x%" PRIx64, - tmq->consumerId, rspType, vgId, tmq->mqueue->numOfItems, requestId); + tmq->consumerId, rspType, vgId, total, requestId); tsem_post(&tmq->rspSem); taosReleaseRef(tmqMgmt.rsetId, refId); @@ -1400,7 +1420,6 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic SMqClientVg clientVg = { .pollCnt = 0, - .currentOffset = offsetNew, .vgId = pVgEp->vgId, .epSet = pVgEp->epSet, .vgStatus = TMQ_VG_STATUS__IDLE, @@ -1409,6 +1428,10 @@ static void initClientTopicFromRsp(SMqClientTopic* pTopic, SMqSubTopicEp* pTopic .numOfRows = numOfRows, }; + clientVg.offsetInfo.currentOffset = offsetNew; + clientVg.offsetInfo.committedOffset = offsetNew; + clientVg.offsetInfo.walVerBegin = -1; + clientVg.offsetInfo.walVerEnd = -1; taosArrayPush(pTopic->vgs, &clientVg); } } @@ -1422,7 +1445,7 @@ static void freeClientVgInfo(void* param) { taosArrayDestroy(pTopic->vgs); } -static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { +static bool doUpdateLocalEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { bool set = false; int32_t topicNumCur = taosArrayGetSize(tmq->clientTopics); @@ -1431,6 +1454,9 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { char vgKey[TSDB_TOPIC_FNAME_LEN + 22]; tscDebug("consumer:0x%" PRIx64 " update ep epoch from %d to epoch %d, incoming topics:%d, existed topics:%d", tmq->consumerId, tmq->epoch, epoch, topicNumGet, topicNumCur); + if (epoch <= tmq->epoch) { + return false; + } SArray* newTopics = taosArrayInit(topicNumGet, sizeof(SMqClientTopic)); if (newTopics == NULL) { @@ -1455,11 +1481,11 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { makeTopicVgroupKey(vgKey, pTopicCur->topicName, pVgCur->vgId); char buf[80]; - tFormatOffset(buf, 80, &pVgCur->currentOffset); + tFormatOffset(buf, 80, &pVgCur->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 ", epoch:%d vgId:%d vgKey:%s, offset:%s", tmq->consumerId, epoch, pVgCur->vgId, vgKey, buf); - SVgroupSaveInfo info = {.offset = pVgCur->currentOffset, .numOfRows = pVgCur->numOfRows}; + SVgroupSaveInfo info = {.offset = pVgCur->offsetInfo.currentOffset, .numOfRows = pVgCur->numOfRows}; taosHashPut(pVgOffsetHashMap, vgKey, strlen(vgKey), &info, sizeof(SVgroupSaveInfo)); } } @@ -1474,14 +1500,11 @@ static bool tmqUpdateEp(tmq_t* tmq, int32_t epoch, const SMqAskEpRsp* pRsp) { taosHashCleanup(pVgOffsetHashMap); - taosThreadMutexLock(&tmq->lock); // destroy current buffered existed topics info if (tmq->clientTopics) { taosArrayDestroyEx(tmq->clientTopics, freeClientVgInfo); } - tmq->clientTopics = newTopics; - taosThreadMutexUnlock(&tmq->lock); int8_t flag = (topicNumGet == 0) ? TMQ_CONSUMER_STATUS__NO_TOPIC : TMQ_CONSUMER_STATUS__READY; atomic_store_8(&tmq->status, flag); @@ -1537,9 +1560,9 @@ int32_t askEpCallbackFn(void* param, SDataBuf* pMsg, int32_t code) { } else { tscDebug("consumer:0x%" PRIx64 ", recv ep, msg epoch %d, current epoch %d, update local ep", tmq->consumerId, head->epoch, epoch); - pParam->pUserFn(tmq, code, pMsg, pParam->pParam); } + pParam->pUserFn(tmq, code, pMsg, pParam->pParam); taosReleaseRef(tmqMgmt.rsetId, pParam->refId); taosMemoryFree(pMsg->pEpSet); @@ -1558,8 +1581,7 @@ void tmqBuildConsumeReqImpl(SMqPollReq* pReq, tmq_t* tmq, int64_t timeout, SMqCl pReq->consumerId = tmq->consumerId; pReq->timeout = timeout; pReq->epoch = tmq->epoch; - /*pReq->currentOffset = reqOffset;*/ - pReq->reqOffset = pVg->currentOffset; + pReq->reqOffset = pVg->offsetInfo.currentOffset; pReq->head.vgId = pVg->vgId; pReq->useSnapshot = tmq->useSnapshot; pReq->reqId = generateRequestId(); @@ -1657,7 +1679,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p pParam->refId = pTmq->refId; pParam->epoch = pTmq->epoch; - pParam->pVg = pVg; // pVg may be released,fix it + pParam->pVg = pVg; pParam->pTopic = pTopic; pParam->vgId = pVg->vgId; pParam->requestId = req.reqId; @@ -1669,12 +1691,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p return handleErrorBeforePoll(pVg, pTmq); } - sendInfo->msgInfo = (SDataBuf){ - .pData = msg, - .len = msgSize, - .handle = NULL, - }; - + sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; sendInfo->requestId = req.reqId; sendInfo->requestObjRefId = 0; sendInfo->param = pParam; @@ -1683,7 +1700,7 @@ static int32_t doTmqPollImpl(tmq_t* pTmq, SMqClientTopic* pTopic, SMqClientVg* p int64_t transporterId = 0; char offsetFormatBuf[80]; - tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->currentOffset); + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pVg->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 " send poll to %s vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, pTmq->consumerId, pTopic->topicName, pVg->vgId, pTmq->epoch, offsetFormatBuf, req.reqId); @@ -1718,13 +1735,6 @@ static int32_t tmqPollImpl(tmq_t* tmq, int64_t timeout) { tscTrace("consumer:0x%" PRIx64 " epoch %d wait poll-rsp, skip vgId:%d skip cnt %d", tmq->consumerId, tmq->epoch, pVg->vgId, vgSkipCnt); continue; -#if 0 - if (skipCnt < 30000) { - continue; - } else { - tscDebug("consumer:0x%" PRIx64 ",skip vgId:%d skip too much reset", tmq->consumerId, pVg->vgId); - } -#endif } atomic_store_32(&pVg->vgSkipCnt, 0); @@ -1745,7 +1755,7 @@ static int32_t tmqHandleNoPollRsp(tmq_t* tmq, SMqRspWrapper* rspWrapper, bool* p if (rspWrapper->epoch > atomic_load_32(&tmq->epoch)) { SMqAskEpRspWrapper* pEpRspWrapper = (SMqAskEpRspWrapper*)rspWrapper; SMqAskEpRsp* rspMsg = &pEpRspWrapper->msg; - tmqUpdateEp(tmq, rspWrapper->epoch, rspMsg); + doUpdateLocalEp(tmq, rspWrapper->epoch, rspMsg); /*tmqClearUnhandleMsg(tmq);*/ tDeleteSMqAskEpRsp(rspMsg); *pReset = true; @@ -1800,9 +1810,16 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { } // update the local offset value only for the returned values. - pVg->currentOffset = pDataRsp->rspOffset; + pVg->offsetInfo.currentOffset = pDataRsp->rspOffset; + + // update the status atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); + // update the valid wal version range + pVg->offsetInfo.walVerBegin = pDataRsp->head.walsver; + pVg->offsetInfo.walVerEnd = pDataRsp->head.walever; + pVg->receiveInfo = true; + char buf[80]; tFormatOffset(buf, 80, &pDataRsp->rspOffset); if (pDataRsp->blockNum == 0) { @@ -1830,6 +1847,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { taosFreeQitem(pollRspWrapper); } } else if (pRspWrapper->tmqRspType == TMQ_MSG_TYPE__POLL_META_RSP) { + // todo handle the wal range and epset for each vgroup SMqPollRspWrapper* pollRspWrapper = (SMqPollRspWrapper*)pRspWrapper; int32_t consumerEpoch = atomic_load_32(&tmq->epoch); @@ -1837,7 +1855,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->metaRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - pVg->currentOffset = pollRspWrapper->metaRsp.rspOffset; + pVg->offsetInfo.currentOffset = pollRspWrapper->metaRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); // build rsp SMqMetaRspObj* pRsp = tmqBuildMetaRspFromWrapper(pollRspWrapper); @@ -1855,7 +1873,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { if (pollRspWrapper->taosxRsp.head.epoch == consumerEpoch) { SMqClientVg* pVg = pollRspWrapper->vgHandle; - pVg->currentOffset = pollRspWrapper->taosxRsp.rspOffset; + pVg->offsetInfo.currentOffset = pollRspWrapper->taosxRsp.rspOffset; atomic_store_32(&pVg->vgStatus, TMQ_VG_STATUS__IDLE); if (pollRspWrapper->taosxRsp.blockNum == 0) { @@ -1881,7 +1899,7 @@ static void* tmqHandleAllRsp(tmq_t* tmq, int64_t timeout, bool pollIfReset) { tmq->totalRows += numOfRows; char buf[80]; - tFormatOffset(buf, 80, &pVg->currentOffset); + tFormatOffset(buf, 80, &pVg->offsetInfo.currentOffset); tscDebug("consumer:0x%" PRIx64 " process taosx poll rsp, vgId:%d, offset:%s, blocks:%d, rows:%" PRId64 ", vg total:%" PRId64 " total:%" PRId64 " reqId:0x%" PRIx64, tmq->consumerId, pVg->vgId, buf, pollRspWrapper->dataRsp.blockNum, numOfRows, pVg->numOfRows, @@ -1917,15 +1935,6 @@ TAOS_RES* tmq_consumer_poll(tmq_t* tmq, int64_t timeout) { tscDebug("consumer:0x%" PRIx64 " start to poll at %" PRId64 ", timeout:%" PRId64, tmq->consumerId, startTime, timeout); -#if 0 - tmqHandleAllDelayedTask(tmq); - tmqPollImpl(tmq, timeout); - rspObj = tmqHandleAllRsp(tmq, timeout, false); - if (rspObj) { - return (TAOS_RES*)rspObj; - } -#endif - // in no topic status, delayed task also need to be processed if (atomic_load_8(&tmq->status) == TMQ_CONSUMER_STATUS__INIT) { tscDebug("consumer:0x%" PRIx64 " poll return since consumer is init", tmq->consumerId); @@ -2121,17 +2130,12 @@ void tmq_commit_async(tmq_t* tmq, const TAOS_RES* pRes, tmq_commit_cb* cb, void* if (pRes == NULL) { // here needs to commit all offsets. asyncCommitAllOffsets(tmq, cb, param); } else { // only commit one offset - asyncCommitOffset(tmq, pRes, cb, param); + asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, cb, param); } } -typedef struct SSyncCommitInfo { - tsem_t sem; - int32_t code; -} SSyncCommitInfo; - -static void commitCallBackFn(tmq_t* pTmq, int32_t code, void* param) { - SSyncCommitInfo* pInfo = (SSyncCommitInfo*)param; +static void commitCallBackFn(tmq_t *UNUSED_PARAM(tmq), int32_t code, void* param) { + SSyncCommitInfo* pInfo = (SSyncCommitInfo*) param; pInfo->code = code; tsem_post(&pInfo->sem); } @@ -2146,7 +2150,7 @@ int32_t tmq_commit_sync(tmq_t* tmq, const TAOS_RES* pRes) { if (pRes == NULL) { asyncCommitAllOffsets(tmq, commitCallBackFn, pInfo); } else { - asyncCommitOffset(tmq, pRes, commitCallBackFn, pInfo); + asyncCommitOffset(tmq, pRes, TDMT_VND_TMQ_COMMIT_OFFSET, commitCallBackFn, pInfo); } tsem_wait(&pInfo->sem); @@ -2168,7 +2172,7 @@ void updateEpCallbackFn(tmq_t* pTmq, int32_t code, SDataBuf* pDataBuf, void* par SMqAskEpRsp rsp; tDecodeSMqAskEpRsp(POINTER_SHIFT(pDataBuf->pData, sizeof(SMqRspHead)), &rsp); - tmqUpdateEp(pTmq, head->epoch, &rsp); + doUpdateLocalEp(pTmq, head->epoch, &rsp); tDeleteSMqAskEpRsp(&rsp); } @@ -2308,3 +2312,294 @@ void commitRspCountDown(SMqCommitCbParamSet* pParamSet, int64_t consumerId, cons waitingRspNum); } } + +SReqResultInfo* tmqGetNextResInfo(TAOS_RES* res, bool convertUcs4) { + SMqRspObj* pRspObj = (SMqRspObj*)res; + pRspObj->resIter++; + + if (pRspObj->resIter < pRspObj->rsp.blockNum) { + SRetrieveTableRsp* pRetrieve = (SRetrieveTableRsp*)taosArrayGetP(pRspObj->rsp.blockData, pRspObj->resIter); + if (pRspObj->rsp.withSchema) { + SSchemaWrapper* pSW = (SSchemaWrapper*)taosArrayGetP(pRspObj->rsp.blockSchema, pRspObj->resIter); + setResSchemaInfo(&pRspObj->resInfo, pSW->pSchema, pSW->nCols); + taosMemoryFreeClear(pRspObj->resInfo.row); + taosMemoryFreeClear(pRspObj->resInfo.pCol); + taosMemoryFreeClear(pRspObj->resInfo.length); + taosMemoryFreeClear(pRspObj->resInfo.convertBuf); + taosMemoryFreeClear(pRspObj->resInfo.convertJson); + } + + setQueryResultFromRsp(&pRspObj->resInfo, pRetrieve, convertUcs4, false); + return &pRspObj->resInfo; + } + + return NULL; +} + +static int32_t tmqGetWalInfoCb(void* param, SDataBuf* pMsg, int32_t code) { + SMqVgWalInfoParam* pParam = param; + SMqVgCommon* pCommon = pParam->pCommon; + + int32_t total = atomic_add_fetch_32(&pCommon->numOfRsp, 1); + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " failed to get the wal info from vgId:%d for topic:%s", pCommon->consumerId, + pParam->vgId, pCommon->pTopicName); + pCommon->code = code; + } else { + SMqDataRsp rsp; + SDecoder decoder; + tDecoderInit(&decoder, POINTER_SHIFT(pMsg->pData, sizeof(SMqRspHead)), pMsg->len - sizeof(SMqRspHead)); + tDecodeMqDataRsp(&decoder, &rsp); + tDecoderClear(&decoder); + + SMqRspHead* pHead = pMsg->pData; + + tmq_topic_assignment assignment = {.begin = pHead->walsver, + .end = pHead->walever, + .currentOffset = rsp.rspOffset.version, + .vgId = pParam->vgId}; + + taosThreadMutexLock(&pCommon->mutex); + taosArrayPush(pCommon->pList, &assignment); + taosThreadMutexUnlock(&pCommon->mutex); + } + + if (total == pParam->totalReq) { + tsem_post(&pCommon->rsp); + } + + taosMemoryFree(pParam); + return 0; +} + +static void destroyCommonInfo(SMqVgCommon* pCommon) { + taosArrayDestroy(pCommon->pList); + tsem_destroy(&pCommon->rsp); + taosThreadMutexDestroy(&pCommon->mutex); + taosMemoryFree(pCommon->pTopicName); + taosMemoryFree(pCommon); +} + +int32_t tmq_get_topic_assignment(tmq_t* tmq, const char* pTopicName, tmq_topic_assignment** assignment, + int32_t* numOfAssignment) { + *numOfAssignment = 0; + *assignment = NULL; + + int32_t accId = tmq->pTscObj->acctId; + char tname[128] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + SMqClientTopic* pTopic = getTopicByName(tmq, tname); + if (pTopic == NULL) { + return TSDB_CODE_INVALID_PARA; + } + + // in case of snapshot is opened, no valid offset will return + *numOfAssignment = taosArrayGetSize(pTopic->vgs); + + *assignment = taosMemoryCalloc(*numOfAssignment, sizeof(tmq_topic_assignment)); + if (*assignment == NULL) { + tscError("consumer:0x%" PRIx64 " failed to malloc buffer, size:%" PRIzu, tmq->consumerId, + (*numOfAssignment) * sizeof(tmq_topic_assignment)); + return TSDB_CODE_OUT_OF_MEMORY; + } + + bool needFetch = false; + + for (int32_t j = 0; j < (*numOfAssignment); ++j) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, j); + if (!pClientVg->receiveInfo) { + needFetch = true; + break; + } + + tmq_topic_assignment* pAssignment = &(*assignment)[j]; + if (pClientVg->offsetInfo.currentOffset.type == TMQ_OFFSET__LOG) { + pAssignment->currentOffset = pClientVg->offsetInfo.currentOffset.version; + } else { + pAssignment->currentOffset = 0; + } + + pAssignment->begin = pClientVg->offsetInfo.walVerBegin; + pAssignment->end = pClientVg->offsetInfo.walVerEnd; + pAssignment->vgId = pClientVg->vgId; + } + + if (needFetch) { + SMqVgCommon* pCommon = taosMemoryCalloc(1, sizeof(SMqVgCommon)); + if (pCommon == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return terrno; + } + + pCommon->pList= taosArrayInit(4, sizeof(tmq_topic_assignment)); + tsem_init(&pCommon->rsp, 0, 0); + taosThreadMutexInit(&pCommon->mutex, 0); + pCommon->pTopicName = taosStrdup(pTopic->topicName); + pCommon->consumerId = tmq->consumerId; + + terrno = TSDB_CODE_OUT_OF_MEMORY; + for (int32_t i = 0; i < (*numOfAssignment); ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + + SMqVgWalInfoParam* pParam = taosMemoryMalloc(sizeof(SMqVgWalInfoParam)); + if (pParam == NULL) { + destroyCommonInfo(pCommon); + return terrno; + } + + pParam->epoch = tmq->epoch; + pParam->vgId = pClientVg->vgId; + pParam->totalReq = *numOfAssignment; + pParam->pCommon = pCommon; + + SMqPollReq req = {0}; + tmqBuildConsumeReqImpl(&req, tmq, 10, pTopic, pClientVg); + + int32_t msgSize = tSerializeSMqPollReq(NULL, 0, &req); + if (msgSize < 0) { + taosMemoryFree(pParam); + destroyCommonInfo(pCommon); + return terrno; + } + + char* msg = taosMemoryCalloc(1, msgSize); + if (NULL == msg) { + taosMemoryFree(pParam); + destroyCommonInfo(pCommon); + return terrno; + } + + if (tSerializeSMqPollReq(msg, msgSize, &req) < 0) { + taosMemoryFree(msg); + taosMemoryFree(pParam); + destroyCommonInfo(pCommon); + return terrno; + } + + SMsgSendInfo* sendInfo = taosMemoryCalloc(1, sizeof(SMsgSendInfo)); + if (sendInfo == NULL) { + taosMemoryFree(pParam); + taosMemoryFree(msg); + destroyCommonInfo(pCommon); + return terrno; + } + + sendInfo->msgInfo = (SDataBuf){.pData = msg, .len = msgSize, .handle = NULL}; + sendInfo->requestId = req.reqId; + sendInfo->requestObjRefId = 0; + sendInfo->param = pParam; + sendInfo->fp = tmqGetWalInfoCb; + sendInfo->msgType = TDMT_VND_TMQ_VG_WALINFO; + + int64_t transporterId = 0; + char offsetFormatBuf[80]; + tFormatOffset(offsetFormatBuf, tListLen(offsetFormatBuf), &pClientVg->offsetInfo.currentOffset); + + tscDebug("consumer:0x%" PRIx64 " %s retrieve wal info vgId:%d, epoch %d, req:%s, reqId:0x%" PRIx64, + tmq->consumerId, pTopic->topicName, pClientVg->vgId, tmq->epoch, offsetFormatBuf, req.reqId); + asyncSendMsgToServer(tmq->pTscObj->pAppInfo->pTransporter, &pClientVg->epSet, &transporterId, sendInfo); + } + + tsem_wait(&pCommon->rsp); + int32_t code = pCommon->code; + + terrno = code; + if (code != TSDB_CODE_SUCCESS) { + taosMemoryFree(*assignment); + *assignment = NULL; + *numOfAssignment = 0; + } else { + int32_t num = taosArrayGetSize(pCommon->pList); + for(int32_t i = 0; i < num; ++i) { + (*assignment)[i] = *(tmq_topic_assignment*)taosArrayGet(pCommon->pList, i); + } + *numOfAssignment = num; + } + + destroyCommonInfo(pCommon); + return code; + } else { + return TSDB_CODE_SUCCESS; + } +} + +int32_t tmq_offset_seek(tmq_t* tmq, const char* pTopicName, int32_t vgId, int64_t offset) { + if (tmq == NULL) { + tscError("invalid tmq handle, null"); + return TSDB_CODE_INVALID_PARA; + } + + int32_t accId = tmq->pTscObj->acctId; + char tname[128] = {0}; + sprintf(tname, "%d.%s", accId, pTopicName); + + SMqClientTopic* pTopic = getTopicByName(tmq, tname); + if (pTopic == NULL) { + tscError("consumer:0x%" PRIx64 " invalid topic name:%s", tmq->consumerId, pTopicName); + return TSDB_CODE_INVALID_PARA; + } + + SMqClientVg* pVg = NULL; + int32_t numOfVgs = taosArrayGetSize(pTopic->vgs); + for (int32_t i = 0; i < numOfVgs; ++i) { + SMqClientVg* pClientVg = taosArrayGet(pTopic->vgs, i); + if (pClientVg->vgId == vgId) { + pVg = pClientVg; + break; + } + } + + if (pVg == NULL) { + tscError("consumer:0x%" PRIx64 " invalid vgroup id:%d", tmq->consumerId, vgId); + return TSDB_CODE_INVALID_PARA; + } + + SVgOffsetInfo* pOffsetInfo = &pVg->offsetInfo; + + int32_t type = pOffsetInfo->currentOffset.type; + if (type != TMQ_OFFSET__LOG) { + tscError("consumer:0x%" PRIx64 " offset type:%d not wal version, seek not allowed", tmq->consumerId, type); + return TSDB_CODE_INVALID_PARA; + } + + if (offset < pOffsetInfo->walVerBegin || offset > pOffsetInfo->walVerEnd) { + tscError("consumer:0x%" PRIx64 " invalid seek params, offset:%" PRId64, tmq->consumerId, offset); + return TSDB_CODE_INVALID_PARA; + } + + // update the offset, and then commit to vnode + if (pOffsetInfo->currentOffset.type == TMQ_OFFSET__LOG) { + pOffsetInfo->currentOffset.version = offset; + pOffsetInfo->committedOffset.version = INT64_MIN; + } + + SMqRspObj rspObj = {.resType = RES_TYPE__TMQ, .vgId = pVg->vgId}; + tstrncpy(rspObj.topic, tname, tListLen(rspObj.topic)); + + tscDebug("consumer:0x%" PRIx64 " seek to %" PRId64 " on vgId:%d", tmq->consumerId, offset, pVg->vgId); + + SSyncCommitInfo* pInfo = taosMemoryMalloc(sizeof(SSyncCommitInfo)); + if (pInfo == NULL) { + tscError("consumer:0x%"PRIx64" failed to prepare seek operation", tmq->consumerId); + return TSDB_CODE_OUT_OF_MEMORY; + } + + tsem_init(&pInfo->sem, 0, 0); + pInfo->code = 0; + + asyncCommitOffset(tmq, &rspObj, TDMT_VND_TMQ_SEEK_TO_OFFSET, commitCallBackFn, pInfo); + + tsem_wait(&pInfo->sem); + int32_t code = pInfo->code; + + tsem_destroy(&pInfo->sem); + taosMemoryFree(pInfo); + + if (code != TSDB_CODE_SUCCESS) { + tscError("consumer:0x%" PRIx64 " failed to send seek to vgId:%d, code:%s", tmq->consumerId, pVg->vgId, + tstrerror(code)); + } + + return code; +} \ No newline at end of file diff --git a/source/client/test/clientTests.cpp b/source/client/test/clientTests.cpp index 517d8e0221136cab9c1075ee06d6e98106b4f7ca..f15a93cb2c300587bfffe880b8a45bf0aef60175 100644 --- a/source/client/test/clientTests.cpp +++ b/source/client/test/clientTests.cpp @@ -30,6 +30,27 @@ #include "taos.h" namespace { + +void printSubResults(void* pRes, int32_t* totalRows) { + char buf[1024]; + + while (1) { + TAOS_ROW row = taos_fetch_row(pRes); + if (row == NULL) { + break; + } + + TAOS_FIELD* fields = taos_fetch_fields(pRes); + int32_t numOfFields = taos_field_count(pRes); + int32_t precision = taos_result_precision(pRes); + taos_print_row(buf, row, fields, numOfFields); + *totalRows += 1; + printf("precision: %d, row content: %s\n", precision, buf); + } + +// taos_free_result(pRes); +} + void showDB(TAOS* pConn) { TAOS_RES* pRes = taos_query(pConn, "show databases"); TAOS_ROW pRow = NULL; @@ -112,7 +133,7 @@ void createNewTable(TAOS* pConn, int32_t index) { } taos_free_result(pRes); - for (int32_t i = 0; i < 100; i += 20) { + for (int32_t i = 0; i < 10000; i += 20) { char sql[1024] = {0}; sprintf(sql, "insert into tu%d values(now+%da, %d)(now+%da, %d)(now+%da, %d)(now+%da, %d)" @@ -162,9 +183,11 @@ void* queryThread(void* arg) { return NULL; } -static int32_t numOfThreads = 1; +int32_t numOfThreads = 1; -void tmq_commit_cb_print(tmq_t* pTmq, int32_t code, void* param) { printf("success, code:%d\n", code); } +void tmq_commit_cb_print(tmq_t* pTmq, int32_t code, void* param) { + printf("auto commit success, code:%d\n\n\n\n", code); +} void* doConsumeData(void* param) { TAOS* pConn = taos_connect("localhost", "root", "taosdata", NULL, 0); @@ -172,7 +195,7 @@ void* doConsumeData(void* param) { tmq_conf_t* conf = tmq_conf_new(); tmq_conf_set(conf, "enable.auto.commit", "true"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName12"); + tmq_conf_set(conf, "group.id", "cgrpName41"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); @@ -801,7 +824,7 @@ TEST(clientCase, projection_query_tables) { } taos_free_result(pRes); - for (int32_t i = 0; i < 10000; ++i) { + for (int32_t i = 0; i < 1; ++i) { printf("create table :%d\n", i); createNewTable(pConn, i); } @@ -988,7 +1011,7 @@ TEST(clientCase, sub_db_test) { tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + tmq_conf_set(conf, "experimental.snapshot.enable", "false"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -998,7 +1021,7 @@ TEST(clientCase, sub_db_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); tmq_list_append(topicList, "topic_t1"); - tmq_list_append(topicList, "topic_s2"); +// tmq_list_append(topicList, "topic_s2"); // 启动订阅 tmq_subscribe(tmq, topicList); @@ -1057,13 +1080,13 @@ TEST(clientCase, sub_tb_test) { ASSERT_NE(pConn, nullptr); tmq_conf_t* conf = tmq_conf_new(); - tmq_conf_set(conf, "enable.auto.commit", "true"); + tmq_conf_set(conf, "enable.auto.commit", "false"); tmq_conf_set(conf, "auto.commit.interval.ms", "1000"); - tmq_conf_set(conf, "group.id", "cgrpName27"); + tmq_conf_set(conf, "group.id", "cgrpName1024"); tmq_conf_set(conf, "td.connect.user", "root"); tmq_conf_set(conf, "td.connect.pass", "taosdata"); tmq_conf_set(conf, "auto.offset.reset", "earliest"); - tmq_conf_set(conf, "experimental.snapshot.enable", "true"); + tmq_conf_set(conf, "experimental.snapshot.enable", "false"); tmq_conf_set(conf, "msg.with.table.name", "true"); tmq_conf_set_auto_commit_cb(conf, tmq_commit_cb_print, NULL); @@ -1072,11 +1095,10 @@ TEST(clientCase, sub_tb_test) { // 创建订阅 topics 列表 tmq_list_t* topicList = tmq_list_new(); - tmq_list_append(topicList, "topic_t2"); + tmq_list_append(topicList, "topic_t1"); // 启动订阅 tmq_subscribe(tmq, topicList); - tmq_list_destroy(topicList); TAOS_FIELD* fields = NULL; @@ -1088,11 +1110,21 @@ TEST(clientCase, sub_tb_test) { int32_t count = 0; + tmq_topic_assignment* pAssign = NULL; + int32_t numOfAssign = 0; + + int32_t code = tmq_get_topic_assignment(tmq, "topic_t1", &pAssign, &numOfAssign); + if (code != 0) { + printf("error occurs:%s\n", tmq_err2str(code)); + tmq_consumer_close(tmq); + taos_close(pConn); + fprintf(stderr, "%d msg consumed, include %d rows\n", msgCnt, totalRows); + return; + } + while (1) { TAOS_RES* pRes = tmq_consumer_poll(tmq, timeout); - if (pRes) { - char buf[1024]; - + if (pRes != NULL) { const char* topicName = tmq_get_topic_name(pRes); const char* dbName = tmq_get_db_name(pRes); int32_t vgroupId = tmq_get_vgroup_id(pRes); @@ -1101,27 +1133,18 @@ TEST(clientCase, sub_tb_test) { printf("db: %s\n", dbName); printf("vgroup id: %d\n", vgroupId); - while (1) { - TAOS_ROW row = taos_fetch_row(pRes); - if (row == NULL) { - break; - } - - fields = taos_fetch_fields(pRes); - numOfFields = taos_field_count(pRes); - precision = taos_result_precision(pRes); - taos_print_row(buf, row, fields, numOfFields); - totalRows += 1; - printf("precision: %d, row content: %s\n", precision, buf); - } + printSubResults(pRes, &totalRows); + } else { +// tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgroupHandle, pAssign[0].begin); +// break; + } + tmq_commit_sync(tmq, pRes); + if (pRes != NULL) { taos_free_result(pRes); - // if ((++count) > 1) { - // break; - // } - } else { - break; } + + tmq_offset_seek(tmq, "topic_t1", pAssign[0].vgId, pAssign[0].begin); } tmq_consumer_close(tmq); diff --git a/source/common/src/systable.c b/source/common/src/systable.c index 2c1598016719169dfddb4423f9b59c9469d37b3f..cd3dd63ef04b56ca1ea933bbb8c3053da8917f4f 100644 --- a/source/common/src/systable.c +++ b/source/common/src/systable.c @@ -116,6 +116,7 @@ static const SSysDbTableSchema userFuncSchema[] = { {.name = "bufsize", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, {.name = "func_language", .bytes = TSDB_TYPE_STR_MAX_LEN - 1 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "func_body", .bytes = TSDB_MAX_BINARY_LEN, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "func_version", .bytes = 4, .type = TSDB_DATA_TYPE_INT, .sysInfo = false}, }; static const SSysDbTableSchema userIdxSchema[] = { @@ -298,7 +299,9 @@ static const SSysDbTableSchema vnodesSchema[] = { static const SSysDbTableSchema userUserPrivilegesSchema[] = { {.name = "user_name", .bytes = TSDB_USER_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, {.name = "privilege", .bytes = 10 + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, - {.name = "object_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "db_name", .bytes = TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "table_name", .bytes = TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, + {.name = "condition", .bytes = TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE, .type = TSDB_DATA_TYPE_VARCHAR, .sysInfo = false}, }; static const SSysTableMeta infosMeta[] = { diff --git a/source/common/src/tdatablock.c b/source/common/src/tdatablock.c index e021e655496cd9f94e787ebfcb00a958519fb50c..f4d2ed01b079b11ff00c7d021a918f5dbe543240 100644 --- a/source/common/src/tdatablock.c +++ b/source/common/src/tdatablock.c @@ -221,6 +221,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c } uint8_t* p = (uint8_t*)pSource->nullbitmap; + pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] &= (0B11111111 << shiftBits); // clear remind bits pColumnInfoData->nullbitmap[BitmapLen(numOfRow1) - 1] |= (p[0] >> remindBits); // copy remind bits if (BitmapLen(numOfRow1) == BitmapLen(total)) { @@ -232,6 +233,7 @@ static void doBitmapMerge(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, c uint8_t* start = (uint8_t*)&pColumnInfoData->nullbitmap[BitmapLen(numOfRow1)]; int32_t overCount = BitmapLen(total) - BitmapLen(numOfRow1); + memset(start, 0, overCount); while (i < len) { // size limit of pSource->nullbitmap if (i >= 1) { start[i - 1] |= (p[i] >> remindBits); // copy remind bits @@ -309,9 +311,11 @@ int32_t colDataMergeCol(SColumnInfoData* pColumnInfoData, int32_t numOfRow1, int pColumnInfoData->pData = tmp; if (BitmapLen(numOfRow1) < BitmapLen(finalNumOfRows)) { char* btmp = taosMemoryRealloc(pColumnInfoData->nullbitmap, BitmapLen(finalNumOfRows)); + if (btmp == NULL) { + return TSDB_CODE_OUT_OF_MEMORY; + } uint32_t extend = BitmapLen(finalNumOfRows) - BitmapLen(numOfRow1); memset(btmp + BitmapLen(numOfRow1), 0, extend); - pColumnInfoData->nullbitmap = btmp; } @@ -1590,12 +1594,13 @@ static void doShiftBitmap(char* nullBitmap, size_t n, size_t total) { i += 1; } } else if (n > 8) { - int32_t gap = len - newLen; + int32_t remain = (total % 8 != 0 && total % 8 <= tail) ? 1 : 0; + int32_t gap = len - newLen - remain; while (i < newLen) { uint8_t v = p[i + gap]; p[i] = (v << tail); - if (i < newLen - 1) { + if (i < newLen - 1 + remain) { uint8_t next = p[i + gap + 1]; p[i] |= (next >> (8 - tail)); } diff --git a/source/common/src/tdataformat.c b/source/common/src/tdataformat.c index d6ab974c6ced8305ddad0ebc86f072a08bf9c978..796c5cf8ff5990b438a48dc6fa54037fe44807b4 100644 --- a/source/common/src/tdataformat.c +++ b/source/common/src/tdataformat.c @@ -500,7 +500,7 @@ int32_t tRowGet(SRow *pRow, STSchema *pTSchema, int32_t iCol, SColVal *pColVal) break; default: ASSERTS(0, "invalid row format"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } if (bv == BIT_FLG_NONE) { @@ -938,7 +938,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * break; default: ASSERTS(0, "Invalid row flag"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } while (pColData) { @@ -963,7 +963,7 @@ static int32_t tRowTupleUpsertColData(SRow *pRow, STSchema *pTSchema, SColData * break; default: ASSERTS(0, "Invalid row flag"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } if (bv == BIT_FLG_NONE) { @@ -1054,7 +1054,7 @@ static int32_t tRowKVUpsertColData(SRow *pRow, STSchema *pTSchema, SColData *aCo pData = pv + ((uint32_t *)pKVIdx->idx)[iCol]; } else { ASSERTS(0, "Invalid KV row format"); - return TSDB_CODE_IVLD_DATA_FMT; + return TSDB_CODE_INVALID_DATA_FMT; } int16_t cid; @@ -2503,9 +2503,11 @@ _exit: int32_t tColDataAddValueByBind(SColData *pColData, TAOS_MULTI_BIND *pBind) { int32_t code = 0; - ASSERT(pColData->type == pBind->buffer_type); - - if (IS_VAR_DATA_TYPE(pBind->buffer_type)) { // var-length data type + if (!(pBind->num == 1 && pBind->is_null && *pBind->is_null)) { + ASSERT(pColData->type == pBind->buffer_type); + } + + if (IS_VAR_DATA_TYPE(pColData->type)) { // var-length data type for (int32_t i = 0; i < pBind->num; ++i) { if (pBind->is_null && pBind->is_null[i]) { code = tColDataAppendValueImpl[pColData->flag][CV_FLAG_NULL](pColData, NULL, 0); diff --git a/source/common/src/tglobal.c b/source/common/src/tglobal.c index 2d579bfd394088bc9cf94f3e88928c5262d474a7..0a29a415f0bbba452e3722ff82eaaca0376e3f8b 100644 --- a/source/common/src/tglobal.c +++ b/source/common/src/tglobal.c @@ -20,6 +20,10 @@ #include "tlog.h" #include "tmisce.h" +#if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) +#include "cus_name.h" +#endif + GRANT_CFG_DECLARE; SConfig *tsCfg = NULL; @@ -99,7 +103,9 @@ char tsSmlChildTableName[TSDB_TABLE_NAME_LEN] = ""; // user defined child table // query int32_t tsQueryPolicy = 1; int32_t tsQueryRspPolicy = 0; +int64_t tsQueryMaxConcurrentTables = 200; // unit is TSDB_TABLE_NUM_UNIT bool tsEnableQueryHb = false; +bool tsEnableScience = false; // on taos-cli show float and doulbe with scientific notation if true int32_t tsQuerySmaOptimize = 0; int32_t tsQueryRsmaTolerance = 1000; // the tolerance time (ms) to judge from which level to query rsma data. bool tsQueryPlannerTrace = false; @@ -111,6 +117,11 @@ int32_t tsRedirectFactor = 2; int32_t tsRedirectMaxPeriod = 1000; int32_t tsMaxRetryWaitTime = 10000; bool tsUseAdapter = false; +int32_t tsSlowLogThreshold = 3; // seconds +int32_t tsSlowLogScope = SLOW_LOG_TYPE_ALL; + + + /* * denote if the server needs to compress response message at the application layer to client, including query rsp, @@ -323,6 +334,7 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "compressColData", tsCompressColData, -1, 100000000, 1) != 0) return -1; if (cfgAddInt32(pCfg, "queryPolicy", tsQueryPolicy, 1, 4, 1) != 0) return -1; if (cfgAddBool(pCfg, "enableQueryHb", tsEnableQueryHb, false) != 0) return -1; + if (cfgAddBool(pCfg, "enableScience", tsEnableScience, false) != 0) return -1; if (cfgAddInt32(pCfg, "querySmaOptimize", tsQuerySmaOptimize, 0, 1, 1) != 0) return -1; if (cfgAddBool(pCfg, "queryPlannerTrace", tsQueryPlannerTrace, true) != 0) return -1; if (cfgAddInt32(pCfg, "queryNodeChunkSize", tsQueryNodeChunkSize, 1024, 128 * 1024, true) != 0) return -1; @@ -336,6 +348,9 @@ static int32_t taosAddClientCfg(SConfig *pCfg) { if (cfgAddInt32(pCfg, "maxRetryWaitTime", tsMaxRetryWaitTime, 0, 86400000, 0) != 0) return -1; if (cfgAddBool(pCfg, "useAdapter", tsUseAdapter, true) != 0) return -1; if (cfgAddBool(pCfg, "crashReporting", tsEnableCrashReport, true) != 0) return -1; + if (cfgAddInt64(pCfg, "queryMaxConcurrentTables", tsQueryMaxConcurrentTables, INT64_MIN, INT64_MAX, 1) != 0) return -1; + if (cfgAddInt32(pCfg, "slowLogThreshold", tsSlowLogThreshold, 0, INT32_MAX, true) != 0) return -1; + if (cfgAddString(pCfg, "slowLogScope", "", true) != 0) return -1; tsNumOfRpcThreads = tsNumOfCores / 2; tsNumOfRpcThreads = TRANGE(tsNumOfRpcThreads, 2, TSDB_MAX_RPC_THREADS); @@ -683,6 +698,42 @@ static void taosSetServerLogCfg(SConfig *pCfg) { metaDebugFlag = cfgGetItem(pCfg, "metaDebugFlag")->i32; } +static int32_t taosSetSlowLogScope(char *pScope) { + if (NULL == pScope || 0 == strlen(pScope)) { + tsSlowLogScope = SLOW_LOG_TYPE_ALL; + return 0; + } + + if (0 == strcasecmp(pScope, "all")) { + tsSlowLogScope = SLOW_LOG_TYPE_ALL; + return 0; + } + + if (0 == strcasecmp(pScope, "query")) { + tsSlowLogScope = SLOW_LOG_TYPE_QUERY; + return 0; + } + + if (0 == strcasecmp(pScope, "insert")) { + tsSlowLogScope = SLOW_LOG_TYPE_INSERT; + return 0; + } + + if (0 == strcasecmp(pScope, "others")) { + tsSlowLogScope = SLOW_LOG_TYPE_OTHERS; + return 0; + } + + if (0 == strcasecmp(pScope, "none")) { + tsSlowLogScope = 0; + return 0; + } + + uError("Invalid slowLog scope value:%s", pScope); + terrno = TSDB_CODE_INVALID_CFG_VALUE; + return -1; +} + static int32_t taosSetClientCfg(SConfig *pCfg) { tstrncpy(tsLocalFqdn, cfgGetItem(pCfg, "fqdn")->str, TSDB_FQDN_LEN); tsServerPort = (uint16_t)cfgGetItem(pCfg, "serverPort")->i32; @@ -724,6 +775,7 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsNumOfTaskQueueThreads = cfgGetItem(pCfg, "numOfTaskQueueThreads")->i32; tsQueryPolicy = cfgGetItem(pCfg, "queryPolicy")->i32; tsEnableQueryHb = cfgGetItem(pCfg, "enableQueryHb")->bval; + tsEnableScience = cfgGetItem(pCfg, "enableScience")->bval; tsQuerySmaOptimize = cfgGetItem(pCfg, "querySmaOptimize")->i32; tsQueryPlannerTrace = cfgGetItem(pCfg, "queryPlannerTrace")->bval; tsQueryNodeChunkSize = cfgGetItem(pCfg, "queryNodeChunkSize")->i32; @@ -731,6 +783,11 @@ static int32_t taosSetClientCfg(SConfig *pCfg) { tsKeepColumnName = cfgGetItem(pCfg, "keepColumnName")->bval; tsUseAdapter = cfgGetItem(pCfg, "useAdapter")->bval; tsEnableCrashReport = cfgGetItem(pCfg, "crashReporting")->bval; + tsQueryMaxConcurrentTables = cfgGetItem(pCfg, "queryMaxConcurrentTables")->i64; + tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { + return -1; + } tsMaxRetryWaitTime = cfgGetItem(pCfg, "maxRetryWaitTime")->i32; @@ -1145,6 +1202,12 @@ int32_t taosSetCfg(SConfig *pCfg, char *name) { sDebugFlag = cfgGetItem(pCfg, "sDebugFlag")->i32; } else if (strcasecmp("smaDebugFlag", name) == 0) { smaDebugFlag = cfgGetItem(pCfg, "smaDebugFlag")->i32; + } else if (strcasecmp("slowLogThreshold", name) == 0) { + tsSlowLogThreshold = cfgGetItem(pCfg, "slowLogThreshold")->i32; + } else if (strcasecmp("slowLogScope", name) == 0) { + if (taosSetSlowLogScope(cfgGetItem(pCfg, "slowLogScope")->str)) { + return -1; + } } break; } diff --git a/source/common/src/tmsg.c b/source/common/src/tmsg.c index c37c0e830ace93b78d95650e38f915fe96f48c34..4139d6c7d449ec4b5c529cd2b0b6814258a9e380 100644 --- a/source/common/src/tmsg.c +++ b/source/common/src/tmsg.c @@ -1368,6 +1368,12 @@ int32_t tSerializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq) if (tEncodeCStr(&encoder, pReq->user) < 0) return -1; if (tEncodeCStr(&encoder, pReq->pass) < 0) return -1; if (tEncodeCStr(&encoder, pReq->objname) < 0) return -1; + int32_t len = strlen(pReq->tabName); + if (tEncodeI32(&encoder, len) < 0) return -1; + if (len > 0) { + if (tEncodeCStr(&encoder, pReq->tabName) < 0) return -1; + } + if (tEncodeBinary(&encoder, pReq->tagCond, pReq->tagCondLen) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1387,6 +1393,16 @@ int32_t tDeserializeSAlterUserReq(void *buf, int32_t bufLen, SAlterUserReq *pReq if (tDecodeCStrTo(&decoder, pReq->user) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->pass) < 0) return -1; if (tDecodeCStrTo(&decoder, pReq->objname) < 0) return -1; + if (!tDecodeIsEnd(&decoder)) { + int32_t len = 0; + if (tDecodeI32(&decoder, &len) < 0) return -1; + if (len > 0) { + if (tDecodeCStrTo(&decoder, pReq->tabName) < 0) return -1; + } + uint64_t tagCondLen = 0; + if (tDecodeBinaryAlloc(&decoder, (void **)&pReq->tagCond, &tagCondLen) < 0) return -1; + pReq->tagCondLen = tagCondLen; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -1429,6 +1445,7 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) int32_t numOfCreatedDbs = taosHashGetSize(pRsp->createdDbs); int32_t numOfReadDbs = taosHashGetSize(pRsp->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pRsp->writeDbs); + if (tEncodeI32(pEncoder, numOfCreatedDbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfReadDbs) < 0) return -1; if (tEncodeI32(pEncoder, numOfWriteDbs) < 0) return -1; @@ -1451,6 +1468,54 @@ int32_t tSerializeSGetUserAuthRspImpl(SEncoder *pEncoder, SGetUserAuthRsp *pRsp) db = taosHashIterate(pRsp->writeDbs, db); } + int32_t numOfReadTbs = taosHashGetSize(pRsp->readTbs); + int32_t numOfWriteTbs = taosHashGetSize(pRsp->writeTbs); + int32_t numOfUseTbs = taosHashGetSize(pRsp->useDbs); + if (tEncodeI32(pEncoder, numOfReadTbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfWriteTbs) < 0) return -1; + if (tEncodeI32(pEncoder, numOfUseTbs) < 0) return -1; + + char *tb = taosHashIterate(pRsp->readTbs, NULL); + while (tb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(tb, &keyLen); + if (tEncodeI32(pEncoder, keyLen) < 0) return -1; + if (tEncodeCStr(pEncoder, key) < 0) return -1; + + size_t valueLen = 0; + valueLen = strlen(tb); + if (tEncodeI32(pEncoder, valueLen) < 0) return -1; + if (tEncodeCStr(pEncoder, tb) < 0) return -1; + + tb = taosHashIterate(pRsp->readTbs, tb); + } + + tb = taosHashIterate(pRsp->writeTbs, NULL); + while (tb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(tb, &keyLen); + if (tEncodeI32(pEncoder, keyLen) < 0) return -1; + if (tEncodeCStr(pEncoder, key) < 0) return -1; + + size_t valueLen = 0; + valueLen = strlen(tb); + if (tEncodeI32(pEncoder, valueLen) < 0) return -1; + if (tEncodeCStr(pEncoder, tb) < 0) return -1; + + tb = taosHashIterate(pRsp->writeTbs, tb); + } + + int32_t *useDb = taosHashIterate(pRsp->useDbs, NULL); + while (useDb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(useDb, &keyLen); + if (tEncodeI32(pEncoder, keyLen) < 0) return -1; + if (tEncodeCStr(pEncoder, key) < 0) return -1; + + if (tEncodeI32(pEncoder, *useDb) < 0) return -1; + useDb = taosHashIterate(pRsp->useDbs, useDb); + } + return 0; } @@ -1473,7 +1538,11 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs pRsp->createdDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->readDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); pRsp->writeDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); - if (pRsp->readDbs == NULL || pRsp->writeDbs == NULL) { + pRsp->readTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->writeTbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pRsp->useDbs = taosHashInit(4, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pRsp->createdDbs == NULL || pRsp->readDbs == NULL || pRsp->writeDbs == NULL || pRsp->readTbs == NULL || + pRsp->writeTbs == NULL || pRsp->useDbs == NULL) { return -1; } @@ -1512,6 +1581,63 @@ int32_t tDeserializeSGetUserAuthRspImpl(SDecoder *pDecoder, SGetUserAuthRsp *pRs taosHashPut(pRsp->writeDbs, db, len, db, len); } + if (!tDecodeIsEnd(pDecoder)) { + int32_t numOfReadTbs = 0; + int32_t numOfWriteTbs = 0; + int32_t numOfUseDbs = 0; + if (tDecodeI32(pDecoder, &numOfReadTbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfWriteTbs) < 0) return -1; + if (tDecodeI32(pDecoder, &numOfUseDbs) < 0) return -1; + + for (int32_t i = 0; i < numOfReadTbs; ++i) { + int32_t keyLen = 0; + if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + + char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + + int32_t valuelen = 0; + if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; + char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + + taosHashPut(pRsp->readTbs, key, strlen(key), value, valuelen + 1); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfWriteTbs; ++i) { + int32_t keyLen = 0; + if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + + char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + + int32_t valuelen = 0; + if (tDecodeI32(pDecoder, &valuelen) < 0) return -1; + char *value = taosMemoryCalloc(valuelen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, value) < 0) return -1; + + taosHashPut(pRsp->writeTbs, key, strlen(key), value, valuelen + 1); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfUseDbs; ++i) { + int32_t keyLen = 0; + if (tDecodeI32(pDecoder, &keyLen) < 0) return -1; + + char *key = taosMemoryCalloc(keyLen + 1, sizeof(char)); + if (tDecodeCStrTo(pDecoder, key) < 0) return -1; + + int32_t ref = 0; + if (tDecodeI32(pDecoder, &ref) < 0) return -1; + taosHashPut(pRsp->useDbs, key, strlen(key), &ref, sizeof(ref)); + } + } + return 0; } @@ -1533,6 +1659,9 @@ void tFreeSGetUserAuthRsp(SGetUserAuthRsp *pRsp) { taosHashCleanup(pRsp->createdDbs); taosHashCleanup(pRsp->readDbs); taosHashCleanup(pRsp->writeDbs); + taosHashCleanup(pRsp->writeTbs); + taosHashCleanup(pRsp->readTbs); + taosHashCleanup(pRsp->useDbs); } int32_t tSerializeSCreateDropMQSNodeReq(void *buf, int32_t bufLen, SMCreateQnodeReq *pReq) { @@ -1702,6 +1831,7 @@ int32_t tSerializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pReq if (tEncodeCStr(&encoder, pReq->pComment) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->orReplace) < 0) return -1; tEndEncode(&encoder); @@ -1746,6 +1876,7 @@ int32_t tDeserializeSCreateFuncReq(void *buf, int32_t bufLen, SCreateFuncReq *pR if (tDecodeCStrTo(&decoder, pReq->pComment) < 0) return -1; } + if (!tDecodeIsEnd(&decoder)) { if (tDecodeI8(&decoder, &pReq->orReplace) < 0) return -1; } else { @@ -1863,6 +1994,13 @@ int32_t tSerializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp * } } + if (pRsp->numOfFuncs != (int32_t)taosArrayGetSize(pRsp->pFuncExtraInfos)) return -1; + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncExtraInfo *extraInfo = taosArrayGet(pRsp->pFuncExtraInfos, i); + if (tEncodeI32(&encoder, extraInfo->funcVersion) < 0) return -1; + if (tEncodeI64(&encoder, extraInfo->funcCreatedTime) < 0) return -1; + } + tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -1910,6 +2048,22 @@ int32_t tDeserializeSRetrieveFuncRsp(void *buf, int32_t bufLen, SRetrieveFuncRsp taosArrayPush(pRsp->pFuncInfos, &fInfo); } + + pRsp->pFuncExtraInfos = taosArrayInit(pRsp->numOfFuncs, sizeof(SFuncExtraInfo)); + if (pRsp->pFuncExtraInfos == NULL) return -1; + if (tDecodeIsEnd(&decoder)) { + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncExtraInfo extraInfo = { 0 }; + taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); + } + } else { + for (int32_t i = 0; i < pRsp->numOfFuncs; ++i) { + SFuncExtraInfo extraInfo = { 0 }; + if (tDecodeI32(&decoder, &extraInfo.funcVersion) < 0) return -1; + if (tDecodeI64(&decoder, &extraInfo.funcCreatedTime) < 0) return -1; + taosArrayPush(pRsp->pFuncExtraInfos, &extraInfo); + } + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -1932,6 +2086,7 @@ void tFreeSRetrieveFuncRsp(SRetrieveFuncRsp *pRsp) { tFreeSFuncInfo(pInfo); } taosArrayDestroy(pRsp->pFuncInfos); + taosArrayDestroy(pRsp->pFuncExtraInfos); } int32_t tSerializeSTableCfgReq(void *buf, int32_t bufLen, STableCfgReq *pReq) { @@ -2781,6 +2936,59 @@ void tFreeSUserAuthBatchRsp(SUserAuthBatchRsp *pRsp) { taosArrayDestroy(pRsp->pArray); } +int32_t tSerializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { + SEncoder encoder = {0}; + tEncoderInit(&encoder, buf, bufLen); + + if (tStartEncode(&encoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tEncodeI32(&encoder, numOfBatch) < 0) return -1; + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserPassRsp *pUserPassRsp = taosArrayGet(pRsp->pArray, i); + if (tEncodeCStr(&encoder, pUserPassRsp->user) < 0) return -1; + if (tEncodeI32(&encoder, pUserPassRsp->version) < 0) return -1; + } + tEndEncode(&encoder); + + int32_t tlen = encoder.pos; + tEncoderClear(&encoder); + return tlen; +} + +int32_t tDeserializeSUserPassBatchRsp(void *buf, int32_t bufLen, SUserPassBatchRsp *pRsp) { + SDecoder decoder = {0}; + tDecoderInit(&decoder, buf, bufLen); + + if (tStartDecode(&decoder) < 0) return -1; + + int32_t numOfBatch = taosArrayGetSize(pRsp->pArray); + if (tDecodeI32(&decoder, &numOfBatch) < 0) return -1; + + pRsp->pArray = taosArrayInit(numOfBatch, sizeof(SGetUserPassRsp)); + if (pRsp->pArray == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + for (int32_t i = 0; i < numOfBatch; ++i) { + SGetUserPassRsp rsp = {0}; + if (tDecodeCStrTo(&decoder, rsp.user) < 0) return -1; + if (tDecodeI32(&decoder, &rsp.version) < 0) return -1; + taosArrayPush(pRsp->pArray, &rsp); + } + tEndDecode(&decoder); + + tDecoderClear(&decoder); + return 0; +} + +void tFreeSUserPassBatchRsp(SUserPassBatchRsp *pRsp) { + if(pRsp) { + taosArrayDestroy(pRsp->pArray); + } +} + int32_t tSerializeSDbCfgReq(void *buf, int32_t bufLen, SDbCfgReq *pReq) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, bufLen); @@ -3821,6 +4029,7 @@ int32_t tSerializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tEncodeI32(&encoder, pRsp->svrTimestamp) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sVer) < 0) return -1; if (tEncodeCStr(&encoder, pRsp->sDetailVer) < 0) return -1; + if (tEncodeI32(&encoder, pRsp->passVer) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -3844,6 +4053,13 @@ int32_t tDeserializeSConnectRsp(void *buf, int32_t bufLen, SConnectRsp *pRsp) { if (tDecodeI32(&decoder, &pRsp->svrTimestamp) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sVer) < 0) return -1; if (tDecodeCStrTo(&decoder, pRsp->sDetailVer) < 0) return -1; + + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI32(&decoder, &pRsp->passVer) < 0) return -1; + } else { + pRsp->passVer = 0; + } + tEndDecode(&decoder); tDecoderClear(&decoder); @@ -3974,6 +4190,12 @@ int32_t tSerializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq *pR for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + if (tEncodeI8(&encoder, pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } tEndEncode(&encoder); @@ -4052,6 +4274,14 @@ int32_t tDeserializeSCreateVnodeReq(void *buf, int32_t bufLen, SCreateVnodeReq * for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -4278,6 +4508,12 @@ int32_t tSerializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnodeRe for (int32_t i = 0; i < 8; ++i) { if (tEncodeI64(&encoder, pReq->reserved[i]) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerSelfIndex) < 0) return -1; + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4301,7 +4537,15 @@ int32_t tDeserializeSAlterVnodeReplicaReq(void *buf, int32_t bufLen, SAlterVnode for (int32_t i = 0; i < 8; ++i) { if (tDecodeI64(&decoder, &pReq->reserved[i]) < 0) return -1; } - + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerSelfIndex) < 0) return -1; + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + } + tEndDecode(&decoder); tDecoderClear(&decoder); return 0; @@ -4612,6 +4856,12 @@ int32_t tSerializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq * SReplica *pReplica = &pReq->replicas[i]; if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; } + if (tEncodeI8(&encoder, pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tEncodeSReplica(&encoder, pReplica) < 0) return -1; + } + if (tEncodeI64(&encoder, pReq->lastIndex) < 0) return -1; tEndEncode(&encoder); int32_t tlen = encoder.pos; @@ -4629,6 +4879,14 @@ int32_t tDeserializeSDCreateMnodeReq(void *buf, int32_t bufLen, SDCreateMnodeReq SReplica *pReplica = &pReq->replicas[i]; if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; } + if (!tDecodeIsEnd(&decoder)) { + if (tDecodeI8(&decoder, &pReq->learnerReplica) < 0) return -1; + for (int32_t i = 0; i < TSDB_MAX_LEARNER_REPLICA; ++i) { + SReplica *pReplica = &pReq->learnerReplicas[i]; + if (tDecodeSReplica(&decoder, pReplica) < 0) return -1; + } + if (tDecodeI64(&decoder, &pReq->lastIndex) < 0) return -1; + } tEndDecode(&decoder); tDecoderClear(&decoder); @@ -6754,6 +7012,18 @@ int32_t tDecodeSTqOffset(SDecoder *pDecoder, STqOffset *pOffset) { return 0; } +int32_t tEncodeMqVgOffset(SEncoder* pEncoder, const SMqVgOffset* pOffset) { + if (tEncodeSTqOffset(pEncoder, &pOffset->offset) < 0) return -1; + if (tEncodeI64(pEncoder, pOffset->consumerId) < 0) return -1; + return 0; +} + +int32_t tDecodeMqVgOffset(SDecoder* pDecoder, SMqVgOffset* pOffset) { + if (tDecodeSTqOffset(pDecoder, &pOffset->offset) < 0) return -1; + if (tDecodeI64(pDecoder, &pOffset->consumerId) < 0) return -1; + return 0; +} + int32_t tEncodeSTqCheckInfo(SEncoder *pEncoder, const STqCheckInfo *pInfo) { if (tEncodeCStr(pEncoder, pInfo->topic) < 0) return -1; if (tEncodeI64(pEncoder, pInfo->ntbUid) < 0) return -1; @@ -6818,21 +7088,21 @@ int32_t tDecodeDeleteRes(SDecoder *pCoder, SDeleteRes *pRes) { return 0; } -int32_t tEncodeSMqMetaRsp(SEncoder *pEncoder, const SMqMetaRsp *pRsp) { +int32_t tEncodeMqMetaRsp(SEncoder *pEncoder, const SMqMetaRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI16(pEncoder, pRsp->resMsgType)) return -1; if (tEncodeBinary(pEncoder, pRsp->metaRsp, pRsp->metaRspLen)) return -1; return 0; } -int32_t tDecodeSMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { +int32_t tDecodeMqMetaRsp(SDecoder *pDecoder, SMqMetaRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI16(pDecoder, &pRsp->resMsgType) < 0) return -1; if (tDecodeBinaryAlloc(pDecoder, &pRsp->metaRsp, (uint64_t *)&pRsp->metaRspLen) < 0) return -1; return 0; } -int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { +int32_t tEncodeMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { if (tEncodeSTqOffsetVal(pEncoder, &pRsp->reqOffset) < 0) return -1; if (tEncodeSTqOffsetVal(pEncoder, &pRsp->rspOffset) < 0) return -1; if (tEncodeI32(pEncoder, pRsp->blockNum) < 0) return -1; @@ -6857,7 +7127,7 @@ int32_t tEncodeSMqDataRsp(SEncoder *pEncoder, const SMqDataRsp *pRsp) { return 0; } -int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { +int32_t tDecodeMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { if (tDecodeSTqOffsetVal(pDecoder, &pRsp->reqOffset) < 0) return -1; if (tDecodeSTqOffsetVal(pDecoder, &pRsp->rspOffset) < 0) return -1; if (tDecodeI32(pDecoder, &pRsp->blockNum) < 0) return -1; @@ -6902,7 +7172,7 @@ int32_t tDecodeSMqDataRsp(SDecoder *pDecoder, SMqDataRsp *pRsp) { return 0; } -void tDeleteSMqDataRsp(SMqDataRsp *pRsp) { +void tDeleteMqDataRsp(SMqDataRsp *pRsp) { pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; @@ -7003,8 +7273,7 @@ int32_t tDecodeSTaosxRsp(SDecoder *pDecoder, STaosxRsp *pRsp) { } void tDeleteSTaosxRsp(STaosxRsp *pRsp) { - taosArrayDestroy(pRsp->blockDataLen); - pRsp->blockDataLen = NULL; + pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); taosArrayDestroyP(pRsp->blockData, (FDelete)taosMemoryFree); pRsp->blockData = NULL; taosArrayDestroyP(pRsp->blockSchema, (FDelete)tDeleteSSchemaWrapper); @@ -7012,8 +7281,7 @@ void tDeleteSTaosxRsp(STaosxRsp *pRsp) { taosArrayDestroyP(pRsp->blockTbName, (FDelete)taosMemoryFree); pRsp->blockTbName = NULL; - taosArrayDestroy(pRsp->createTableLen); - pRsp->createTableLen = NULL; + pRsp->createTableLen = taosArrayDestroy(pRsp->createTableLen); taosArrayDestroyP(pRsp->createTableReq, (FDelete)taosMemoryFree); pRsp->createTableReq = NULL; } @@ -7291,6 +7559,7 @@ void tDestroySSubmitReq2(SSubmitReq2 *pReq, int32_t flag) { tDestroySSubmitTbData(&aSubmitTbData[i], flag); } taosArrayDestroy(pReq->aSubmitTbData); + pReq->aSubmitTbData = NULL; } int32_t tEncodeSSubmitRsp2(SEncoder *pCoder, const SSubmitRsp2 *pRsp) { @@ -7385,3 +7654,41 @@ void tDestroySSubmitRsp2(SSubmitRsp2 *pRsp, int32_t flag) { } } } + +int32_t tEncodeMqSubTopicEp(void **buf, const SMqSubTopicEp *pTopicEp) { + int32_t tlen = 0; + tlen += taosEncodeString(buf, pTopicEp->topic); + tlen += taosEncodeString(buf, pTopicEp->db); + int32_t sz = taosArrayGetSize(pTopicEp->vgs); + tlen += taosEncodeFixedI32(buf, sz); + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp *pVgEp = (SMqSubVgEp *)taosArrayGet(pTopicEp->vgs, i); + tlen += tEncodeSMqSubVgEp(buf, pVgEp); + } + tlen += taosEncodeSSchemaWrapper(buf, &pTopicEp->schema); + return tlen; +} + +void *tDecodeMqSubTopicEp(void *buf, SMqSubTopicEp *pTopicEp) { + buf = taosDecodeStringTo(buf, pTopicEp->topic); + buf = taosDecodeStringTo(buf, pTopicEp->db); + int32_t sz; + buf = taosDecodeFixedI32(buf, &sz); + pTopicEp->vgs = taosArrayInit(sz, sizeof(SMqSubVgEp)); + if (pTopicEp->vgs == NULL) { + return NULL; + } + for (int32_t i = 0; i < sz; i++) { + SMqSubVgEp vgEp; + buf = tDecodeSMqSubVgEp(buf, &vgEp); + taosArrayPush(pTopicEp->vgs, &vgEp); + } + buf = taosDecodeSSchemaWrapper(buf, &pTopicEp->schema); + return buf; +} + +void tDeleteMqSubTopicEp(SMqSubTopicEp *pSubTopicEp) { + taosMemoryFreeClear(pSubTopicEp->schema.pSchema); + pSubTopicEp->schema.nCols = 0; + taosArrayDestroy(pSubTopicEp->vgs); +} \ No newline at end of file diff --git a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h index 600a1f6829e11a1d38ffa262e7d360687c67b90d..6c40fa808fcaf008b3d4614594eb7bab262525f6 100644 --- a/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h +++ b/source/dnode/mgmt/mgmt_dnode/inc/dmInt.h @@ -32,6 +32,7 @@ typedef struct SDnodeMgmt { TdThread crashReportThread; SSingleWorker mgmtWorker; ProcessCreateNodeFp processCreateNodeFp; + ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c index 228f301aec7712fce516ddad2d22f4b2823a36b0..19982698968c8c91bc3f53edd3868f173b070005 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmHandle.c @@ -352,6 +352,7 @@ SArray *dmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SERVER_STATUS, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_SYSTABLE_RETRIEVE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; // Requests handled by MNODE if (dmSetMgmtHandle(pArray, TDMT_MND_GRANT, dmPutNodeMsgToMgmtQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c index 51df293ba70bc37be14c763ecb4a5ea296077bd4..09783a5ea9495506392ec68f9ec5395fa4281d6e 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmInt.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmInt.c @@ -48,6 +48,7 @@ static int32_t dmOpenMgmt(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { pMgmt->path = pInput->path; pMgmt->name = pInput->name; pMgmt->processCreateNodeFp = pInput->processCreateNodeFp; + pMgmt->processAlterNodeTypeFp = pInput->processAlterNodeTypeFp; pMgmt->processDropNodeFp = pInput->processDropNodeFp; pMgmt->sendMonitorReportFp = pInput->sendMonitorReportFp; pMgmt->getVnodeLoadsFp = pInput->getVnodeLoadsFp; diff --git a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c index 0d3b423771267cc567a4cfc95f2b550c508342d3..06b6221940a4f0991fe520a58e2918409139380b 100644 --- a/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c +++ b/source/dnode/mgmt/mgmt_dnode/src/dmWorker.c @@ -227,6 +227,9 @@ static void dmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_DND_DROP_SNODE: code = (*pMgmt->processDropNodeFp)(SNODE, pMsg); break; + case TDMT_DND_ALTER_MNODE_TYPE: + code = (*pMgmt->processAlterNodeTypeFp)(MNODE, pMsg); + break; case TDMT_DND_SERVER_STATUS: code = dmProcessServerRunStatus(pMgmt, pMsg); break; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c index f06669a610142a38b6b2937c95bd150c324df568..cb0849f4b96f945a41a2e27a0abcba2244484bba 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmFile.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmFile.c @@ -24,12 +24,16 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { if (code < 0) return -1; tjsonGetInt32ValueFromDouble(pJson, "selfIndex", pOption->selfIndex, code); if (code < 0) return 0; + tjsonGetInt32ValueFromDouble(pJson, "lastIndex", pOption->lastIndex, code); + if (code < 0) return 0; SJson *replicas = tjsonGetObjectItem(pJson, "replicas"); if (replicas == NULL) return 0; - pOption->numOfReplicas = tjsonGetArraySize(replicas); + pOption->numOfTotalReplicas = tjsonGetArraySize(replicas); + + pOption->numOfReplicas = 0; - for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonGetArrayItem(replicas, i); if (replica == NULL) return -1; @@ -40,6 +44,14 @@ static int32_t mmDecodeOption(SJson *pJson, SMnodeOpt *pOption) { if (code < 0) return -1; tjsonGetUInt16ValueFromDouble(replica, "port", pReplica->port, code); if (code < 0) return -1; + tjsonGetInt32ValueFromDouble(replica, "role", pOption->nodeRoles[i], code); + if (code < 0) return -1; + if(pOption->nodeRoles[i] == TAOS_SYNC_ROLE_VOTER){ + pOption->numOfReplicas++; + } + } + + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { } return 0; @@ -112,14 +124,14 @@ _OVER: } static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { - if (pOption->deploy && pOption->numOfReplicas > 0) { + if (pOption->deploy && pOption->numOfTotalReplicas > 0) { if (tjsonAddDoubleToObject(pJson, "selfIndex", pOption->selfIndex) < 0) return -1; SJson *replicas = tjsonCreateArray(); if (replicas == NULL) return -1; if (tjsonAddItemToObject(pJson, "replicas", replicas) < 0) return -1; - for (int32_t i = 0; i < pOption->numOfReplicas; ++i) { + for (int32_t i = 0; i < pOption->numOfTotalReplicas; ++i) { SJson *replica = tjsonCreateObject(); if (replica == NULL) return -1; @@ -127,10 +139,13 @@ static int32_t mmEncodeOption(SJson *pJson, const SMnodeOpt *pOption) { if (tjsonAddDoubleToObject(replica, "id", pReplica->id) < 0) return -1; if (tjsonAddStringToObject(replica, "fqdn", pReplica->fqdn) < 0) return -1; if (tjsonAddDoubleToObject(replica, "port", pReplica->port) < 0) return -1; + if (tjsonAddDoubleToObject(replica, "role", pOption->nodeRoles[i]) < 0) return -1; if (tjsonAddItemToArray(replicas, replica) < 0) return -1; } } + if (tjsonAddDoubleToObject(pJson, "lastIndex", pOption->lastIndex) < 0) return -1; + if (tjsonAddDoubleToObject(pJson, "deployed", pOption->deploy) < 0) return -1; return 0; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c index 76977dd4a8178792f3867b3a615ef552c5449c1d..76fe0ec2decb3f17f9c5c9c82bcafd1b47af8f49 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmHandle.c @@ -33,12 +33,24 @@ int32_t mmProcessCreateReq(const SMgmtInputOpt *pInput, SRpcMsg *pMsg) { return -1; } - SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, .selfIndex = -1}; + SMnodeOpt option = {.deploy = true, .numOfReplicas = createReq.replica, + .numOfTotalReplicas = createReq.replica + createReq.learnerReplica, + .selfIndex = -1, .lastIndex = createReq.lastIndex}; + memcpy(option.replicas, createReq.replicas, sizeof(createReq.replicas)); - for (int32_t i = 0; i < option.numOfReplicas; ++i) { + for (int32_t i = 0; i < createReq.replica; ++i) { if (createReq.replicas[i].id == pInput->pData->dnodeId) { option.selfIndex = i; } + option.nodeRoles[i] = TAOS_SYNC_ROLE_VOTER; + } + + memcpy(&(option.replicas[createReq.replica]), createReq.learnerReplicas, sizeof(createReq.learnerReplicas)); + for (int32_t i = 0; i < createReq.learnerReplica; ++i) { + if (createReq.learnerReplicas[i].id == pInput->pData->dnodeId) { + option.selfIndex = createReq.replica + i; + } + option.nodeRoles[createReq.replica + i] = TAOS_SYNC_ROLE_LEARNER; } if (option.selfIndex == -1) { @@ -92,6 +104,8 @@ SArray *mmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE_RSP, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CONFIG_DNODE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_MNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE_RSP, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CONNECT, mmPutMsgToReadQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_MND_CREATE_ACCT, mmPutMsgToWriteQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c index e7f71ad4200bd2b1c3c5fcf10e82aa9aa0ea417f..7840528db9329f7b5c10449bee0d8a213e87ebe6 100644 --- a/source/dnode/mgmt/mgmt_mnode/src/mmInt.c +++ b/source/dnode/mgmt/mgmt_mnode/src/mmInt.c @@ -45,9 +45,11 @@ static void mmBuildOptionForDeploy(SMnodeMgmt *pMgmt, const SMgmtInputOpt *pInpu pOption->dnodeId = pMgmt->pData->dnodeId; pOption->selfIndex = 0; pOption->numOfReplicas = 1; + pOption->numOfTotalReplicas = 1; pOption->replicas[0].id = 1; pOption->replicas[0].port = tsServerPort; tstrncpy(pOption->replicas[0].fqdn, tsLocalFqdn, TSDB_FQDN_LEN); + pOption->lastIndex = SYNC_INDEX_INVALID; } static void mmBuildOptionForOpen(SMnodeMgmt *pMgmt, SMnodeOpt *pOption) { @@ -123,9 +125,10 @@ static int32_t mmOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { } tmsgReportStartup("mnode-worker", "initialized"); - if (option.numOfReplicas > 0) { + if (option.numOfTotalReplicas > 0) { option.deploy = true; option.numOfReplicas = 0; + option.numOfTotalReplicas = 0; if (mmWriteFile(pMgmt->path, &option) != 0) { dError("failed to write mnode file since %s", terrstr()); return -1; @@ -152,6 +155,14 @@ static void mmStop(SMnodeMgmt *pMgmt) { mndStop(pMgmt->pMnode); } +static int32_t mmSyncIsCatchUp(SMnodeMgmt *pMgmt) { + return mndIsCatchUp(pMgmt->pMnode); +} + +static ESyncRole mmSyncGetRole(SMnodeMgmt *pMgmt) { + return mndGetRole(pMgmt->pMnode); +} + SMgmtFunc mmGetMgmtFunc() { SMgmtFunc mgmtFunc = {0}; mgmtFunc.openFp = mmOpen; @@ -162,6 +173,8 @@ SMgmtFunc mmGetMgmtFunc() { mgmtFunc.dropFp = (NodeDropFp)mmProcessDropReq; mgmtFunc.requiredFp = mmRequire; mgmtFunc.getHandlesFp = mmGetMsgHandles; + mgmtFunc.isCatchUpFp = (NodeIsCatchUpFp)mmSyncIsCatchUp; + mgmtFunc.nodeRoleFp = (NodeRole)mmSyncGetRole; return mgmtFunc; } diff --git a/source/dnode/mgmt/mgmt_snode/src/smInt.c b/source/dnode/mgmt/mgmt_snode/src/smInt.c index 28097311ac3530788d83c965ed4918d4621effc4..e2223497671362e1c8af19e3481c797e8a2b3972 100644 --- a/source/dnode/mgmt/mgmt_snode/src/smInt.c +++ b/source/dnode/mgmt/mgmt_snode/src/smInt.c @@ -55,6 +55,7 @@ int32_t smOpen(SMgmtInputOpt *pInput, SMgmtOutputOpt *pOutput) { smClose(pMgmt); return -1; } + tmsgReportStartup("snode-impl", "initialized"); if (smStartWorker(pMgmt) != 0) { diff --git a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h index 4374ae363cce8b4138ebb224ddf3528eae2b0784..83fb331dbd5fc368f39c8ab866426200c99ab6bc 100644 --- a/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h +++ b/source/dnode/mgmt/mgmt_vnode/inc/vmInt.h @@ -90,6 +90,7 @@ int32_t vmProcessDropVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); int32_t vmProcessAlterHashRangeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); +int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg); // vmFile.c int32_t vmGetVnodeListFromFile(SVnodeMgmt *pMgmt, SWrapperCfg **ppCfgs, int32_t *numOfVnodes); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c index d61eb3ec0390af04e97c2977131d8381c26b351a..9dbc12cf622af70028629cc152c9f60593c21004 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmHandle.c @@ -131,15 +131,34 @@ static void vmGenerateVnodeCfg(SCreateVnodeReq *pCreate, SVnodeCfg *pCfg) { pCfg->tsdbPageSize = pCreate->tsdbPageSize * 1024; pCfg->standby = 0; - pCfg->syncCfg.myIndex = pCreate->selfIndex; - pCfg->syncCfg.replicaNum = pCreate->replica; + pCfg->syncCfg.replicaNum = 0; + pCfg->syncCfg.totalReplicaNum = 0; + memset(&pCfg->syncCfg.nodeInfo, 0, sizeof(pCfg->syncCfg.nodeInfo)); for (int32_t i = 0; i < pCreate->replica; ++i) { SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; - pNode->nodeId = pCreate->replicas[i].id; - pNode->nodePort = pCreate->replicas[i].port; - tstrncpy(pNode->nodeFqdn, pCreate->replicas[i].fqdn, TSDB_FQDN_LEN); + pNode->nodeId = pCreate->replicas[pCfg->syncCfg.replicaNum].id; + pNode->nodePort = pCreate->replicas[pCfg->syncCfg.replicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; + tstrncpy(pNode->nodeFqdn, pCreate->replicas[pCfg->syncCfg.replicaNum].fqdn, TSDB_FQDN_LEN); + tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + pCfg->syncCfg.replicaNum++; + } + if(pCreate->selfIndex != -1){ + pCfg->syncCfg.myIndex = pCreate->selfIndex; + } + for (int32_t i = pCfg->syncCfg.replicaNum; i < pCreate->replica + pCreate->learnerReplica; ++i) { + SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; + pNode->nodeId = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].id; + pNode->nodePort = pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; + tstrncpy(pNode->nodeFqdn, pCreate->learnerReplicas[pCfg->syncCfg.totalReplicaNum].fqdn, TSDB_FQDN_LEN); tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + pCfg->syncCfg.totalReplicaNum++; + } + pCfg->syncCfg.totalReplicaNum += pCfg->syncCfg.replicaNum; + if(pCreate->learnerSelfIndex != -1){ + pCfg->syncCfg.myIndex = pCreate->replica + pCreate->learnerSelfIndex; } } @@ -182,23 +201,40 @@ int32_t vmProcessCreateVnodeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 + if(req.learnerReplica == 0) + { + req.learnerSelfIndex = -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, start to create vnode, page:%d pageSize:%d buffer:%d szPage:%d szBuf:%" PRIu64 ", cacheLast:%d cacheLastSize:%d sstTrigger:%d tsdbPageSize:%d %d dbname:%s dbId:%" PRId64 ", days:%d keep0:%d keep1:%d keep2:%d tsma:%d precision:%d compression:%d minRows:%d maxRows:%d" ", wal fsync:%d level:%d retentionPeriod:%d retentionSize:%" PRId64 " rollPeriod:%d segSize:%" PRId64 - ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d strict:%d", - req.vgId, req.pages, req.pageSize, req.buffer, req.pageSize * 1024, (uint64_t)req.buffer * 1024 * 1024, + ", hash method:%d begin:%u end:%u prefix:%d surfix:%d replica:%d selfIndex:%d " + "learnerReplica:%d learnerSelfIndex:%d strict:%d", + req.vgId, TMSG_INFO(pMsg->msgType), req.pages, req.pageSize, req.buffer, req.pageSize * 1024, + (uint64_t)req.buffer * 1024 * 1024, req.cacheLast, req.cacheLastSize, req.sstTrigger, req.tsdbPageSize, req.tsdbPageSize * 1024, req.db, req.dbUid, req.daysPerFile, req.daysToKeep0, req.daysToKeep1, req.daysToKeep2, req.isTsma, req.precision, req.compression, req.minRows, req.maxRows, req.walFsyncPeriod, req.walLevel, req.walRetentionPeriod, req.walRetentionSize, req.walRollPeriod, req.walSegmentSize, req.hashMethod, req.hashBegin, req.hashEnd, req.hashPrefix, - req.hashSuffix, req.replica, req.selfIndex, req.strict); + req.hashSuffix, req.replica, req.selfIndex, req.learnerReplica, req.learnerSelfIndex, req.strict); for (int32_t i = 0; i < req.replica; ++i) { dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", req.vgId, i, req.replicas[i].fqdn, req.replicas[i].port, req.replicas[i].id); } + for (int32_t i = 0; i < req.learnerReplica; ++i) { + dInfo("vgId:%d, learnerReplica:%d ep:%s:%u dnode:%d", req.vgId, i, req.learnerReplicas[i].fqdn, + req.learnerReplicas[i].port, req.replicas[i].id); + } - SReplica *pReplica = &req.replicas[req.selfIndex]; + SReplica *pReplica = NULL; + if(req.selfIndex != -1){ + pReplica = &req.replicas[req.selfIndex]; + } + else{ + pReplica = &req.learnerReplicas[req.learnerSelfIndex]; + } if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -275,7 +311,8 @@ _OVER: vnodeClose(pImpl); vnodeDestroy(path, pMgmt->pTfs); } else { - dInfo("vgId:%d, vnode is created", req.vgId); + dInfo("vgId:%d, vnode management handle msgType:%s, end to create vnode, vnode is created", + req.vgId, TMSG_INFO(pMsg->msgType)); } tFreeSCreateVnodeReq(&req); @@ -283,6 +320,122 @@ _OVER: return code; } +int32_t vmProcessAlterVnodeTypeReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { + SAlterVnodeTypeReq req = {0}; + if (tDeserializeSAlterVnodeReplicaReq(pMsg->pCont, pMsg->contLen, &req) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + if(req.learnerReplicas == 0){ + req.learnerSelfIndex = -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, start to process alter-node-type-request", + req.vgId, TMSG_INFO(pMsg->msgType)); + + SVnodeObj *pVnode = vmAcquireVnode(pMgmt, req.vgId); + if (pVnode == NULL) { + dError("vgId:%d, failed to alter vnode type since %s", req.vgId, terrstr()); + terrno = TSDB_CODE_VND_NOT_EXIST; + return -1; + } + + ESyncRole role = vnodeGetRole(pVnode->pImpl); + dInfo("vgId:%d, checking node role:%d", req.vgId, role); + if(role == TAOS_SYNC_ROLE_VOTER){ + terrno = TSDB_CODE_VND_ALREADY_IS_VOTER; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + dInfo("vgId:%d, checking node catch up", req.vgId); + if(vnodeIsCatchUp(pVnode->pImpl) != 1){ + terrno = TSDB_CODE_VND_NOT_CATCH_UP; + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pMgmt->name); + + int32_t vgId = req.vgId; + dInfo("vgId:%d, start to alter vnode type replica:%d selfIndex:%d strict:%d", vgId, req.replica, req.selfIndex, + req.strict); + for (int32_t i = 0; i < req.replica; ++i) { + SReplica *pReplica = &req.replicas[i]; + dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); + } + for (int32_t i = 0; i < req.learnerReplica; ++i) { + SReplica *pReplica = &req.learnerReplicas[i]; + dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->id); + } + + if (req.replica <= 0 || + (req.selfIndex < 0 && req.learnerSelfIndex <0)|| + req.selfIndex >= req.replica || req.learnerSelfIndex >= req.learnerReplica) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, failed to alter replica since invalid msg", vgId); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + SReplica *pReplica = NULL; + if(req.selfIndex > 0){ + pReplica = &req.replicas[req.selfIndex]; + } + else{ + pReplica = &req.learnerReplicas[req.learnerSelfIndex]; + } + + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || + strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { + terrno = TSDB_CODE_INVALID_MSG; + dError("vgId:%d, dnodeId:%d ep:%s:%u not matched with local dnode", vgId, pReplica->id, pReplica->fqdn, + pReplica->port); + vmReleaseVnode(pMgmt, pVnode); + return -1; + } + + dInfo("vgId:%d, start to close vnode", vgId); + SWrapperCfg wrapperCfg = { + .dropped = pVnode->dropped, + .vgId = pVnode->vgId, + .vgVersion = pVnode->vgVersion, + }; + tstrncpy(wrapperCfg.path, pVnode->path, sizeof(wrapperCfg.path)); + vmCloseVnode(pMgmt, pVnode, false); + + char path[TSDB_FILENAME_LEN] = {0}; + snprintf(path, TSDB_FILENAME_LEN, "vnode%svnode%d", TD_DIRSEP, vgId); + + dInfo("vgId:%d, start to alter vnode replica at %s", vgId, path); + if (vnodeAlterReplica(path, &req, pMgmt->pTfs) < 0) { + dError("vgId:%d, failed to alter vnode at %s since %s", vgId, path, terrstr()); + return -1; + } + + dInfo("vgId:%d, begin to open vnode", vgId); + SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); + if (pImpl == NULL) { + dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); + return -1; + } + + if (vmOpenVnode(pMgmt, &wrapperCfg, pImpl) != 0) { + dError("vgId:%d, failed to open vnode mgmt since %s", vgId, terrstr()); + return -1; + } + + if (vnodeStart(pImpl) != 0) { + dError("vgId:%d, failed to start sync since %s", vgId, terrstr()); + return -1; + } + + dInfo("vgId:%d, vnode management handle msgType:%s, end to process alter-node-type-request, vnode config is altered", + req.vgId, TMSG_INFO(pMsg->msgType)); + return 0; +} + int32_t vmProcessDisableVnodeWriteReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { SDisableVnodeWriteReq req = {0}; if (tDeserializeSDisableVnodeWriteReq(pMsg->pCont, pMsg->contLen, &req) != 0) { @@ -377,21 +530,40 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } + if(alterReq.learnerReplica == 0){ + alterReq.learnerSelfIndex = -1; + } + int32_t vgId = alterReq.vgId; - dInfo("vgId:%d, start to alter vnode replica:%d selfIndex:%d strict:%d", vgId, alterReq.replica, alterReq.selfIndex, - alterReq.strict); + dInfo("vgId:%d,vnode management handle msgType:%s, start to alter vnode replica:%d selfIndex:%d leanerReplica:%d " + "learnerSelfIndex:%d strict:%d", + vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { SReplica *pReplica = &alterReq.replicas[i]; dInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); } - - if (alterReq.replica <= 0 || alterReq.selfIndex < 0 || alterReq.selfIndex >= alterReq.replica) { + for (int32_t i = 0; i < alterReq.learnerReplica; ++i) { + SReplica *pReplica = &alterReq.learnerReplicas[i]; + dInfo("vgId:%d, learnerReplicas:%d ep:%s:%u dnode:%d", vgId, i, pReplica->fqdn, pReplica->port, pReplica->port); + } + + if (alterReq.replica <= 0 || + (alterReq.selfIndex < 0 && alterReq.learnerSelfIndex <0)|| + alterReq.selfIndex >= alterReq.replica || alterReq.learnerSelfIndex >= alterReq.learnerReplica) { terrno = TSDB_CODE_INVALID_MSG; dError("vgId:%d, failed to alter replica since invalid msg", vgId); return -1; } - SReplica *pReplica = &alterReq.replicas[alterReq.selfIndex]; + SReplica *pReplica = NULL; + if(alterReq.selfIndex != -1){ + pReplica = &alterReq.replicas[alterReq.selfIndex]; + } + else{ + pReplica = &alterReq.learnerReplicas[alterReq.learnerSelfIndex]; + } + if (pReplica->id != pMgmt->pData->dnodeId || pReplica->port != tsServerPort || strcmp(pReplica->fqdn, tsLocalFqdn) != 0) { terrno = TSDB_CODE_INVALID_MSG; @@ -425,7 +597,7 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, close vnode", vgId); + dInfo("vgId:%d, begin to open vnode", vgId); SVnode *pImpl = vnodeOpen(path, pMgmt->pTfs, pMgmt->msgCb); if (pImpl == NULL) { dError("vgId:%d, failed to open vnode at %s since %s", vgId, path, terrstr()); @@ -442,7 +614,10 @@ int32_t vmProcessAlterVnodeReplicaReq(SVnodeMgmt *pMgmt, SRpcMsg *pMsg) { return -1; } - dInfo("vgId:%d, vnode config is altered", vgId); + dInfo("vgId:%d, vnode management handle msgType:%s, end to alter vnode replica:%d selfIndex:%d leanerReplica:%d " + "learnerSelfIndex:%d strict:%d", + vgId, TMSG_INFO(pMsg->msgType), alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); return 0; } @@ -517,9 +692,11 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SUBSCRIBE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DELETE_SUB, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_COMMIT_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_SEEK_TO_OFFSET, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_ADD_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_DEL_CHECKINFO, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_CONSUME, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_VND_TMQ_VG_WALINFO, vmPutMsgToFetchQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_DELETE, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_BATCH_DEL, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_VND_COMMIT, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; @@ -548,6 +725,7 @@ SArray *vmGetMsgHandles() { if (dmSetMgmtHandle(pArray, TDMT_VND_TRIM, vmPutMsgToWriteQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_CREATE_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_DND_DROP_VNODE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; + if (dmSetMgmtHandle(pArray, TDMT_DND_ALTER_VNODE_TYPE, vmPutMsgToMgmtQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_TIMEOUT_ELECTION, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; if (dmSetMgmtHandle(pArray, TDMT_SYNC_CLIENT_REQUEST, vmPutMsgToSyncQueue, 0) == NULL) goto _OVER; diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c index 4c5b1246e7f21f2a9ca8862e8cc2580407d13264..0244a4fd6e34c8f362047a9829ecc56285ff31fa 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmInt.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmInt.c @@ -15,6 +15,7 @@ #define _DEFAULT_SOURCE #include "vmInt.h" +#include "vnd.h" SVnodeObj *vmAcquireVnode(SVnodeMgmt *pMgmt, int32_t vgId) { SVnodeObj *pVnode = NULL; @@ -78,6 +79,11 @@ int32_t vmOpenVnode(SVnodeMgmt *pMgmt, SWrapperCfg *pCfg, SVnode *pImpl) { void vmCloseVnode(SVnodeMgmt *pMgmt, SVnodeObj *pVnode, bool commitAndRemoveWal) { char path[TSDB_FILENAME_LEN] = {0}; + bool atExit = true; + + if (vnodeIsLeader(pVnode->pImpl)) { + vnodeProposeCommitOnNeed(pVnode->pImpl, atExit); + } taosThreadRwlockWrlock(&pMgmt->lock); taosHashRemove(pMgmt->hash, &pVnode->vgId, sizeof(int32_t)); diff --git a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c index e4e0d608de4de3ae95786f3b1754b18ca3805cdd..a7c4b2e70e13155d955fb2fd8d4862fc6725b2ea 100644 --- a/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c +++ b/source/dnode/mgmt/mgmt_vnode/src/vmWorker.c @@ -49,6 +49,9 @@ static void vmProcessMgmtQueue(SQueueInfo *pInfo, SRpcMsg *pMsg) { case TDMT_VND_ALTER_HASHRANGE: code = vmProcessAlterHashRangeReq(pMgmt, pMsg); break; + case TDMT_DND_ALTER_VNODE_TYPE: + code = vmProcessAlterVnodeTypeReq(pMgmt, pMsg); + break; default: terrno = TSDB_CODE_MSG_NOT_PROCESSED; dGError("msg:%p, not processed in vnode-mgmt queue", pMsg); @@ -114,11 +117,11 @@ static void vmProcessFetchQueue(SQueueInfo *pInfo, STaosQall *qall, int32_t numO int32_t code = vnodeProcessFetchMsg(pVnode->pImpl, pMsg, pInfo); if (code != 0) { if (terrno != 0) code = terrno; - dGError("vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr(code)); + dGError("vnodeProcessFetchMsg vgId:%d, msg:%p failed to fetch since %s", pVnode->vgId, pMsg, terrstr()); vmSendRsp(pMsg, code); } - dGTrace("vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); + dGTrace("vnodeProcessFetchMsg vgId:%d, msg:%p is freed, code:0x%x", pVnode->vgId, pMsg, code); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); } @@ -163,8 +166,8 @@ static int32_t vmPutMsgToQueue(SVnodeMgmt *pMgmt, SRpcMsg *pMsg, EQueueType qtyp SVnodeObj *pVnode = vmAcquireVnode(pMgmt, pHead->vgId); if (pVnode == NULL) { - dGError("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, - terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); + dGWarn("vgId:%d, msg:%p failed to put into vnode queue since %s, type:%s qtype:%d contLen:%d", pHead->vgId, pMsg, + terrstr(), TMSG_INFO(pMsg->msgType), qtype, pHead->contLen); terrno = (terrno != 0) ? terrno : -1; return terrno; } diff --git a/source/dnode/mgmt/node_mgmt/src/dmEnv.c b/source/dnode/mgmt/node_mgmt/src/dmEnv.c index acf96ad397c405c35fe7089aa4d618dcd8eb13e5..3459af1a3a99693f6667728ffe4ab6f2b3923c91 100644 --- a/source/dnode/mgmt/node_mgmt/src/dmEnv.c +++ b/source/dnode/mgmt/node_mgmt/src/dmEnv.c @@ -169,6 +169,8 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return -1; } + dInfo("start to process create-node-request"); + pWrapper = &pDnode->wrappers[ntype]; if (taosMkDir(pWrapper->path) != 0) { dmReleaseWrapper(pWrapper); @@ -198,6 +200,74 @@ static int32_t dmProcessCreateNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { return code; } +static int32_t dmProcessAlterNodeTypeReq(EDndNodeType ntype, SRpcMsg *pMsg) { + SDnode *pDnode = dmInstance(); + + SMgmtWrapper *pWrapper = dmAcquireWrapper(pDnode, ntype); + if (pWrapper == NULL) { + dError("fail to process alter node type since node not exist"); + return -1; + } + dmReleaseWrapper(pWrapper); + + dInfo("node:%s, start to process alter-node-type-request", pWrapper->name); + + pWrapper = &pDnode->wrappers[ntype]; + + if(pWrapper->func.nodeRoleFp != NULL){ + ESyncRole role = (*pWrapper->func.nodeRoleFp)(pWrapper->pMgmt); + dInfo("node:%s, checking node role:%d", pWrapper->name, role); + if(role == TAOS_SYNC_ROLE_VOTER){ + terrno = TSDB_CODE_MNODE_ALREADY_IS_VOTER; + return -1; + } + } + + if(pWrapper->func.isCatchUpFp != NULL){ + dInfo("node:%s, checking node catch up", pWrapper->name); + if((*pWrapper->func.isCatchUpFp)(pWrapper->pMgmt) != 1){ + terrno = TSDB_CODE_MNODE_NOT_CATCH_UP; + return -1; + } + } + + dInfo("node:%s, catched up leader, continue to process alter-node-type-request", pWrapper->name); + + taosThreadMutexLock(&pDnode->mutex); + + dInfo("node:%s, stopping node", pWrapper->name); + dmStopNode(pWrapper); + dInfo("node:%s, closing node", pWrapper->name); + dmCloseNode(pWrapper); + + pWrapper = &pDnode->wrappers[ntype]; + if (taosMkDir(pWrapper->path) != 0) { + dmReleaseWrapper(pWrapper); + terrno = TAOS_SYSTEM_ERROR(errno); + dError("failed to create dir:%s since %s", pWrapper->path, terrstr()); + return -1; + } + + SMgmtInputOpt input = dmBuildMgmtInputOpt(pWrapper); + + dInfo("node:%s, start to create", pWrapper->name); + int32_t code = (*pWrapper->func.createFp)(&input, pMsg); + if (code != 0) { + dError("node:%s, failed to create since %s", pWrapper->name, terrstr()); + } else { + dInfo("node:%s, has been created", pWrapper->name); + code = dmOpenNode(pWrapper); + if (code == 0) { + code = dmStartNode(pWrapper); + } + pWrapper->deployed = true; + pWrapper->required = true; + } + + taosThreadMutexUnlock(&pDnode->mutex); + return code; +} + static int32_t dmProcessDropNodeReq(EDndNodeType ntype, SRpcMsg *pMsg) { SDnode *pDnode = dmInstance(); @@ -251,6 +321,7 @@ SMgmtInputOpt dmBuildMgmtInputOpt(SMgmtWrapper *pWrapper) { .name = pWrapper->name, .pData = &pWrapper->pDnode->data, .processCreateNodeFp = dmProcessCreateNodeReq, + .processAlterNodeTypeFp = dmProcessAlterNodeTypeReq, .processDropNodeFp = dmProcessDropNodeReq, .sendMonitorReportFp = dmSendMonitorReport, .getVnodeLoadsFp = dmGetVnodeLoads, diff --git a/source/dnode/mgmt/node_util/inc/dmUtil.h b/source/dnode/mgmt/node_util/inc/dmUtil.h index cfdea40477b1dae1cabc35e8566c1ff7ad86a148..98ef8cd95ba9c201ec23ff9fd8515948016eec4f 100644 --- a/source/dnode/mgmt/node_util/inc/dmUtil.h +++ b/source/dnode/mgmt/node_util/inc/dmUtil.h @@ -89,6 +89,7 @@ typedef void (*SendMonitorReportFp)(); typedef void (*GetVnodeLoadsFp)(SMonVloadInfo *pInfo); typedef void (*GetMnodeLoadsFp)(SMonMloadInfo *pInfo); typedef void (*GetQnodeLoadsFp)(SQnodeLoad *pInfo); +typedef int32_t (*ProcessAlterNodeTypeFp)(EDndNodeType ntype, SRpcMsg *pMsg); typedef struct { int32_t dnodeId; @@ -112,6 +113,7 @@ typedef struct { SDnodeData *pData; SMsgCb msgCb; ProcessCreateNodeFp processCreateNodeFp; + ProcessAlterNodeTypeFp processAlterNodeTypeFp; ProcessDropNodeFp processDropNodeFp; SendMonitorReportFp sendMonitorReportFp; GetVnodeLoadsFp getVnodeLoadsFp; @@ -132,6 +134,8 @@ typedef int32_t (*NodeCreateFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeDropFp)(const SMgmtInputOpt *pInput, SRpcMsg *pMsg); typedef int32_t (*NodeRequireFp)(const SMgmtInputOpt *pInput, bool *required); typedef SArray *(*NodeGetHandlesFp)(); // array of SMgmtHandle +typedef bool (*NodeIsCatchUpFp)(void *pMgmt); +typedef bool (*NodeRole)(void *pMgmt); typedef struct { NodeOpenFp openFp; @@ -142,6 +146,8 @@ typedef struct { NodeDropFp dropFp; NodeRequireFp requiredFp; NodeGetHandlesFp getHandlesFp; + NodeIsCatchUpFp isCatchUpFp; + NodeRole nodeRoleFp; } SMgmtFunc; typedef struct { diff --git a/source/dnode/mnode/impl/CMakeLists.txt b/source/dnode/mnode/impl/CMakeLists.txt index 493ba48601382bdd25ce95b1639708c9f25c202d..b9aa8eb674b9b6f760f0c782f2a9e7aabe55eeb0 100644 --- a/source/dnode/mnode/impl/CMakeLists.txt +++ b/source/dnode/mnode/impl/CMakeLists.txt @@ -5,6 +5,7 @@ ENDIF () IF (TD_ENTERPRISE) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/privilege/src/privilege.c) LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndDb.c) + LIST(APPEND MNODE_SRC ${TD_ENTERPRISE_DIR}/src/plugins/mnode/src/mndVgroup.c) ENDIF () add_library(mnode STATIC ${MNODE_SRC}) diff --git a/source/dnode/mnode/impl/inc/mndConsumer.h b/source/dnode/mnode/impl/inc/mndConsumer.h index 99b0d06936ec16b59b0abda4bad52748ff9a6e45..aa38b94fd7d6a6b0a5118d6aac2a760e7717cd94 100644 --- a/source/dnode/mnode/impl/inc/mndConsumer.h +++ b/source/dnode/mnode/impl/inc/mndConsumer.h @@ -24,10 +24,10 @@ extern "C" { enum { MQ_CONSUMER_STATUS__MODIFY = 1, - MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore +// MQ_CONSUMER_STATUS__MODIFY_IN_REB, // this value is not used anymore MQ_CONSUMER_STATUS__READY, MQ_CONSUMER_STATUS__LOST, - MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore +// MQ_CONSUMER_STATUS__LOST_IN_REB, // this value is not used anymore MQ_CONSUMER_STATUS__LOST_REBD, MQ_CONSUMER_STATUS__REMOVED, }; diff --git a/source/dnode/mnode/impl/inc/mndDef.h b/source/dnode/mnode/impl/inc/mndDef.h index 876f2651bc652a128ac3bbf99a97e9cf895658f9..f547ce025d67d5e04441699b2e6b48eb3efd52e2 100644 --- a/source/dnode/mnode/impl/inc/mndDef.h +++ b/source/dnode/mnode/impl/inc/mndDef.h @@ -215,6 +215,8 @@ typedef struct { bool syncRestore; int64_t stateStartTime; SDnodeObj* pDnode; + int32_t role; + SyncIndex lastIndex; } SMnodeObj; typedef struct { @@ -278,9 +280,13 @@ typedef struct { int8_t reserve; int32_t acctId; int32_t authVersion; + int32_t passVersion; SHashObj* readDbs; SHashObj* writeDbs; SHashObj* topics; + SHashObj* readTbs; + SHashObj* writeTbs; + SHashObj* useDbs; SRWLatch lock; } SUserObj; @@ -338,6 +344,7 @@ typedef struct { ESyncState syncState; bool syncRestore; bool syncCanRead; + ESyncRole nodeRole; } SVnodeGid; typedef struct { @@ -358,7 +365,7 @@ typedef struct { int8_t compact; int8_t isTsma; int8_t replica; - SVnodeGid vnodeGid[TSDB_MAX_REPLICA]; + SVnodeGid vnodeGid[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; void* pTsma; int32_t numOfCachedTables; } SVgObj; @@ -447,6 +454,8 @@ typedef struct { int32_t codeSize; char* pComment; char* pCode; + int32_t funcVersion; + SRWLatch lock; } SFuncObj; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndInt.h b/source/dnode/mnode/impl/inc/mndInt.h index ffb2443808b9efdd2e0a3dfbe89662b88962f6be..ec83f8a7e6daed8e7a0685087d35c5aad90e390e 100644 --- a/source/dnode/mnode/impl/inc/mndInt.h +++ b/source/dnode/mnode/impl/inc/mndInt.h @@ -92,8 +92,11 @@ typedef struct { int64_t transSeq; TdThreadMutex lock; int8_t selfIndex; + int8_t numOfTotalReplicas; int8_t numOfReplicas; - SReplica replicas[TSDB_MAX_REPLICA]; + SReplica replicas[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + ESyncRole nodeRoles[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex lastIndex; } SSyncMgmt; typedef struct { diff --git a/source/dnode/mnode/impl/inc/mndStb.h b/source/dnode/mnode/impl/inc/mndStb.h index ac0924aab9965de8ccf25d135e5f4b2082121cbe..66d0ed1d12160b5ea6e9c30f145e0db6faa22b31 100644 --- a/source/dnode/mnode/impl/inc/mndStb.h +++ b/source/dnode/mnode/impl/inc/mndStb.h @@ -38,6 +38,7 @@ void mndFreeStb(SStbObj *pStb); int32_t mndBuildSMCreateStbRsp(SMnode *pMnode, char *dbFName, char *stbFName, void **pCont, int32_t *pLen); void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst); +void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst); void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize); const char *mndGetStbStr(const char *src); diff --git a/source/dnode/mnode/impl/inc/mndUser.h b/source/dnode/mnode/impl/inc/mndUser.h index 8943ba703ee47442a49fe25ab44ed7fcfb7867cd..aa7f97f0870dfd83ba4c3296ba1293e091b6fdba 100644 --- a/source/dnode/mnode/impl/inc/mndUser.h +++ b/source/dnode/mnode/impl/inc/mndUser.h @@ -31,9 +31,12 @@ void mndReleaseUser(SMnode *pMnode, SUserObj *pUser); // for trans test SSdbRaw *mndUserActionEncode(SUserObj *pUser); SHashObj *mndDupDbHash(SHashObj *pOld); +SHashObj *mndDupTableHash(SHashObj *pOld); SHashObj *mndDupTopicHash(SHashObj *pOld); int32_t mndValidateUserAuthInfo(SMnode *pMnode, SUserAuthVersion *pUsers, int32_t numOfUses, void **ppRsp, int32_t *pRspLen); +int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen); int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db); int32_t mndUserRemoveTopic(SMnode *pMnode, STrans *pTrans, char *topic); diff --git a/source/dnode/mnode/impl/src/mndCluster.c b/source/dnode/mnode/impl/src/mndCluster.c index 94584dfe58f3bd40dc4df5b67cde6c415b071693..4d05637a2b3ab872d5a0ba99c022e21f79ca9b05 100644 --- a/source/dnode/mnode/impl/src/mndCluster.c +++ b/source/dnode/mnode/impl/src/mndCluster.c @@ -67,7 +67,7 @@ int32_t mndGetClusterName(SMnode *pMnode, char *clusterName, int32_t len) { return 0; } -static SClusterObj *mndAcquireCluster(SMnode *pMnode) { +static SClusterObj *mndAcquireCluster(SMnode *pMnode, void **ppIter) { SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; @@ -76,23 +76,27 @@ static SClusterObj *mndAcquireCluster(SMnode *pMnode) { pIter = sdbFetch(pSdb, SDB_CLUSTER, pIter, (void **)&pCluster); if (pIter == NULL) break; + *ppIter = pIter; + return pCluster; } return NULL; } -static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster) { +static void mndReleaseCluster(SMnode *pMnode, SClusterObj *pCluster, void *pIter) { SSdb *pSdb = pMnode->pSdb; + sdbCancelFetch(pSdb, pIter); sdbRelease(pSdb, pCluster); } int64_t mndGetClusterId(SMnode *pMnode) { int64_t clusterId = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { clusterId = pCluster->id; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return clusterId; @@ -100,10 +104,11 @@ int64_t mndGetClusterId(SMnode *pMnode) { int64_t mndGetClusterCreateTime(SMnode *pMnode) { int64_t createTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { createTime = pCluster->createdTime; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return createTime; @@ -121,10 +126,11 @@ static int32_t mndGetClusterUpTimeImp(SClusterObj *pCluster) { float mndGetClusterUpTime(SMnode *pMnode) { int64_t upTime = 0; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { upTime = mndGetClusterUpTimeImp(pCluster); - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } return upTime / 86400.0f; @@ -321,11 +327,12 @@ static void mndCancelGetNextCluster(SMnode *pMnode, void *pIter) { static int32_t mndProcessUptimeTimer(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SClusterObj clusterObj = {0}; - SClusterObj *pCluster = mndAcquireCluster(pMnode); + void *pIter = NULL; + SClusterObj *pCluster = mndAcquireCluster(pMnode, &pIter); if (pCluster != NULL) { memcpy(&clusterObj, pCluster, sizeof(SClusterObj)); clusterObj.upTime += tsUptimeInterval; - mndReleaseCluster(pMnode, pCluster); + mndReleaseCluster(pMnode, pCluster, pIter); } if (clusterObj.id <= 0) { diff --git a/source/dnode/mnode/impl/src/mndConsumer.c b/source/dnode/mnode/impl/src/mndConsumer.c index 8f1cfbff931d74df3dab3f6f3fb9669894811876..0d75b5fd68242b2eed202e2ec432847d5874a211 100644 --- a/source/dnode/mnode/impl/src/mndConsumer.c +++ b/source/dnode/mnode/impl/src/mndConsumer.c @@ -335,7 +335,7 @@ static int32_t mndProcessMqTimerMsg(SRpcMsg *pMsg) { taosArrayPush(pRebSub->removedConsumers, &pConsumer->consumerId); } taosRUnLockLatch(&pConsumer->lock); - } else if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + } else if (status == MQ_CONSUMER_STATUS__MODIFY) { taosRLockLatch(&pConsumer->lock); int32_t newTopicNum = taosArrayGetSize(pConsumer->rebNewTopics); @@ -556,9 +556,14 @@ static int32_t mndProcessAskEpReq(SRpcMsg *pMsg) { return -1; } - ((SMqRspHead *)buf)->mqMsgType = TMQ_MSG_TYPE__EP_RSP; - ((SMqRspHead *)buf)->epoch = serverEpoch; - ((SMqRspHead *)buf)->consumerId = pConsumer->consumerId; + SMqRspHead* pHead = buf; + + pHead->mqMsgType = TMQ_MSG_TYPE__EP_RSP; + pHead->epoch = serverEpoch; + pHead->consumerId = pConsumer->consumerId; + pHead->walsver = 0; + pHead->walever = 0; + void *abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); tEncodeSMqAskEpRsp(&abuf, &rsp); @@ -873,17 +878,11 @@ static void updateConsumerStatus(SMqConsumerObj *pConsumer) { int32_t status = pConsumer->status; if (taosArrayGetSize(pConsumer->rebNewTopics) == 0 && taosArrayGetSize(pConsumer->rebRemovedTopics) == 0) { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { + if (status == MQ_CONSUMER_STATUS__MODIFY) { pConsumer->status = MQ_CONSUMER_STATUS__READY; - } else if (status == MQ_CONSUMER_STATUS__LOST_IN_REB || status == MQ_CONSUMER_STATUS__LOST) { + } else if (status == MQ_CONSUMER_STATUS__LOST) { pConsumer->status = MQ_CONSUMER_STATUS__LOST_REBD; } - } else { - if (status == MQ_CONSUMER_STATUS__MODIFY || status == MQ_CONSUMER_STATUS__MODIFY_IN_REB) { - pConsumer->status = MQ_CONSUMER_STATUS__MODIFY; - } else if (status == MQ_CONSUMER_STATUS__LOST || status == MQ_CONSUMER_STATUS__LOST_IN_REB) { - pConsumer->status = MQ_CONSUMER_STATUS__LOST; - } } } @@ -1192,10 +1191,8 @@ static const char *mndConsumerStatusName(int status) { return "ready"; case MQ_CONSUMER_STATUS__LOST: case MQ_CONSUMER_STATUS__LOST_REBD: - case MQ_CONSUMER_STATUS__LOST_IN_REB: return "lost"; case MQ_CONSUMER_STATUS__MODIFY: - case MQ_CONSUMER_STATUS__MODIFY_IN_REB: return "rebalancing"; default: return "unknown"; diff --git a/source/dnode/mnode/impl/src/mndDef.c b/source/dnode/mnode/impl/src/mndDef.c index fb81a764f133f53f786b1d227e6c8d80f25d9c49..c69f08eb6b244b358fa73312d5a86e7ff4818097 100644 --- a/source/dnode/mnode/impl/src/mndDef.c +++ b/source/dnode/mnode/impl/src/mndDef.c @@ -70,7 +70,7 @@ int32_t tEncodeSStreamObj(SEncoder *pEncoder, const SStreamObj *pObj) { if (tEncodeI32(pEncoder, innerSz) < 0) return -1; for (int32_t j = 0; j < innerSz; j++) { SStreamTask *pTask = taosArrayGetP(pArray, j); - if (tEncodeSStreamTask(pEncoder, pTask) < 0) return -1; + if (tEncodeStreamTask(pEncoder, pTask) < 0) return -1; } } @@ -130,7 +130,7 @@ int32_t tDecodeSStreamObj(SDecoder *pDecoder, SStreamObj *pObj, int32_t sver) { taosArrayDestroy(pArray); return -1; } - if (tDecodeSStreamTask(pDecoder, pTask) < 0) { + if (tDecodeStreamTask(pDecoder, pTask) < 0) { taosMemoryFree(pTask); taosArrayDestroy(pArray); return -1; @@ -158,7 +158,10 @@ void tFreeStreamObj(SStreamObj *pStream) { taosMemoryFree(pStream->sql); taosMemoryFree(pStream->ast); taosMemoryFree(pStream->physicalPlan); - if (pStream->outputSchema.nCols) taosMemoryFree(pStream->outputSchema.pSchema); + + if (pStream->outputSchema.nCols) { + taosMemoryFree(pStream->outputSchema.pSchema); + } int32_t sz = taosArrayGetSize(pStream->tasks); for (int32_t i = 0; i < sz; i++) { @@ -166,11 +169,14 @@ void tFreeStreamObj(SStreamObj *pStream) { int32_t taskSz = taosArrayGetSize(pLevel); for (int32_t j = 0; j < taskSz; j++) { SStreamTask *pTask = taosArrayGetP(pLevel, j); - tFreeSStreamTask(pTask); + tFreeStreamTask(pTask); } + taosArrayDestroy(pLevel); } + taosArrayDestroy(pStream->tasks); + // tagSchema.pSchema if (pStream->tagSchema.nCols > 0) { taosMemoryFree(pStream->tagSchema.pSchema); diff --git a/source/dnode/mnode/impl/src/mndDump.c b/source/dnode/mnode/impl/src/mndDump.c index 44a7d49fff672e33af43bb892350ee0b5e02debd..a991bddda8a997c6074ee21e811f3829e0370fe8 100644 --- a/source/dnode/mnode/impl/src/mndDump.c +++ b/source/dnode/mnode/impl/src/mndDump.c @@ -421,6 +421,7 @@ void dumpUser(SSdb *pSdb, SJson *json) { tjsonAddStringToObject(item, "updateTime", i642str(pObj->updateTime)); tjsonAddStringToObject(item, "superUser", i642str(pObj->superUser)); tjsonAddStringToObject(item, "authVersion", i642str(pObj->authVersion)); + tjsonAddStringToObject(item, "passVersion", i642str(pObj->passVersion)); tjsonAddStringToObject(item, "numOfReadDbs", i642str(taosHashGetSize(pObj->readDbs))); tjsonAddStringToObject(item, "numOfWriteDbs", i642str(taosHashGetSize(pObj->writeDbs))); sdbRelease(pSdb, pObj); diff --git a/source/dnode/mnode/impl/src/mndFunc.c b/source/dnode/mnode/impl/src/mndFunc.c index 7a475c61b6c0c95469d94a690fd80c96ac11b5c4..51a231daf463c357eb494c87fad4b0187a9c43f6 100644 --- a/source/dnode/mnode/impl/src/mndFunc.c +++ b/source/dnode/mnode/impl/src/mndFunc.c @@ -21,7 +21,7 @@ #include "mndTrans.h" #include "mndUser.h" -#define SDB_FUNC_VER 1 +#define SDB_FUNC_VER 2 #define SDB_FUNC_RESERVE_SIZE 64 static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc); @@ -83,6 +83,7 @@ static SSdbRaw *mndFuncActionEncode(SFuncObj *pFunc) { SDB_SET_BINARY(pRaw, dataPos, pFunc->pComment, pFunc->commentSize, _OVER) } SDB_SET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) + SDB_SET_INT32(pRaw, dataPos, pFunc->funcVersion, _OVER) SDB_SET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER); @@ -107,7 +108,7 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != SDB_FUNC_VER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -144,8 +145,15 @@ static SSdbRow *mndFuncActionDecode(SSdbRaw *pRaw) { goto _OVER; } SDB_GET_BINARY(pRaw, dataPos, pFunc->pCode, pFunc->codeSize, _OVER) + + if (sver >= 2) { + SDB_GET_INT32(pRaw, dataPos, &pFunc->funcVersion, _OVER) + } + SDB_GET_RESERVE(pRaw, dataPos, SDB_FUNC_RESERVE_SIZE, _OVER) + taosInitRWLatch(&pFunc->lock); + terrno = 0; _OVER: @@ -173,6 +181,44 @@ static int32_t mndFuncActionDelete(SSdb *pSdb, SFuncObj *pFunc) { static int32_t mndFuncActionUpdate(SSdb *pSdb, SFuncObj *pOld, SFuncObj *pNew) { mTrace("func:%s, perform update action, old row:%p new row:%p", pOld->name, pOld, pNew); + + taosWLockLatch(&pOld->lock); + + pOld->align = pNew->align; + pOld->bufSize = pNew->bufSize; + pOld->codeSize = pNew->codeSize; + pOld->commentSize = pNew->commentSize; + pOld->createdTime = pNew->createdTime; + pOld->funcType = pNew->funcType; + pOld->funcVersion = pNew->funcVersion; + pOld->outputLen = pNew->outputLen; + pOld->outputType = pNew->outputType; + + if (pOld->pComment != NULL) { + taosMemoryFree(pOld->pComment); + pOld->pComment = NULL; + } + if (pNew->commentSize > 0 && pNew->pComment != NULL) { + pOld->commentSize = pNew->commentSize; + pOld->pComment = taosMemoryMalloc(pOld->commentSize); + memcpy(pOld->pComment, pNew->pComment, pOld->commentSize); + } + + if (pOld->pCode != NULL) { + taosMemoryFree(pOld->pCode); + pOld->pCode = NULL; + } + if (pNew->codeSize > 0 && pNew->pCode != NULL) { + pOld->codeSize = pNew->codeSize; + pOld->pCode = taosMemoryMalloc(pOld->codeSize); + memcpy(pOld->pCode, pNew->pCode, pOld->codeSize); + } + + pOld->scriptType = pNew->scriptType; + pOld->signature = pNew->signature; + + taosWUnLockLatch(&pOld->lock); + return 0; } @@ -225,26 +271,47 @@ static int32_t mndCreateFunc(SMnode *pMnode, SRpcMsg *pReq, SCreateFuncReq *pCre pTrans = mndTransCreate(pMnode, TRN_POLICY_ROLLBACK, TRN_CONFLICT_NOTHING, pReq, "create-func"); if (pTrans == NULL) goto _OVER; - mInfo("trans:%d, used to create func:%s", pTrans->id, pCreate->name); - SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); - if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; + SFuncObj *oldFunc = mndAcquireFunc(pMnode, pCreate->name); + if (pCreate->orReplace == 1 && oldFunc != NULL) { + func.funcVersion = oldFunc->funcVersion + 1; + func.createdTime = oldFunc->createdTime; + + SSdbRaw *pRedoRaw = mndFuncActionEncode(oldFunc); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_READY) != 0) goto _OVER; + + SSdbRaw *pUndoRaw = mndFuncActionEncode(oldFunc); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_READY) != 0) goto _OVER; + + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + } else { + SSdbRaw *pRedoRaw = mndFuncActionEncode(&func); + if (pRedoRaw == NULL || mndTransAppendRedolog(pTrans, pRedoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pRedoRaw, SDB_STATUS_CREATING) != 0) goto _OVER; - SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); - if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; + SSdbRaw *pUndoRaw = mndFuncActionEncode(&func); + if (pUndoRaw == NULL || mndTransAppendUndolog(pTrans, pUndoRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pUndoRaw, SDB_STATUS_DROPPED) != 0) goto _OVER; - SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); - if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; - if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + SSdbRaw *pCommitRaw = mndFuncActionEncode(&func); + if (pCommitRaw == NULL || mndTransAppendCommitlog(pTrans, pCommitRaw) != 0) goto _OVER; + if (sdbSetRawStatus(pCommitRaw, SDB_STATUS_READY) != 0) goto _OVER; + } if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; _OVER: + if (oldFunc != NULL) { + mndReleaseFunc(pMnode, oldFunc); + } + taosMemoryFree(func.pCode); taosMemoryFree(func.pComment); mndTransDrop(pTrans); @@ -304,6 +371,9 @@ static int32_t mndProcessCreateFuncReq(SRpcMsg *pReq) { mInfo("func:%s, already exist, ignore exist is set", createReq.name); code = 0; goto _OVER; + } else if (createReq.orReplace) { + mInfo("func:%s, replace function is set", createReq.name); + code = 0; } else { terrno = TSDB_CODE_MND_FUNC_ALREADY_EXIST; goto _OVER; @@ -413,6 +483,12 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { goto RETRIEVE_FUNC_OVER; } + retrieveRsp.pFuncExtraInfos = taosArrayInit(retrieveReq.numOfFuncs, sizeof(SFuncExtraInfo)); + if (retrieveRsp.pFuncExtraInfos == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + goto RETRIEVE_FUNC_OVER; + } + for (int32_t i = 0; i < retrieveReq.numOfFuncs; ++i) { char *funcName = taosArrayGet(retrieveReq.pFuncNames, i); @@ -451,6 +527,11 @@ static int32_t mndProcessRetrieveFuncReq(SRpcMsg *pReq) { } } taosArrayPush(retrieveRsp.pFuncInfos, &funcInfo); + SFuncExtraInfo extraInfo = {0}; + extraInfo.funcVersion = pFunc->funcVersion; + extraInfo.funcCreatedTime = pFunc->createdTime; + taosArrayPush(retrieveRsp.pFuncExtraInfos, &extraInfo); + mndReleaseFunc(pMnode, pFunc); } @@ -547,7 +628,7 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->bufSize, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - char* language = ""; + char *language = ""; if (pFunc->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) { language = "C"; } else if (pFunc->scriptType == TSDB_FUNC_SCRIPT_PYTHON) { @@ -559,13 +640,18 @@ static int32_t mndRetrieveFuncs(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBl colDataSetVal(pColInfo, numOfRows, (const char *)varLang, false); pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - int32_t varCodeLen = (pFunc->codeSize + VARSTR_HEADER_SIZE) > TSDB_MAX_BINARY_LEN ? TSDB_MAX_BINARY_LEN : pFunc->codeSize + VARSTR_HEADER_SIZE; - char *b4 = taosMemoryMalloc(varCodeLen); + int32_t varCodeLen = (pFunc->codeSize + VARSTR_HEADER_SIZE) > TSDB_MAX_BINARY_LEN + ? TSDB_MAX_BINARY_LEN + : pFunc->codeSize + VARSTR_HEADER_SIZE; + char *b4 = taosMemoryMalloc(varCodeLen); memcpy(varDataVal(b4), pFunc->pCode, varCodeLen - VARSTR_HEADER_SIZE); varDataSetLen(b4, varCodeLen - VARSTR_HEADER_SIZE); - colDataSetVal(pColInfo, numOfRows, (const char*)b4, false); + colDataSetVal(pColInfo, numOfRows, (const char *)b4, false); taosMemoryFree(b4); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)&pFunc->funcVersion, false); + numOfRows++; sdbRelease(pSdb, pFunc); } diff --git a/source/dnode/mnode/impl/src/mndMain.c b/source/dnode/mnode/impl/src/mndMain.c index 5c20887cf5b548fc291d5ae37aac10e9a1f506b1..a9f52128a685215fed934eaddf684cc77ea1ff30 100644 --- a/source/dnode/mnode/impl/src/mndMain.c +++ b/source/dnode/mnode/impl/src/mndMain.c @@ -491,7 +491,10 @@ static void mndSetOptions(SMnode *pMnode, const SMnodeOpt *pOption) { pMnode->selfDnodeId = pOption->dnodeId; pMnode->syncMgmt.selfIndex = pOption->selfIndex; pMnode->syncMgmt.numOfReplicas = pOption->numOfReplicas; + pMnode->syncMgmt.numOfTotalReplicas = pOption->numOfTotalReplicas; + pMnode->syncMgmt.lastIndex = pOption->lastIndex; memcpy(pMnode->syncMgmt.replicas, pOption->replicas, sizeof(pOption->replicas)); + memcpy(pMnode->syncMgmt.nodeRoles, pOption->nodeRoles, sizeof(pOption->nodeRoles)); } SMnode *mndOpen(const char *path, const SMnodeOpt *pOption) { @@ -582,6 +585,16 @@ int32_t mndStart(SMnode *pMnode) { return mndInitTimer(pMnode); } +int32_t mndIsCatchUp(SMnode *pMnode) { + int64_t rid = pMnode->syncMgmt.sync; + return syncIsCatchUp(rid); +} + +ESyncRole mndGetRole(SMnode *pMnode){ + int64_t rid = pMnode->syncMgmt.sync; + return syncGetRole(rid); +} + void mndStop(SMnode *pMnode) { mndSetStop(pMnode); mndSyncStop(pMnode); diff --git a/source/dnode/mnode/impl/src/mndMnode.c b/source/dnode/mnode/impl/src/mndMnode.c index 50fab447e3c7b8fb85ece7c025059358f6ab9e6f..5e3476859a446340212ed9d687f84a87dd16c3ea 100644 --- a/source/dnode/mnode/impl/src/mndMnode.c +++ b/source/dnode/mnode/impl/src/mndMnode.c @@ -23,7 +23,7 @@ #include "mndTrans.h" #include "tmisce.h" -#define MNODE_VER_NUMBER 1 +#define MNODE_VER_NUMBER 2 #define MNODE_RESERVE_SIZE 64 static int32_t mndCreateDefaultMnode(SMnode *pMnode); @@ -53,6 +53,7 @@ int32_t mndInitMnode(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_MND_CREATE_MNODE, mndProcessCreateMnodeReq); mndSetMsgHandle(pMnode, TDMT_DND_CREATE_MNODE_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_MNODE_TYPE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE, mndProcessAlterMnodeReq); mndSetMsgHandle(pMnode, TDMT_MND_ALTER_MNODE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_DROP_MNODE, mndProcessDropMnodeReq); @@ -126,6 +127,8 @@ static SSdbRaw *mndMnodeActionEncode(SMnodeObj *pObj) { SDB_SET_INT32(pRaw, dataPos, pObj->id, _OVER) SDB_SET_INT64(pRaw, dataPos, pObj->createdTime, _OVER) SDB_SET_INT64(pRaw, dataPos, pObj->updateTime, _OVER) + SDB_SET_INT32(pRaw, dataPos, pObj->role, _OVER) + SDB_SET_INT64(pRaw, dataPos, pObj->lastIndex, _OVER) SDB_SET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER) terrno = 0; @@ -149,7 +152,7 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) return NULL; - if (sver != MNODE_VER_NUMBER) { + if (sver != 1 && sver != 2) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -164,6 +167,10 @@ static SSdbRow *mndMnodeActionDecode(SSdbRaw *pRaw) { SDB_GET_INT32(pRaw, dataPos, &pObj->id, _OVER) SDB_GET_INT64(pRaw, dataPos, &pObj->createdTime, _OVER) SDB_GET_INT64(pRaw, dataPos, &pObj->updateTime, _OVER) + if(sver >=2){ + SDB_GET_INT32(pRaw, dataPos, &pObj->role, _OVER) + SDB_GET_INT64(pRaw, dataPos, &pObj->lastIndex, _OVER) + } SDB_GET_RESERVE(pRaw, dataPos, MNODE_RESERVE_SIZE, _OVER) terrno = 0; @@ -204,7 +211,9 @@ static int32_t mndMnodeActionDelete(SSdb *pSdb, SMnodeObj *pObj) { static int32_t mndMnodeActionUpdate(SSdb *pSdb, SMnodeObj *pOld, SMnodeObj *pNew) { mTrace("mnode:%d, perform update action, old row:%p new row:%p", pOld->id, pOld, pNew); + pOld->role = pNew->role; pOld->updateTime = pNew->updateTime; + pOld->lastIndex = pNew->lastIndex; mndReloadSyncConfig(pSdb->pMnode); return 0; @@ -302,6 +311,28 @@ static int32_t mndBuildCreateMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *p return 0; } +static int32_t mndBuildAlterMnodeTypeRedoAction(STrans *pTrans, + SDAlterMnodeTypeReq *pAlterMnodeTypeReq, SEpSet *pAlterMnodeTypeEpSet) { + int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterMnodeTypeReq); + void *pReq = taosMemoryMalloc(contLen); + tSerializeSDCreateMnodeReq(pReq, contLen, pAlterMnodeTypeReq); + + STransAction action = { + .epSet = *pAlterMnodeTypeEpSet, + .pCont = pReq, + .contLen = contLen, + .msgType = TDMT_DND_ALTER_MNODE_TYPE, + .retryCode = TSDB_CODE_MNODE_NOT_CATCH_UP, + .acceptableCode = TSDB_CODE_MNODE_ALREADY_IS_VOTER, + }; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + return 0; +} + static int32_t mndBuildAlterMnodeRedoAction(STrans *pTrans, SDCreateMnodeReq *pAlterReq, SEpSet *pAlterEpSet) { int32_t contLen = tSerializeSDCreateMnodeReq(NULL, 0, pAlterReq); void *pReq = taosMemoryMalloc(contLen); @@ -347,6 +378,7 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno SSdb *pSdb = pMnode->pSdb; void *pIter = NULL; int32_t numOfReplicas = 0; + int32_t numOfLearnerReplicas = 0; SDCreateMnodeReq createReq = {0}; SEpSet createEpset = {0}; @@ -355,18 +387,29 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); if (pIter == NULL) break; - createReq.replicas[numOfReplicas].id = pMObj->id; - createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; - memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + if(pMObj->role == TAOS_SYNC_ROLE_VOTER){ + createReq.replicas[numOfReplicas].id = pMObj->id; + createReq.replicas[numOfReplicas].port = pMObj->pDnode->port; + memcpy(createReq.replicas[numOfReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + numOfReplicas++; + } + else{ + createReq.learnerReplicas[numOfLearnerReplicas].id = pMObj->id; + createReq.learnerReplicas[numOfLearnerReplicas].port = pMObj->pDnode->port; + memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + numOfLearnerReplicas++; + } - numOfReplicas++; sdbRelease(pSdb, pMObj); } - createReq.replica = numOfReplicas + 1; - createReq.replicas[numOfReplicas].id = pDnode->id; - createReq.replicas[numOfReplicas].port = pDnode->port; - memcpy(createReq.replicas[numOfReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + createReq.replica = numOfReplicas; + createReq.learnerReplica = numOfLearnerReplicas + 1; + createReq.learnerReplicas[numOfLearnerReplicas].id = pDnode->id; + createReq.learnerReplicas[numOfLearnerReplicas].port = pDnode->port; + memcpy(createReq.learnerReplicas[numOfLearnerReplicas].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + + createReq.lastIndex = pObj->lastIndex; createEpset.inUse = 0; createEpset.numOfEps = 1; @@ -378,23 +421,78 @@ static int32_t mndSetCreateMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDno return 0; } +static int32_t mndSetAlterMnodeTypeRedoActions(SMnode *pMnode, STrans *pTrans, SDnodeObj *pDnode, SMnodeObj *pObj) { + SSdb *pSdb = pMnode->pSdb; + void *pIter = NULL; + SDAlterMnodeTypeReq alterReq = {0}; + SEpSet createEpset = {0}; + + while (1) { + SMnodeObj *pMObj = NULL; + pIter = sdbFetch(pSdb, SDB_MNODE, pIter, (void **)&pMObj); + if (pIter == NULL) break; + + if(pMObj->role == TAOS_SYNC_ROLE_VOTER){ + alterReq.replicas[alterReq.replica].id = pMObj->id; + alterReq.replicas[alterReq.replica].port = pMObj->pDnode->port; + memcpy(alterReq.replicas[alterReq.replica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.replica++; + } + else{ + alterReq.learnerReplicas[alterReq.learnerReplica].id = pMObj->id; + alterReq.learnerReplicas[alterReq.learnerReplica].port = pMObj->pDnode->port; + memcpy(alterReq.learnerReplicas[alterReq.learnerReplica].fqdn, pMObj->pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.learnerReplica++; + } + + sdbRelease(pSdb, pMObj); + } + + alterReq.replicas[alterReq.replica].id = pDnode->id; + alterReq.replicas[alterReq.replica].port = pDnode->port; + memcpy(alterReq.replicas[alterReq.replica].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + alterReq.replica++; + + alterReq.lastIndex = pObj->lastIndex; + + createEpset.inUse = 0; + createEpset.numOfEps = 1; + createEpset.eps[0].port = pDnode->port; + memcpy(createEpset.eps[0].fqdn, pDnode->fqdn, TSDB_FQDN_LEN); + + if (mndBuildAlterMnodeTypeRedoAction(pTrans, &alterReq, &createEpset) != 0) return -1; + + return 0; +} + static int32_t mndCreateMnode(SMnode *pMnode, SRpcMsg *pReq, SDnodeObj *pDnode, SMCreateMnodeReq *pCreate) { int32_t code = -1; - SMnodeObj mnodeObj = {0}; - mnodeObj.id = pDnode->id; - mnodeObj.createdTime = taosGetTimestampMs(); - mnodeObj.updateTime = mnodeObj.createdTime; - STrans *pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_GLOBAL, pReq, "create-mnode"); if (pTrans == NULL) goto _OVER; mndTransSetSerial(pTrans); mInfo("trans:%d, used to create mnode:%d", pTrans->id, pCreate->dnodeId); if (mndTrancCheckConflict(pMnode, pTrans) != 0) goto _OVER; + SMnodeObj mnodeObj = {0}; + mnodeObj.id = pDnode->id; + mnodeObj.createdTime = taosGetTimestampMs(); + mnodeObj.updateTime = mnodeObj.createdTime; + mnodeObj.role = TAOS_SYNC_ROLE_LEARNER; + mnodeObj.lastIndex = pMnode->applied; + if (mndSetCreateMnodeRedoActions(pMnode, pTrans, pDnode, &mnodeObj) != 0) goto _OVER; if (mndSetCreateMnodeRedoLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; - if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeObj) != 0) goto _OVER; + + SMnodeObj mnodeLeaderObj = {0}; + mnodeLeaderObj.id = pDnode->id; + mnodeLeaderObj.createdTime = taosGetTimestampMs(); + mnodeLeaderObj.updateTime = mnodeLeaderObj.createdTime; + mnodeLeaderObj.role = TAOS_SYNC_ROLE_VOTER; + mnodeLeaderObj.lastIndex = pMnode->applied + 1; + + if (mndSetAlterMnodeTypeRedoActions(pMnode, pTrans, pDnode, &mnodeLeaderObj) != 0) goto _OVER; + if (mndSetCreateMnodeCommitLogs(pMnode, pTrans, &mnodeLeaderObj) != 0) goto _OVER; if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; code = 0; @@ -514,6 +612,7 @@ static int32_t mndSetDropMnodeRedoActions(SMnode *pMnode, STrans *pTrans, SDnode int32_t mndSetDropMnodeInfoToTrans(SMnode *pMnode, STrans *pTrans, SMnodeObj *pObj, bool force) { if (pObj == NULL) return 0; + pObj->lastIndex = pMnode->applied; if (mndSetDropMnodeRedoActions(pMnode, pTrans, pObj->pDnode, pObj, force) != 0) return -1; if (mndSetDropMnodeCommitLogs(pMnode, pTrans, pObj) != 0) return -1; return 0; @@ -730,7 +829,8 @@ static void mndReloadSyncConfig(SMnode *pMnode) { void *pIter = NULL; int32_t updatingMnodes = 0; int32_t readyMnodes = 0; - SSyncCfg cfg = {.myIndex = -1}; + SSyncCfg cfg = {.myIndex = -1, .lastIndex = 0,}; + SyncIndex maxIndex = 0; while (1) { pIter = sdbFetchAll(pSdb, SDB_MNODE, pIter, (void **)&pObj, &objStatus, false); @@ -745,26 +845,41 @@ static void mndReloadSyncConfig(SMnode *pMnode) { } if (objStatus == SDB_STATUS_READY || objStatus == SDB_STATUS_CREATING) { - SNodeInfo *pNode = &cfg.nodeInfo[cfg.replicaNum]; + SNodeInfo *pNode = &cfg.nodeInfo[cfg.totalReplicaNum]; pNode->nodeId = pObj->pDnode->id; pNode->clusterId = mndGetClusterId(pMnode); pNode->nodePort = pObj->pDnode->port; + pNode->nodeRole = pObj->role; tstrncpy(pNode->nodeFqdn, pObj->pDnode->fqdn, TSDB_FQDN_LEN); (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, ep:%s:%u dnode:%d", pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); if (pObj->pDnode->id == pMnode->selfDnodeId) { - cfg.myIndex = cfg.replicaNum; + cfg.myIndex = cfg.totalReplicaNum; + } + if(pNode->nodeRole == TAOS_SYNC_ROLE_VOTER){ + cfg.replicaNum++; + } + cfg.totalReplicaNum++; + if(pObj->lastIndex > cfg.lastIndex){ + cfg.lastIndex = pObj->lastIndex; + } + } + + if (objStatus == SDB_STATUS_DROPPING) { + if(pObj->lastIndex > cfg.lastIndex){ + cfg.lastIndex = pObj->lastIndex; } - cfg.replicaNum++; } + mInfo("vgId:1, mnode:%d, role:%d, lastIndex:%" PRId64, pObj->id, pObj->role, pObj->lastIndex); + sdbReleaseLock(pSdb, pObj, false); } - if (readyMnodes <= 0 || updatingMnodes <= 0) { - mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); - return; - } + //if (readyMnodes <= 0 || updatingMnodes <= 0) { + // mInfo("vgId:1, mnode sync not reconfig since readyMnodes:%d updatingMnodes:%d", readyMnodes, updatingMnodes); + // return; + //} if (cfg.myIndex == -1) { #if 1 @@ -777,12 +892,14 @@ static void mndReloadSyncConfig(SMnode *pMnode) { return; } - if (updatingMnodes > 0) { - mInfo("vgId:1, mnode sync reconfig, replica:%d myIndex:%d", cfg.replicaNum, cfg.myIndex); - for (int32_t i = 0; i < cfg.replicaNum; ++i) { + if (pMnode->syncMgmt.sync > 0) { + mInfo("vgId:1, mnode sync reconfig, totalReplica:%d replica:%d myIndex:%d", + cfg.totalReplicaNum, cfg.replicaNum, cfg.myIndex); + + for (int32_t i = 0; i < cfg.totalReplicaNum; ++i) { SNodeInfo *pNode = &cfg.nodeInfo[i]; - mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, - pNode->clusterId); + mInfo("vgId:1, index:%d, ep:%s:%u dnode:%d cluster:%" PRId64 " role:%d", i, pNode->nodeFqdn, pNode->nodePort, + pNode->nodeId, pNode->clusterId, pNode->nodeRole); } int32_t code = syncReconfig(pMnode->syncMgmt.sync, &cfg); diff --git a/source/dnode/mnode/impl/src/mndPrivilege.c b/source/dnode/mnode/impl/src/mndPrivilege.c index ccb4140b83bc6bcc58eacdd52b357fc19ed3455b..de0374c6e89c506ce1b9cad966e2a4dd8905b7b7 100644 --- a/source/dnode/mnode/impl/src/mndPrivilege.c +++ b/source/dnode/mnode/impl/src/mndPrivilege.c @@ -35,6 +35,7 @@ int32_t mndCheckTopicPrivilegeByName(SMnode *pMnode, const char *user, EOperType int32_t mndSetUserAuthRsp(SMnode *pMnode, SUserObj *pUser, SGetUserAuthRsp *pRsp) { memcpy(pRsp->user, pUser->user, TSDB_USER_LEN); pRsp->superAuth = 1; + pRsp->enable = pUser->enable; pRsp->version = pUser->authVersion; return 0; } diff --git a/source/dnode/mnode/impl/src/mndProfile.c b/source/dnode/mnode/impl/src/mndProfile.c index 50e502f4ab9b240e8518bdb4fe531929149e974d..2ebb5aeb9904cbabcc95086846b9ded2cdd2f1ef 100644 --- a/source/dnode/mnode/impl/src/mndProfile.c +++ b/source/dnode/mnode/impl/src/mndProfile.c @@ -283,6 +283,7 @@ static int32_t mndProcessConnectReq(SRpcMsg *pReq) { connectRsp.connType = connReq.connType; connectRsp.dnodeNum = mndGetDnodeSize(pMnode); connectRsp.svrTimestamp = taosGetTimestampSec(); + connectRsp.passVer = pUser->passVersion; strcpy(connectRsp.sVer, version); snprintf(connectRsp.sDetailVer, sizeof(connectRsp.sDetailVer), "ver:%s\nbuild:%s\ngitinfo:%s", version, buildinfo, @@ -547,6 +548,16 @@ static int32_t mndProcessQueryHeartBeat(SMnode *pMnode, SRpcMsg *pMsg, SClientHb } break; } + case HEARTBEAT_KEY_USER_PASSINFO: { + void *rspMsg = NULL; + int32_t rspLen = 0; + mndValidateUserPassInfo(pMnode, kv->value, kv->valueLen / sizeof(SUserPassVersion), &rspMsg, &rspLen); + if (rspMsg && rspLen > 0) { + SKv kv1 = {.key = HEARTBEAT_KEY_USER_PASSINFO, .valueLen = rspLen, .value = rspMsg}; + taosArrayPush(hbRsp.info, &kv1); + } + break; + } default: mError("invalid kv key:%d", kv->key); hbRsp.status = TSDB_CODE_APP_ERROR; diff --git a/source/dnode/mnode/impl/src/mndScheduler.c b/source/dnode/mnode/impl/src/mndScheduler.c index d1671aa12a5b4c5dfbc2a3a1ede49825b7e384a2..734f624be0e19c942c10244f28263570d6ea4504 100644 --- a/source/dnode/mnode/impl/src/mndScheduler.c +++ b/source/dnode/mnode/impl/src/mndScheduler.c @@ -138,7 +138,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream for (int32_t j = 0; j < sinkLvSize; j++) { SStreamTask* pLastLevelTask = taosArrayGetP(sinkLv, j); if (pLastLevelTask->nodeId == pVgInfo->vgId) { - pVgInfo->taskId = pLastLevelTask->taskId; + pVgInfo->taskId = pLastLevelTask->id.taskId; break; } } @@ -149,7 +149,7 @@ int32_t mndAddDispatcherToInnerTask(SMnode* pMnode, SStreamObj* pStream, SStream SArray* pArray = taosArrayGetP(pStream->tasks, 0); // one sink only SStreamTask* lastLevelTask = taosArrayGetP(pArray, 0); - pTask->fixedEpDispatcher.taskId = lastLevelTask->taskId; + pTask->fixedEpDispatcher.taskId = lastLevelTask->id.taskId; pTask->fixedEpDispatcher.nodeId = lastLevelTask->nodeId; pTask->fixedEpDispatcher.epSet = lastLevelTask->epSet; } @@ -224,7 +224,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -260,7 +260,7 @@ int32_t mndAddShuffleSinkTasksToStream(SMnode* pMnode, SStreamObj* pStream) { int32_t mndAddFixedSinkTaskToStream(SMnode* pMnode, SStreamObj* pStream) { SArray* tasks = taosArrayGetP(pStream->tasks, 0); - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -350,12 +350,13 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { return -1; } - pInnerTask = tNewSStreamTask(pStream->uid); + pInnerTask = tNewStreamTask(pStream->uid); if (pInnerTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; qDestroyQueryPlan(pPlan); return -1; } + pInnerTask->fillHistory = pStream->fillHistory; mndAddTaskToTaskSet(taskInnerLevel, pInnerTask); @@ -421,7 +422,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; sdbRelease(pSdb, pVgroup); @@ -440,7 +441,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pTask->dispatchMsgType = TDMT_STREAM_TASK_DISPATCH; pTask->outputType = TASK_OUTPUT__FIXED_DISPATCH; - pTask->fixedEpDispatcher.taskId = pInnerTask->taskId; + pTask->fixedEpDispatcher.taskId = pInnerTask->id.taskId; pTask->fixedEpDispatcher.nodeId = pInnerTask->nodeId; pTask->fixedEpDispatcher.epSet = pInnerTask->epSet; @@ -460,7 +461,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { pEpInfo->childId = pTask->selfChildId; pEpInfo->epSet = pTask->epSet; pEpInfo->nodeId = pTask->nodeId; - pEpInfo->taskId = pTask->taskId; + pEpInfo->taskId = pTask->id.taskId; taosArrayPush(pInnerTask->childEpInfo, &pEpInfo); sdbRelease(pSdb, pVgroup); } @@ -491,7 +492,7 @@ int32_t mndScheduleStream(SMnode* pMnode, SStreamObj* pStream) { continue; } - SStreamTask* pTask = tNewSStreamTask(pStream->uid); + SStreamTask* pTask = tNewStreamTask(pStream->uid); if (pTask == NULL) { sdbRelease(pSdb, pVgroup); qDestroyQueryPlan(pPlan); diff --git a/source/dnode/mnode/impl/src/mndStb.c b/source/dnode/mnode/impl/src/mndStb.c index b3da66a035e7f071b5245dad4ac8edda9abc1151..63bcef2a5bd80839ad5a7e6e052f18b46f1106c4 100644 --- a/source/dnode/mnode/impl/src/mndStb.c +++ b/source/dnode/mnode/impl/src/mndStb.c @@ -2614,6 +2614,13 @@ void mndExtractDbNameFromStbFullName(const char *stbFullName, char *dst) { tNameGetFullDbName(&name, dst); } +void mndExtractShortDbNameFromStbFullName(const char *stbFullName, char *dst) { + SName name = {0}; + tNameFromString(&name, stbFullName, T_NAME_ACCT | T_NAME_DB | T_NAME_TABLE); + + tNameGetDbName(&name, dst); +} + void mndExtractTbNameFromStbFullName(const char *stbFullName, char *dst, int32_t dstSize) { int32_t pos = -1; int32_t num = 0; diff --git a/source/dnode/mnode/impl/src/mndStream.c b/source/dnode/mnode/impl/src/mndStream.c index ff759f5e785728998d6b8c90824a244cd80ff3f6..76bb144fcb0b33ff5a832b01c9dd0b9ad9b95175 100644 --- a/source/dnode/mnode/impl/src/mndStream.c +++ b/source/dnode/mnode/impl/src/mndStream.c @@ -35,12 +35,12 @@ static int32_t mndStreamActionInsert(SSdb *pSdb, SStreamObj *pStream); static int32_t mndStreamActionDelete(SSdb *pSdb, SStreamObj *pStream); -static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pStream, SStreamObj *pNewStream); +static int32_t mndStreamActionUpdate(SSdb *pSdb, SStreamObj *pOldStream, SStreamObj *pNewStream); static int32_t mndProcessCreateStreamReq(SRpcMsg *pReq); static int32_t mndProcessDropStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamCheckpointTmr(SRpcMsg *pReq); -// static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); -/*static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq);*/ +static int32_t mndProcessStreamDoCheckpoint(SRpcMsg *pReq); +static int32_t mndProcessRecoverStreamReq(SRpcMsg *pReq); static int32_t mndProcessStreamMetaReq(SRpcMsg *pReq); static int32_t mndGetStreamMeta(SRpcMsg *pReq, SShowObj *pShow, STableMetaRsp *pMeta); static int32_t mndRetrieveStream(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows); @@ -418,7 +418,7 @@ FAIL: int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { SEncoder encoder; tEncoderInit(&encoder, NULL, 0); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); int32_t size = encoder.pos; int32_t tlen = sizeof(SMsgHead) + size; tEncoderClear(&encoder); @@ -430,7 +430,7 @@ int32_t mndPersistTaskDeployReq(STrans *pTrans, const SStreamTask *pTask) { ((SMsgHead *)buf)->vgId = htonl(pTask->nodeId); void *abuf = POINTER_SHIFT(buf, sizeof(SMsgHead)); tEncoderInit(&encoder, abuf, size); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); STransAction action = {0}; @@ -601,7 +601,7 @@ static int32_t mndPersistTaskDropReq(STrans *pTrans, SStreamTask *pTask) { return -1; } pReq->head.vgId = htonl(pTask->nodeId); - pReq->taskId = pTask->taskId; + pReq->taskId = pTask->id.taskId; STransAction action = {0}; memcpy(&action.epSet, &pTask->epSet, sizeof(SEpSet)); action.pCont = pReq; @@ -1209,7 +1209,7 @@ static int32_t mndRetrieveStreamTask(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock // task id pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); - colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->taskId, false); + colDataSetVal(pColInfo, numOfRows, (const char *)&pTask->id.taskId, false); // node type char nodeType[20 + VARSTR_HEADER_SIZE] = {0}; diff --git a/source/dnode/mnode/impl/src/mndSubscribe.c b/source/dnode/mnode/impl/src/mndSubscribe.c index 6a745a0a4f94ff4668e55927b99596bfd705d5ca..49921c9a1a0572c8498ca1ff01fb5491b23cd595 100644 --- a/source/dnode/mnode/impl/src/mndSubscribe.c +++ b/source/dnode/mnode/impl/src/mndSubscribe.c @@ -671,7 +671,7 @@ static int32_t mndProcessRebalanceReq(SRpcMsg *pMsg) { // possibly no vg is changed // when each topic is re-balanced, issue an trans to save the results in sdb. if (mndPersistRebResult(pMnode, pMsg, &rebOutput) < 0) { - mError("mq re-balance persist output error, possibly vnode splitted or dropped"); + mError("mq re-balance persist output error, possibly vnode splitted or dropped,msg:%s", terrstr()); } taosArrayDestroy(rebOutput.newConsumers); diff --git a/source/dnode/mnode/impl/src/mndSync.c b/source/dnode/mnode/impl/src/mndSync.c index 4965d5c34a2f09b3ce25edf4bc6ed902a2f23a42..0a6df02f5fbf2636de06358831b577917846b797 100644 --- a/source/dnode/mnode/impl/src/mndSync.c +++ b/source/dnode/mnode/impl/src/mndSync.c @@ -237,6 +237,23 @@ static void mndBecomeFollower(const SSyncFSM *pFsm) { taosThreadMutexUnlock(&pMgmt->lock); } +static void mndBecomeLearner(const SSyncFSM *pFsm) { + SMnode *pMnode = pFsm->data; + SSyncMgmt *pMgmt = &pMnode->syncMgmt; + mInfo("vgId:1, become learner"); + + taosThreadMutexLock(&pMgmt->lock); + if (pMgmt->transId != 0) { + mInfo("vgId:1, become learner and post sem, trans:%d, failed to propose since not leader", pMgmt->transId); + pMgmt->transId = 0; + pMgmt->transSec = 0; + pMgmt->transSeq = 0; + pMgmt->errCode = TSDB_CODE_SYN_NOT_LEADER; + tsem_post(&pMgmt->syncSem); + } + taosThreadMutexUnlock(&pMgmt->lock); +} + static void mndBecomeLeader(const SSyncFSM *pFsm) { mInfo("vgId:1, become leader"); SMnode *pMnode = pFsm->data; @@ -278,6 +295,7 @@ SSyncFSM *mndSyncMakeFsm(SMnode *pMnode) { pFsm->FpReConfigCb = NULL; pFsm->FpBecomeLeaderCb = mndBecomeLeader; pFsm->FpBecomeFollowerCb = mndBecomeFollower; + pFsm->FpBecomeLearnerCb = mndBecomeLearner; pFsm->FpGetSnapshot = mndSyncGetSnapshot; pFsm->FpGetSnapshotInfo = mndSyncGetSnapshotInfo; pFsm->FpSnapshotStartRead = mndSnapshotStartRead; @@ -317,13 +335,16 @@ int32_t mndInitSync(SMnode *pMnode) { mInfo("vgId:1, start to open sync, replica:%d selfIndex:%d", pMgmt->numOfReplicas, pMgmt->selfIndex); SSyncCfg *pCfg = &syncInfo.syncCfg; + pCfg->totalReplicaNum = pMgmt->numOfTotalReplicas; pCfg->replicaNum = pMgmt->numOfReplicas; pCfg->myIndex = pMgmt->selfIndex; - for (int32_t i = 0; i < pMgmt->numOfReplicas; ++i) { + pCfg->lastIndex = pMgmt->lastIndex; + for (int32_t i = 0; i < pMgmt->numOfTotalReplicas; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pMgmt->replicas[i].id; pNode->nodePort = pMgmt->replicas[i].port; tstrncpy(pNode->nodeFqdn, pMgmt->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeRole = pMgmt->nodeRoles[i]; (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); mInfo("vgId:1, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); diff --git a/source/dnode/mnode/impl/src/mndUser.c b/source/dnode/mnode/impl/src/mndUser.c index 92b73aed964ee47f3751ffae84af81ac6f5458de..f6e5895cda2ee31b838f1203c861a04f3c49ac17 100644 --- a/source/dnode/mnode/impl/src/mndUser.c +++ b/source/dnode/mnode/impl/src/mndUser.c @@ -18,11 +18,12 @@ #include "mndDb.h" #include "mndPrivilege.h" #include "mndShow.h" +#include "mndStb.h" #include "mndTopic.h" #include "mndTrans.h" #include "tbase64.h" -#define USER_VER_NUMBER 2 +#define USER_VER_NUMBER 4 #define USER_RESERVE_SIZE 64 static int32_t mndCreateDefaultUsers(SMnode *pMnode); @@ -124,9 +125,40 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); + int32_t numOfReadStbs = taosHashGetSize(pUser->readTbs); + int32_t numOfWriteStbs = taosHashGetSize(pUser->writeTbs); int32_t numOfTopics = taosHashGetSize(pUser->topics); - int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + (numOfReadDbs + numOfWriteDbs) * TSDB_DB_FNAME_LEN + - numOfTopics * TSDB_TOPIC_FNAME_LEN; + int32_t numOfUseDbs = taosHashGetSize(pUser->useDbs); + int32_t size = sizeof(SUserObj) + USER_RESERVE_SIZE + + (numOfReadDbs + numOfWriteDbs + numOfUseDbs) * TSDB_DB_FNAME_LEN + numOfTopics * TSDB_TOPIC_FNAME_LEN; + + char *stb = taosHashIterate(pUser->readTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + size += sizeof(int32_t); + size += keyLen; + + size_t valueLen = 0; + valueLen = strlen(stb); + size += sizeof(int32_t); + size += valueLen; + stb = taosHashIterate(pUser->readTbs, stb); + } + + stb = taosHashIterate(pUser->writeTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + size += sizeof(int32_t); + size += keyLen; + + size_t valueLen = 0; + valueLen = strlen(stb); + size += sizeof(int32_t); + size += valueLen; + stb = taosHashIterate(pUser->writeTbs, stb); + } SSdbRaw *pRaw = sdbAllocRaw(SDB_USER, USER_VER_NUMBER, size); if (pRaw == NULL) goto _OVER; @@ -142,6 +174,7 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { SDB_SET_INT8(pRaw, dataPos, pUser->enable, _OVER) SDB_SET_INT8(pRaw, dataPos, pUser->reserve, _OVER) SDB_SET_INT32(pRaw, dataPos, pUser->authVersion, _OVER) + SDB_SET_INT32(pRaw, dataPos, pUser->passVersion, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfReadDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfWriteDbs, _OVER) SDB_SET_INT32(pRaw, dataPos, numOfTopics, _OVER) @@ -164,6 +197,49 @@ SSdbRaw *mndUserActionEncode(SUserObj *pUser) { topic = taosHashIterate(pUser->topics, topic); } + SDB_SET_INT32(pRaw, dataPos, numOfReadStbs, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfWriteStbs, _OVER) + SDB_SET_INT32(pRaw, dataPos, numOfUseDbs, _OVER) + + stb = taosHashIterate(pUser->readTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + size_t valueLen = 0; + valueLen = strlen(stb) + 1; + SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER); + stb = taosHashIterate(pUser->readTbs, stb); + } + + stb = taosHashIterate(pUser->writeTbs, NULL); + while (stb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(stb, &keyLen); + SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + size_t valueLen = 0; + valueLen = strlen(stb) + 1; + SDB_SET_INT32(pRaw, dataPos, valueLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, stb, valueLen, _OVER); + stb = taosHashIterate(pUser->writeTbs, stb); + } + + int32_t *useDb = taosHashIterate(pUser->useDbs, NULL); + while (useDb != NULL) { + size_t keyLen = 0; + void *key = taosHashGetKey(useDb, &keyLen); + SDB_SET_INT32(pRaw, dataPos, keyLen, _OVER) + SDB_SET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + SDB_SET_INT32(pRaw, dataPos, *useDb, _OVER) + useDb = taosHashIterate(pUser->writeTbs, useDb); + } + SDB_SET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) SDB_SET_DATALEN(pRaw, dataPos, _OVER) @@ -188,7 +264,7 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { int8_t sver = 0; if (sdbGetRawSoftVer(pRaw, &sver) != 0) goto _OVER; - if (sver != 1 && sver != 2) { + if (sver < 1 || sver > USER_VER_NUMBER) { terrno = TSDB_CODE_SDB_INVALID_DATA_VER; goto _OVER; } @@ -210,6 +286,9 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { SDB_GET_INT8(pRaw, dataPos, &pUser->enable, _OVER) SDB_GET_INT8(pRaw, dataPos, &pUser->reserve, _OVER) SDB_GET_INT32(pRaw, dataPos, &pUser->authVersion, _OVER) + if (sver >= 4) { + SDB_GET_INT32(pRaw, dataPos, &pUser->passVersion, _OVER) + } int32_t numOfReadDbs = 0; int32_t numOfWriteDbs = 0; @@ -249,6 +328,75 @@ static SSdbRow *mndUserActionDecode(SSdbRaw *pRaw) { } } + if (sver >= 3) { + int32_t numOfReadStbs = 0; + int32_t numOfWriteStbs = 0; + int32_t numOfUseDbs = 0; + SDB_GET_INT32(pRaw, dataPos, &numOfReadStbs, _OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfWriteStbs, _OVER) + SDB_GET_INT32(pRaw, dataPos, &numOfUseDbs, _OVER) + + pUser->readTbs = + taosHashInit(numOfReadStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pUser->writeTbs = + taosHashInit(numOfWriteStbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + pUser->useDbs = taosHashInit(numOfUseDbs, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + + for (int32_t i = 0; i < numOfReadStbs; ++i) { + int32_t keyLen = 0; + SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); + + char *key = taosMemoryCalloc(keyLen, sizeof(char)); + memset(key, 0, keyLen); + SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + int32_t valuelen = 0; + SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); + char *value = taosMemoryCalloc(valuelen, sizeof(char)); + memset(value, 0, valuelen); + SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) + + taosHashPut(pUser->readTbs, key, keyLen, value, valuelen); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfWriteStbs; ++i) { + int32_t keyLen = 0; + SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); + + char *key = taosMemoryCalloc(keyLen, sizeof(char)); + memset(key, 0, keyLen); + SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + int32_t valuelen = 0; + SDB_GET_INT32(pRaw, dataPos, &valuelen, _OVER); + char *value = taosMemoryCalloc(valuelen, sizeof(char)); + memset(value, 0, valuelen); + SDB_GET_BINARY(pRaw, dataPos, value, valuelen, _OVER) + + taosHashPut(pUser->writeTbs, key, keyLen, value, valuelen); + + taosMemoryFree(key); + taosMemoryFree(value); + } + + for (int32_t i = 0; i < numOfUseDbs; ++i) { + int32_t keyLen = 0; + SDB_GET_INT32(pRaw, dataPos, &keyLen, _OVER); + + char *key = taosMemoryCalloc(keyLen, sizeof(char)); + memset(key, 0, keyLen); + SDB_GET_BINARY(pRaw, dataPos, key, keyLen, _OVER); + + int32_t ref = 0; + SDB_GET_INT32(pRaw, dataPos, &ref, _OVER); + + taosHashPut(pUser->useDbs, key, keyLen, &ref, sizeof(ref)); + } + } + SDB_GET_RESERVE(pRaw, dataPos, USER_RESERVE_SIZE, _OVER) taosInitRWLatch(&pUser->lock); @@ -261,6 +409,9 @@ _OVER: taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); + taosHashCleanup(pUser->readTbs); + taosHashCleanup(pUser->writeTbs); + taosHashCleanup(pUser->useDbs); } taosMemoryFreeClear(pRow); return NULL; @@ -285,6 +436,57 @@ static int32_t mndUserActionInsert(SSdb *pSdb, SUserObj *pUser) { return 0; } +SHashObj *mndDupTableHash(SHashObj *pOld) { + SHashObj *pNew = + taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + char *tb = taosHashIterate(pOld, NULL); + while (tb != NULL) { + size_t keyLen = 0; + char *key = taosHashGetKey(tb, &keyLen); + + int32_t valueLen = strlen(tb) + 1; + if (taosHashPut(pNew, key, keyLen, tb, valueLen) != 0) { + taosHashCancelIterate(pOld, tb); + taosHashCleanup(pNew); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + tb = taosHashIterate(pOld, tb); + } + + return pNew; +} + +SHashObj *mndDupUseDbHash(SHashObj *pOld) { + SHashObj *pNew = + taosHashInit(taosHashGetSize(pOld), taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY), true, HASH_ENTRY_LOCK); + if (pNew == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + + int32_t *db = taosHashIterate(pOld, NULL); + while (db != NULL) { + size_t keyLen = 0; + char *key = taosHashGetKey(db, &keyLen); + + if (taosHashPut(pNew, key, keyLen, db, sizeof(*db)) != 0) { + taosHashCancelIterate(pOld, db); + taosHashCleanup(pNew); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return NULL; + } + db = taosHashIterate(pOld, db); + } + + return pNew; +} + static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { memcpy(pNew, pUser, sizeof(SUserObj)); pNew->authVersion++; @@ -293,7 +495,10 @@ static int32_t mndUserDupObj(SUserObj *pUser, SUserObj *pNew) { taosRLockLatch(&pUser->lock); pNew->readDbs = mndDupDbHash(pUser->readDbs); pNew->writeDbs = mndDupDbHash(pUser->writeDbs); + pNew->readTbs = mndDupTableHash(pUser->readTbs); + pNew->writeTbs = mndDupTableHash(pUser->writeTbs); pNew->topics = mndDupTopicHash(pUser->topics); + pNew->useDbs = mndDupUseDbHash(pUser->useDbs); taosRUnLockLatch(&pUser->lock); if (pNew->readDbs == NULL || pNew->writeDbs == NULL || pNew->topics == NULL) { @@ -306,9 +511,15 @@ static void mndUserFreeObj(SUserObj *pUser) { taosHashCleanup(pUser->readDbs); taosHashCleanup(pUser->writeDbs); taosHashCleanup(pUser->topics); + taosHashCleanup(pUser->readTbs); + taosHashCleanup(pUser->writeTbs); + taosHashCleanup(pUser->useDbs); pUser->readDbs = NULL; pUser->writeDbs = NULL; pUser->topics = NULL; + pUser->readTbs = NULL; + pUser->writeTbs = NULL; + pUser->useDbs = NULL; } static int32_t mndUserActionDelete(SSdb *pSdb, SUserObj *pUser) { @@ -322,12 +533,16 @@ static int32_t mndUserActionUpdate(SSdb *pSdb, SUserObj *pOld, SUserObj *pNew) { taosWLockLatch(&pOld->lock); pOld->updateTime = pNew->updateTime; pOld->authVersion = pNew->authVersion; + pOld->passVersion = pNew->passVersion; pOld->sysInfo = pNew->sysInfo; pOld->enable = pNew->enable; memcpy(pOld->pass, pNew->pass, TSDB_PASSWORD_LEN); TSWAP(pOld->readDbs, pNew->readDbs); TSWAP(pOld->writeDbs, pNew->writeDbs); TSWAP(pOld->topics, pNew->topics); + TSWAP(pOld->readTbs, pNew->readTbs); + TSWAP(pOld->writeTbs, pNew->writeTbs); + TSWAP(pOld->useDbs, pNew->useDbs); taosWUnLockLatch(&pOld->lock); return 0; @@ -498,6 +713,71 @@ SHashObj *mndDupDbHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_DB_FNAM SHashObj *mndDupTopicHash(SHashObj *pOld) { return mndDupObjHash(pOld, TSDB_TOPIC_FNAME_LEN); } +static int32_t mndTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq, + SSdb *pSdb) { + void *pIter = NULL; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + + snprintf(tbFName, sizeof(tbFName), "%s.%s", alterReq->objname, alterReq->tabName); + int32_t len = strlen(tbFName) + 1; + + if (alterReq->tagCond != NULL && alterReq->tagCondLen != 0) { + char *value = taosHashGet(hash, tbFName, len); + if (value != NULL) { + terrno = TSDB_CODE_MND_PRIVILEDGE_EXIST; + return -1; + } + + int32_t condLen = alterReq->tagCondLen; + if (taosHashPut(hash, tbFName, len, alterReq->tagCond, condLen) != 0) { + return -1; + } + } else { + if (taosHashPut(hash, tbFName, len, "t", 2) != 0) { + return -1; + } + } + + int32_t dbKeyLen = strlen(alterReq->objname) + 1; + int32_t ref = 1; + int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen); + if (NULL != currRef) { + ref = (*currRef) + 1; + } + if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) { + return -1; + } + + return 0; +} + +static int32_t mndRemoveTablePriviledge(SMnode *pMnode, SHashObj *hash, SHashObj *useDbHash, SAlterUserReq *alterReq, + SSdb *pSdb) { + void *pIter = NULL; + char tbFName[TSDB_TABLE_FNAME_LEN] = {0}; + snprintf(tbFName, sizeof(tbFName), "%s.%s", alterReq->objname, alterReq->tabName); + int32_t len = strlen(tbFName) + 1; + + if (taosHashRemove(hash, tbFName, len) != 0) { + return -1; + } + + int32_t dbKeyLen = strlen(alterReq->objname) + 1; + int32_t *currRef = taosHashGet(useDbHash, alterReq->objname, dbKeyLen); + if (NULL == currRef || 1 == *currRef) { + if (taosHashRemove(useDbHash, alterReq->objname, dbKeyLen) != 0) { + return -1; + } + return 0; + } + int32_t ref = (*currRef) - 1; + if (taosHashPut(useDbHash, alterReq->objname, dbKeyLen, &ref, sizeof(ref)) != 0) { + return -1; + } + + return 0; +} + static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -543,10 +823,14 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { if (mndUserDupObj(pUser, &newUser) != 0) goto _OVER; + newUser.passVersion = pUser->passVersion; if (alterReq.alterType == TSDB_ALTER_USER_PASSWD) { char pass[TSDB_PASSWORD_LEN + 1] = {0}; taosEncryptPass_c((uint8_t *)alterReq.pass, strlen(alterReq.pass), pass); memcpy(newUser.pass, pass, TSDB_PASSWORD_LEN); + if (0 != strncmp(pUser->pass, pass, TSDB_PASSWORD_LEN)) { + ++newUser.passVersion; + } } if (alterReq.alterType == TSDB_ALTER_USER_SUPERUSER) { @@ -637,6 +921,22 @@ static int32_t mndProcessAlterUserReq(SRpcMsg *pReq) { } } + if (alterReq.alterType == TSDB_ALTER_USER_ADD_READ_TABLE) { + if (mndTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + + if (alterReq.alterType == TSDB_ALTER_USER_ADD_WRITE_TABLE) { + if (mndTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_READ_TABLE) { + if (mndRemoveTablePriviledge(pMnode, newUser.readTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + + if (alterReq.alterType == TSDB_ALTER_USER_REMOVE_WRITE_TABLE) { + if (mndRemoveTablePriviledge(pMnode, newUser.writeTbs, newUser.useDbs, &alterReq, pSdb) != 0) goto _OVER; + } + if (alterReq.alterType == TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC) { int32_t len = strlen(alterReq.objname) + 1; SMqTopicObj *pTopic = mndAcquireTopic(pMnode, alterReq.objname); @@ -830,6 +1130,70 @@ static void mndCancelGetNextUser(SMnode *pMnode, void *pIter) { sdbCancelFetch(pSdb, pIter); } +static void mndLoopHash(SHashObj *hash, char *priType, SSDataBlock *pBlock, int32_t *numOfRows, char *user, + SShowObj *pShow) { + char *value = taosHashIterate(hash, NULL); + int32_t cols = 0; + + while (value != NULL) { + cols = 0; + char userName[TSDB_USER_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(userName, user, pShow->pMeta->pSchemas[cols].bytes); + SColumnInfoData *pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)userName, false); + + char privilege[20] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(privilege, priType, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)privilege, false); + + size_t keyLen = 0; + void *key = taosHashGetKey(value, &keyLen); + + char dbName[TSDB_DB_NAME_LEN] = {0}; + mndExtractShortDbNameFromStbFullName(key, dbName); + char dbNameContent[TSDB_DB_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(dbNameContent, dbName, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)dbNameContent, false); + + char tableName[TSDB_TABLE_NAME_LEN] = {0}; + mndExtractTbNameFromStbFullName(key, tableName, TSDB_TABLE_NAME_LEN); + char tableNameContent[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableNameContent, tableName, pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)tableNameContent, false); + + if (strcmp("t", value) != 0) { + SNode *pAst = NULL; + int32_t sqlLen = 0; + char sql[TSDB_EXPLAIN_RESULT_ROW_SIZE] = {0}; + + if (nodesStringToNode(value, &pAst) == 0) { + nodesNodeToSQL(pAst, sql, TSDB_EXPLAIN_RESULT_ROW_SIZE, &sqlLen); + nodesDestroyNode(pAst); + } else { + sqlLen = 5; + sprintf(sql, "error"); + } + + char obj[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(obj, sql, pShow->pMeta->pSchemas[cols].bytes); + + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)obj, false); + } else { + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, *numOfRows, (const char *)condition, false); + } + + (*numOfRows)++; + value = taosHashIterate(hash, value); + } +} + static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock *pBlock, int32_t rows) { SMnode *pMnode = pReq->info.node; SSdb *pSdb = pMnode->pSdb; @@ -845,7 +1209,9 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock int32_t numOfReadDbs = taosHashGetSize(pUser->readDbs); int32_t numOfWriteDbs = taosHashGetSize(pUser->writeDbs); int32_t numOfTopics = taosHashGetSize(pUser->topics); - if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics >= rows) break; + int32_t numOfReadTbs = taosHashGetSize(pUser->readTbs); + int32_t numOfWriteTbs = taosHashGetSize(pUser->writeTbs); + if (numOfRows + numOfReadDbs + numOfWriteDbs + numOfTopics + numOfReadTbs + numOfWriteTbs >= rows) break; if (pUser->superUser) { cols = 0; @@ -864,6 +1230,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; } @@ -888,6 +1264,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; db = taosHashIterate(pUser->readDbs, db); } @@ -913,10 +1299,24 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)objName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; db = taosHashIterate(pUser->writeDbs, db); } + mndLoopHash(pUser->readTbs, "read", pBlock, &numOfRows, pUser->user, pShow); + + mndLoopHash(pUser->writeTbs, "write", pBlock, &numOfRows, pUser->user, pShow); + char *topic = taosHashIterate(pUser->topics, NULL); while (topic != NULL) { cols = 0; @@ -936,6 +1336,16 @@ static int32_t mndRetrievePrivileges(SRpcMsg *pReq, SShowObj *pShow, SSDataBlock pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); colDataSetVal(pColInfo, numOfRows, (const char *)topicName, false); + char tableName[TSDB_TABLE_NAME_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(tableName, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)tableName, false); + + char condition[TSDB_PRIVILEDGE_CONDITION_LEN + VARSTR_HEADER_SIZE] = {0}; + STR_WITH_MAXSIZE_TO_VARSTR(condition, "", pShow->pMeta->pSchemas[cols].bytes); + pColInfo = taosArrayGet(pBlock->pDataBlock, cols++); + colDataSetVal(pColInfo, numOfRows, (const char *)condition, false); + numOfRows++; topic = taosHashIterate(pUser->topics, topic); } @@ -1019,6 +1429,69 @@ _OVER: return code; } +int32_t mndValidateUserPassInfo(SMnode *pMnode, SUserPassVersion *pUsers, int32_t numOfUses, void **ppRsp, + int32_t *pRspLen) { + int32_t code = 0; + SUserPassBatchRsp batchRsp = {0}; + + for (int32_t i = 0; i < numOfUses; ++i) { + SUserObj *pUser = mndAcquireUser(pMnode, pUsers[i].user); + if (pUser == NULL) { + mError("user:%s, failed to validate user pass since %s", pUsers[i].user, terrstr()); + continue; + } + + pUsers[i].version = ntohl(pUsers[i].version); + if (pUser->passVersion <= pUsers[i].version) { + mTrace("user:%s, not update since mnd passVer %d <= client passVer %d", pUsers[i].user, pUser->passVersion, + pUsers[i].version); + mndReleaseUser(pMnode, pUser); + continue; + } + + SGetUserPassRsp rsp = {0}; + memcpy(rsp.user, pUser->user, TSDB_USER_LEN); + rsp.version = pUser->passVersion; + + if (!batchRsp.pArray && !(batchRsp.pArray = taosArrayInit(numOfUses, sizeof(SGetUserPassRsp)))) { + code = TSDB_CODE_OUT_OF_MEMORY; + mndReleaseUser(pMnode, pUser); + goto _OVER; + } + + taosArrayPush(batchRsp.pArray, &rsp); + mndReleaseUser(pMnode, pUser); + } + + if (taosArrayGetSize(batchRsp.pArray) <= 0) { + goto _OVER; + } + + int32_t rspLen = tSerializeSUserPassBatchRsp(NULL, 0, &batchRsp); + if (rspLen < 0) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + void *pRsp = taosMemoryMalloc(rspLen); + if (pRsp == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; + goto _OVER; + } + tSerializeSUserPassBatchRsp(pRsp, rspLen, &batchRsp); + + *ppRsp = pRsp; + *pRspLen = rspLen; + +_OVER: + if (code) { + *ppRsp = NULL; + *pRspLen = 0; + } + + tFreeSUserPassBatchRsp(&batchRsp); + return code; +} + int32_t mndUserRemoveDb(SMnode *pMnode, STrans *pTrans, char *db) { int32_t code = 0; SSdb *pSdb = pMnode->pSdb; diff --git a/source/dnode/mnode/impl/src/mndVgroup.c b/source/dnode/mnode/impl/src/mndVgroup.c index 31924e04717e0bfffb485b7726a533454d718194..f0bece6e5eb8d77538e2d02eeb4787e2f3c55049 100644 --- a/source/dnode/mnode/impl/src/mndVgroup.c +++ b/source/dnode/mnode/impl/src/mndVgroup.c @@ -62,6 +62,7 @@ int32_t mndInitVgroup(SMnode *pMnode) { mndSetMsgHandle(pMnode, TDMT_VND_COMPACT_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_VND_DISABLE_WRITE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_SYNC_FORCE_FOLLOWER_RSP, mndTransProcessRsp); + mndSetMsgHandle(pMnode, TDMT_DND_ALTER_VNODE_TYPE_RSP, mndTransProcessRsp); mndSetMsgHandle(pMnode, TDMT_MND_REDISTRIBUTE_VGROUP, mndProcessRedistributeVgroupMsg); mndSetMsgHandle(pMnode, TDMT_MND_SPLIT_VGROUP, mndProcessSplitVgroupMsg); @@ -203,7 +204,7 @@ static int32_t mndVgroupActionUpdate(SSdb *pSdb, SVgObj *pOld, SVgObj *pNew) { pNew->compStorage = pOld->compStorage; pNew->pointsWritten = pOld->pointsWritten; pNew->compact = pOld->compact; - memcpy(pOld->vnodeGid, pNew->vnodeGid, TSDB_MAX_REPLICA * sizeof(SVnodeGid)); + memcpy(pOld->vnodeGid, pNew->vnodeGid, (TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA) * sizeof(SVnodeGid)); return 0; } @@ -244,8 +245,10 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.compression = pDb->cfg.compression; createReq.strict = pDb->cfg.strict; createReq.cacheLast = pDb->cfg.cacheLast; - createReq.replica = pVgroup->replica; + createReq.replica = 0; + createReq.learnerReplica = 0; createReq.selfIndex = -1; + createReq.learnerSelfIndex = -1; createReq.hashBegin = pVgroup->hashBegin; createReq.hashEnd = pVgroup->hashEnd; createReq.hashMethod = pDb->cfg.hashMethod; @@ -263,7 +266,15 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg createReq.tsdbPageSize = pDb->cfg.tsdbPageSize; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &createReq.replicas[v]; + SReplica *pReplica = NULL; + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + pReplica = &createReq.replicas[createReq.replica]; + } + else{ + pReplica = &createReq.learnerReplicas[createReq.learnerReplica]; + } + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) { @@ -275,21 +286,40 @@ void *mndBuildCreateVnodeReq(SMnode *pMnode, SDnodeObj *pDnode, SDbObj *pDb, SVg memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - if (pDnode->id == pVgid->dnodeId) { - createReq.selfIndex = v; + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + if (pDnode->id == pVgid->dnodeId) { + createReq.selfIndex = createReq.replica; + } + } + else{ + if (pDnode->id == pVgid->dnodeId) { + createReq.learnerSelfIndex = createReq.learnerReplica; + } + } + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + createReq.replica++; + } + else{ + createReq.learnerReplica++; } } - if (createReq.selfIndex == -1) { + if (createReq.selfIndex == -1 && createReq.learnerSelfIndex == -1) { terrno = TSDB_CODE_APP_ERROR; return NULL; } - mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d strict:%d", createReq.vgId, createReq.replica, - createReq.selfIndex, createReq.strict); + mInfo("vgId:%d, build create vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d", + createReq.vgId, createReq.replica, createReq.selfIndex, createReq.learnerReplica, + createReq.learnerReplica, createReq.strict); for (int32_t i = 0; i < createReq.replica; ++i) { mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.replicas[i].fqdn, createReq.replicas[i].port); } + for (int32_t i = 0; i < createReq.learnerReplica; ++i) { + mInfo("vgId:%d, replica:%d ep:%s:%u", createReq.vgId, i, createReq.learnerReplicas[i].fqdn, + createReq.learnerReplicas[i].port); + } int32_t contLen = tSerializeSCreateVnodeReq(NULL, 0, &createReq); if (contLen < 0) { @@ -356,12 +386,24 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p SAlterVnodeReplicaReq alterReq = { .vgId = pVgroup->vgId, .strict = pDb->cfg.strict, - .replica = pVgroup->replica, + .replica = 0, + .learnerReplica = 0, .selfIndex = -1, + .learnerSelfIndex = -1, }; for (int32_t v = 0; v < pVgroup->replica; ++v) { - SReplica *pReplica = &alterReq.replicas[v]; + SReplica *pReplica = NULL; + + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + pReplica = &alterReq.replicas[alterReq.replica]; + alterReq.replica++; + } + else{ + pReplica = &alterReq.learnerReplicas[alterReq.learnerReplica]; + alterReq.learnerReplica++; + } + SVnodeGid *pVgid = &pVgroup->vnodeGid[v]; SDnodeObj *pVgidDnode = mndAcquireDnode(pMnode, pVgid->dnodeId); if (pVgidDnode == NULL) return NULL; @@ -371,18 +413,30 @@ static void *mndBuildAlterVnodeReplicaReq(SMnode *pMnode, SDbObj *pDb, SVgObj *p memcpy(pReplica->fqdn, pVgidDnode->fqdn, TSDB_FQDN_LEN); mndReleaseDnode(pMnode, pVgidDnode); - if (dnodeId == pVgid->dnodeId) { - alterReq.selfIndex = v; + if(pVgroup->vnodeGid[v].nodeRole == TAOS_SYNC_ROLE_VOTER){ + if (dnodeId == pVgid->dnodeId) { + alterReq.selfIndex = v; + } + } + else{ + if (dnodeId == pVgid->dnodeId) { + alterReq.learnerSelfIndex = v; + } } } - alterReq.replica = pVgroup->replica; - mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d strict:%d", alterReq.vgId, alterReq.replica, - alterReq.selfIndex, alterReq.strict); + + mInfo("vgId:%d, build alter vnode req, replica:%d selfIndex:%d learnerReplica:%d learnerSelfIndex:%d strict:%d", + alterReq.vgId, alterReq.replica, alterReq.selfIndex, alterReq.learnerReplica, + alterReq.learnerSelfIndex, alterReq.strict); for (int32_t i = 0; i < alterReq.replica; ++i) { mInfo("vgId:%d, replica:%d ep:%s:%u", alterReq.vgId, i, alterReq.replicas[i].fqdn, alterReq.replicas[i].port); } + for (int32_t i = 0; i < alterReq.learnerReplica; ++i) { + mInfo("vgId:%d, learnerReplica:%d ep:%s:%u", alterReq.vgId, i, + alterReq.learnerReplicas[i].fqdn, alterReq.learnerReplicas[i].port); + } - if (alterReq.selfIndex == -1) { + if (alterReq.selfIndex == -1 && alterReq.learnerSelfIndex == -1) { terrno = TSDB_CODE_APP_ERROR; return NULL; } @@ -1194,6 +1248,32 @@ int32_t mndAddAlterVnodeReplicaAction(SMnode *pMnode, STrans *pTrans, SDbObj *pD return 0; } +int32_t mndAddAlterVnodeTypeAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { + SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); + if (pDnode == NULL) return -1; + + STransAction action = {0}; + action.epSet = mndGetDnodeEpset(pDnode); + mndReleaseDnode(pMnode, pDnode); + + int32_t contLen = 0; + void *pReq = mndBuildAlterVnodeReplicaReq(pMnode, pDb, pVgroup, dnodeId, &contLen); + if (pReq == NULL) return -1; + + action.pCont = pReq; + action.contLen = contLen; + action.msgType = TDMT_DND_ALTER_VNODE_TYPE; + action.acceptableCode = TSDB_CODE_VND_ALREADY_IS_VOTER; + action.retryCode = TSDB_CODE_VND_NOT_CATCH_UP; + + if (mndTransAppendRedoAction(pTrans, &action) != 0) { + taosMemoryFree(pReq); + return -1; + } + + return 0; +} + static int32_t mndAddDisableVnodeWriteAction(SMnode *pMnode, STrans *pTrans, SDbObj *pDb, SVgObj *pVgroup, int32_t dnodeId) { SDnodeObj *pDnode = mndAcquireDnode(pMnode, dnodeId); @@ -1891,56 +1971,17 @@ int32_t mndAddVgroupBalanceToTrans(SMnode *pMnode, SVgObj *pVgroup, STrans *pTra return 0; } -int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq) { - int32_t code = -1; - - SBalanceVgroupLeaderReq req = {0}; - if (tDeserializeSBalanceVgroupLeaderReq(pReq->pCont, pReq->contLen, &req) != 0) { - terrno = TSDB_CODE_INVALID_MSG; - return code; - } - - SMnode *pMnode = pReq->info.node; - SSdb *pSdb = pMnode->pSdb; - - int32_t total = sdbGetSize(pSdb, SDB_VGROUP); - if(total <= 0) { - terrno = TSDB_CODE_TSC_INVALID_OPERATION; - return code; - } - - STrans *pTrans = NULL; - pTrans = mndTransCreate(pMnode, TRN_POLICY_RETRY, TRN_CONFLICT_NOTHING, pReq, "bal-vg-leader"); - if (pTrans == NULL) goto _OVER; - mndTransSetSerial(pTrans); - mInfo("trans:%d, used to balance vgroup leader", pTrans->id); - - void *pIter = NULL; - int32_t count = 0; - while (1) { - SVgObj *pVgroup = NULL; - pIter = sdbFetch(pSdb, SDB_VGROUP, pIter, (void **)&pVgroup); - if (pIter == NULL) break; - - if(mndAddVgroupBalanceToTrans(pMnode, pVgroup, pTrans) == 0){ - count++; - } +extern int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq); - sdbRelease(pSdb, pVgroup); - } - - if(count == 0) { - terrno = TSDB_CODE_TSC_INVALID_OPERATION; - goto _OVER; - } - - if (mndTransPrepare(pMnode, pTrans) != 0) goto _OVER; - code = 0; +int32_t mndProcessVgroupBalanceLeaderMsg(SRpcMsg *pReq) { + return mndProcessVgroupBalanceLeaderMsgImp(pReq); +} -_OVER: - mndTransDrop(pTrans); - return code; +#ifndef TD_ENTERPRISE +int32_t mndProcessVgroupBalanceLeaderMsgImp(SRpcMsg *pReq) { + return 0; } +#endif static int32_t mndCheckDnodeMemory(SMnode *pMnode, SDbObj *pOldDb, SDbObj *pNewDb, SVgObj *pOldVgroup, SVgObj *pNewVgroup, SArray *pArray) { @@ -1992,18 +2033,33 @@ int32_t mndBuildAlterVgroupAction(SMnode *pMnode, STrans *pTrans, SDbObj *pOldDb mInfo("db:%s, vgId:%d, will add 2 vnodes, vn:0 dnode:%d", pVgroup->dbName, pVgroup->vgId, pVgroup->vnodeGid[0].dnodeId); + //add first if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_LEARNER; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; + if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[1]) != 0) return -1; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeTypeAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) + return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; + //add second if (mndAddVnodeToVgroup(pMnode, pTrans, &newVgroup, pArray) != 0) return -1; + newVgroup.vnodeGid[0].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[1].nodeRole = TAOS_SYNC_ROLE_VOTER; + newVgroup.vnodeGid[2].nodeRole = TAOS_SYNC_ROLE_VOTER; + if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[0].dnodeId) != 0) return -1; if (mndAddAlterVnodeReplicaAction(pMnode, pTrans, pNewDb, &newVgroup, newVgroup.vnodeGid[1].dnodeId) != 0) return -1; if (mndAddCreateVnodeAction(pMnode, pTrans, pNewDb, &newVgroup, &newVgroup.vnodeGid[2]) != 0) return -1; + if (mndAddAlterVnodeConfirmAction(pMnode, pTrans, pNewDb, &newVgroup) != 0) return -1; } else if (newVgroup.replica == 3 && pNewDb->cfg.replications == 1) { mInfo("db:%s, vgId:%d, will remove 2 vnodes, vn:0 dnode:%d vn:1 dnode:%d vn:2 dnode:%d", pVgroup->dbName, diff --git a/source/dnode/snode/src/snode.c b/source/dnode/snode/src/snode.c index 3d1b356f8c995847c1001cd7b288f7e7cac8fd89..cefc4fa63eb7ba79a46a8527ba0f7c2652b0e6ab 100644 --- a/source/dnode/snode/src/snode.c +++ b/source/dnode/snode/src/snode.c @@ -32,6 +32,7 @@ void sndEnqueueStreamDispatch(SSnode *pSnode, SRpcMsg *pMsg) { tDecoderClear(&decoder); goto FAIL; } + tDecoderClear(&decoder); int32_t taskId = req.taskId; @@ -65,7 +66,7 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { ASSERT(taosArrayGetSize(pTask->childEpInfo) != 0); pTask->refCnt = 1; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -76,21 +77,19 @@ int32_t sndExpandTask(SSnode *pSnode, SStreamTask *pTask, int64_t ver) { pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pSnode->msgCb; - pTask->startVer = ver; + pTask->chkInfo.version = ver; + pTask->pMeta = pSnode->pMeta; pTask->pState = streamStateOpen(pSnode->path, pTask, false, -1, -1); if (pTask->pState == NULL) { return -1; } - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - .pStateBackend = pTask->pState, - }; + int32_t numOfChildEp = taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfChildEp, .pStateBackend = pTask->pState }; - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); - ASSERT(pTask->exec.executor); + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, 0); + ASSERT(pTask->exec.pExecutor); streamSetupTrigger(pTask); return 0; @@ -140,9 +139,10 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { if (pTask == NULL) { return -1; } + SDecoder decoder; tDecoderInit(&decoder, (uint8_t *)msg, msgLen); - code = tDecodeSStreamTask(&decoder, pTask); + code = tDecodeStreamTask(&decoder, pTask); if (code < 0) { tDecoderClear(&decoder); taosMemoryFree(pTask); @@ -153,7 +153,7 @@ int32_t sndProcessTaskDeployReq(SSnode *pSnode, char *msg, int32_t msgLen) { ASSERT(pTask->taskLevel == TASK_LEVEL__AGG); // 2.save task - code = streamMetaAddTask(pSnode->pMeta, -1, pTask); + code = streamMetaAddDeployedTask(pSnode->pMeta, -1, pTask); if (code < 0) { return -1; } diff --git a/source/dnode/vnode/CMakeLists.txt b/source/dnode/vnode/CMakeLists.txt index 9911752f8e23827b7f3700118d499601569e5980..c713d1e247a8cd358fbe0c8a4a529f884e8810b6 100644 --- a/source/dnode/vnode/CMakeLists.txt +++ b/source/dnode/vnode/CMakeLists.txt @@ -57,6 +57,7 @@ target_sources( # tq "src/tq/tq.c" + "src/tq/tqUtil.c" "src/tq/tqScan.c" "src/tq/tqMeta.c" "src/tq/tqRead.c" @@ -64,6 +65,7 @@ target_sources( "src/tq/tqPush.c" "src/tq/tqSink.c" "src/tq/tqCommit.c" + "src/tq/tqRestore.c" "src/tq/tqSnapshot.c" "src/tq/tqOffsetSnapshot.c" ) diff --git a/source/dnode/vnode/inc/vnode.h b/source/dnode/vnode/inc/vnode.h index 9636dc2872c82c50a9e501e744a5b8712f5bb0fd..828a173108f8c3a97529cf31be09f668d47b98aa 100644 --- a/source/dnode/vnode/inc/vnode.h +++ b/source/dnode/vnode/inc/vnode.h @@ -68,6 +68,8 @@ void vnodeGetSnapshot(SVnode *pVnode, SSnapshot *pSnapshot); void vnodeGetInfo(SVnode *pVnode, const char **dbname, int32_t *vgId); int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen); int32_t vnodeGetAllTableList(SVnode *pVnode, uint64_t uid, SArray *list); +int32_t vnodeIsCatchUp(SVnode *pVnode); +ESyncRole vnodeGetRole(SVnode *pVnode); int32_t vnodeGetCtbIdList(SVnode *pVnode, int64_t suid, SArray *list); int32_t vnodeGetCtbIdListByFilter(SVnode *pVnode, int64_t suid, SArray *list, bool (*filter)(void *arg), void *arg); @@ -92,7 +94,7 @@ int32_t vnodeProcessQueryMsg(SVnode *pVnode, SRpcMsg *pMsg); int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo); void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); void vnodeApplyWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs); -void vnodeProposeCommitOnNeed(SVnode *pVnode); +void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit); // meta typedef struct SMeta SMeta; // todo: remove @@ -228,19 +230,12 @@ typedef struct SSnapContext { SArray *idList; int32_t index; bool withMeta; - bool queryMetaOrData; // true-get meta, false-get data + bool queryMeta; // true-get meta, false-get data } SSnapContext; typedef struct STqReader { - // const SSubmitReq *pMsg; - // SSubmitBlk *pBlock; - // SSubmitMsgIter msgIter; - // SSubmitBlkIter blkIter; - - int64_t ver; SPackedData msg2; - int8_t setMsg; SSubmitReq2 submit; int32_t nextBlk; @@ -263,15 +258,16 @@ void tqCloseReader(STqReader *); void tqReaderSetColIdList(STqReader *pReader, SArray *pColIdList); int32_t tqReaderSetTbUidList(STqReader *pReader, const SArray *tbUidList); -int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *tbUidList); +int32_t tqReaderAddTbUidList(STqReader *pReader, const SArray *pTableUidList); int32_t tqReaderRemoveTbUidList(STqReader *pReader, const SArray *tbUidList); int32_t tqSeekVer(STqReader *pReader, int64_t ver, const char *id); -int32_t tqNextBlock(STqReader *pReader, SFetchRet *ret); +void tqNextBlock(STqReader *pReader, SFetchRet *ret); +int32_t extractSubmitMsgFromWal(SWalReader *pReader, SPackedData *pPackedData); -int32_t tqReaderSetSubmitReq2(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); +int32_t tqReaderSetSubmitMsg(STqReader *pReader, void *msgStr, int32_t msgLen, int64_t ver); // int32_t tqReaderSetDataMsg(STqReader *pReader, const SSubmitReq *pMsg, int64_t ver); -bool tqNextDataBlock2(STqReader *pReader); +bool tqNextDataBlock(STqReader *pReader); bool tqNextDataBlockFilterOut2(STqReader *pReader, SHashObj *filterOutUids); int32_t tqRetrieveDataBlock2(SSDataBlock *pBlock, STqReader *pReader, SSubmitTbData **pSubmitTbDataRet); int32_t tqRetrieveTaosxBlock2(STqReader *pReader, SArray *blocks, SArray *schemas, SSubmitTbData **pSubmitTbDataRet); diff --git a/source/dnode/vnode/src/inc/tq.h b/source/dnode/vnode/src/inc/tq.h index 9037644602034819fd1da260c1cef02b7ae29a97..142a1f6fb3d9fe064d8b8544f2bfd6c622b770d1 100644 --- a/source/dnode/vnode/src/inc/tq.h +++ b/source/dnode/vnode/src/inc/tq.h @@ -80,7 +80,7 @@ typedef struct { typedef struct { int8_t subType; - STqReader* pExecReader; + STqReader* pTqReader; qTaskInfo_t task; union { STqExecCol execCol; @@ -106,6 +106,7 @@ typedef struct { SMqDataRsp* pDataRsp; char subKey[TSDB_SUBSCRIBE_KEY_LEN]; SRpcHandleInfo info; + STqHandle* pHandle; } STqPushEntry; struct STQ { @@ -128,6 +129,10 @@ typedef struct { tmr_h timer; } STqMgmt; +typedef struct { + int32_t size; +} STqOffsetHead; + static STqMgmt tqMgmt = {0}; int32_t tEncodeSTqHandle(SEncoder* pEncoder, const STqHandle* pHandle); @@ -141,8 +146,9 @@ int32_t tqFetchLog(STQ* pTq, STqHandle* pHandle, int64_t* fetchOffset, SWalCkHea // tqExec int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxRsp* pRsp, int32_t* totalRows); int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t numOfCols, int8_t precision); -int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type); -int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, + int32_t type, int32_t vgId); +int32_t tqPushDataRsp(STqPushEntry* pPushEntry, int32_t vgId); // tqMeta int32_t tqMetaOpen(STQ* pTq); @@ -154,10 +160,6 @@ int32_t tqMetaSaveCheckInfo(STQ* pTq, const char* key, const void* value, int32_ int32_t tqMetaDeleteCheckInfo(STQ* pTq, const char* key); int32_t tqMetaRestoreCheckInfo(STQ* pTq); -typedef struct { - int32_t size; -} STqOffsetHead; - STqOffsetStore* tqOffsetOpen(STQ* pTq); void tqOffsetClose(STqOffsetStore*); STqOffset* tqOffsetRead(STqOffsetStore* pStore, const char* subscribeKey); @@ -176,6 +178,20 @@ int32_t tqOffsetRestoreFromFile(STqOffsetStore* pStore, const char* fname); // tqStream int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver); +int32_t tqStreamTasksScanWal(STQ* pTq); + +// tq util +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId); +void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId); +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver); +int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg); +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, + int32_t type, int64_t sver, int64_t ever); + +int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq); +void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver); +void saveOffsetForAllTasks(STQ* pTq, int64_t ver); +void initOffsetForAllRestoreTasks(STQ* pTq); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/tsdb.h b/source/dnode/vnode/src/inc/tsdb.h index 0c4ada2cb1306c4c46d37d652e995dd8bf3f3904..2a85b191a42839a580da6ff8b65ee4e9c65430b5 100644 --- a/source/dnode/vnode/src/inc/tsdb.h +++ b/source/dnode/vnode/src/inc/tsdb.h @@ -123,12 +123,12 @@ int32_t tsdbRowIterOpen(STSDBRowIter *pIter, TSDBROW *pRow, STSchema *pTSchema) void tsdbRowClose(STSDBRowIter *pIter); SColVal *tsdbRowIterNext(STSDBRowIter *pIter); // SRowMerger -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema); int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); -void tsdbRowMergerClear(SRowMerger *pMerger); -int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); +// int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema); +void tsdbRowMergerClear(SRowMerger *pMerger); +// int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow); int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow); // TABLEID int32_t tTABLEIDCmprFn(const void *p1, const void *p2); @@ -224,7 +224,7 @@ int32_t tsdbTbDataIterCreate(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, void *tsdbTbDataIterDestroy(STbDataIter *pIter); void tsdbTbDataIterOpen(STbData *pTbData, TSDBKEY *pFrom, int8_t backward, STbDataIter *pIter); bool tsdbTbDataIterNext(STbDataIter *pIter); -void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj* pTableMap, int64_t *rowsNum); +void tsdbMemTableCountRows(SMemTable *pMemTable, SHashObj *pTableMap, int64_t *rowsNum); // STbData int32_t tsdbGetNRowsInTbData(STbData *pTbData); diff --git a/source/dnode/vnode/src/inc/vnd.h b/source/dnode/vnode/src/inc/vnd.h index 134909090f1c14f2972b98bf00d9d7c0486b5588..a67f246e73d9656f9c184e99b796edf8ee8dc954 100644 --- a/source/dnode/vnode/src/inc/vnd.h +++ b/source/dnode/vnode/src/inc/vnd.h @@ -96,8 +96,7 @@ int32_t vnodeGetBatchMeta(SVnode* pVnode, SRpcMsg* pMsg); // vnodeCommit.c int32_t vnodeBegin(SVnode* pVnode); -int32_t vnodeShouldCommit(SVnode* pVnode); -void vnodeUpdCommitSched(SVnode* pVnode); +int32_t vnodeShouldCommit(SVnode* pVnode, bool atExit); void vnodeRollback(SVnode* pVnode); int32_t vnodeSaveInfo(const char* dir, const SVnodeInfo* pCfg); int32_t vnodeCommitInfo(const char* dir); @@ -115,7 +114,6 @@ void vnodeSyncClose(SVnode* pVnode); void vnodeRedirectRpcMsg(SVnode* pVnode, SRpcMsg* pMsg, int32_t code); bool vnodeIsLeader(SVnode* pVnode); bool vnodeIsRoleLeader(SVnode* pVnode); -int vnodeShouldCommit(SVnode* pVnode); #ifdef __cplusplus } diff --git a/source/dnode/vnode/src/inc/vnodeInt.h b/source/dnode/vnode/src/inc/vnodeInt.h index 253d5aebcec2353c1e951c5399256a23681f6850..70b997aba2cadb4e9088c11cb5115a4079382ea4 100644 --- a/source/dnode/vnode/src/inc/vnodeInt.h +++ b/source/dnode/vnode/src/inc/vnodeInt.h @@ -192,9 +192,10 @@ void tqCleanUp(); STQ* tqOpen(const char* path, SVnode* pVnode); void tqClose(STQ*); int tqPushMsg(STQ*, void* msg, int32_t msgLen, tmsg_t msgType, int64_t ver); -int tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, +int tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type); -int tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer); +int tqStartStreamTasks(STQ* pTq); // restore all stream tasks after vnode launching completed. int tqCommit(STQ*); int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd); @@ -205,7 +206,10 @@ int32_t tqProcessDelCheckInfoReq(STQ* pTq, int64_t version, char* msg, int32_t m int32_t tqProcessSubscribeReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); +int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen); int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg); +int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg); + // tq-stream int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); int32_t tqProcessTaskDropReq(STQ* pTq, int64_t version, char* msg, int32_t msgLen); @@ -377,7 +381,6 @@ struct SVnode { STQ* pTq; SSink* pSink; tsem_t canCommit; - SVCommitSched commitSched; int64_t sync; TdThreadMutex lock; bool blocked; @@ -386,9 +389,6 @@ struct SVnode { int32_t blockSec; int64_t blockSeq; SQHandle* pQuery; -#if 0 - SRpcHandleInfo blockInfo; -#endif }; #define TD_VID(PVNODE) ((PVNODE)->config.vgId) diff --git a/source/dnode/vnode/src/meta/metaCache.c b/source/dnode/vnode/src/meta/metaCache.c index 9501bf4b8ef6055f2e4f28242f8f259896623725..795f281ab2107acdce93ec2373ab99e1391b61ba 100644 --- a/source/dnode/vnode/src/meta/metaCache.c +++ b/source/dnode/vnode/src/meta/metaCache.c @@ -531,10 +531,11 @@ static void freePayload(const void* key, size_t keyLen, void* value) { return; } - SHashObj* pHashObj = (SHashObj*)p[0]; + SHashObj* pHashObj = (SHashObj*)p[0]; + STagFilterResEntry** pEntry = taosHashGet(pHashObj, &p[1], sizeof(uint64_t)); - { + if (pEntry != NULL && (*pEntry) != NULL) { int64_t st = taosGetTimestampUs(); SListIter iter = {0}; @@ -547,9 +548,9 @@ static void freePayload(const void* key, size_t keyLen, void* value) { void* tmp = tdListPopNode(&((*pEntry)->list), pNode); taosMemoryFree(tmp); - int64_t et = taosGetTimestampUs(); - metaInfo("clear items in cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), - (et - st) / 1000.0); + double el = (taosGetTimestampUs() - st) / 1000.0; + metaInfo("clear items in meta-cache, remain cached item:%d, elapsed time:%.2fms", listNEles(&((*pEntry)->list)), + el); break; } } diff --git a/source/dnode/vnode/src/meta/metaSnapshot.c b/source/dnode/vnode/src/meta/metaSnapshot.c index 67ade45732865893d730f4920179bdc96a9b4925..0126d29cc93f3b2f69f3f12539c8cda62f1fa19a 100644 --- a/source/dnode/vnode/src/meta/metaSnapshot.c +++ b/source/dnode/vnode/src/meta/metaSnapshot.c @@ -268,7 +268,7 @@ int32_t buildSnapContext(SMeta* pMeta, int64_t snapVersion, int64_t suid, int8_t ctx->snapVersion = snapVersion; ctx->suid = suid; ctx->subType = subType; - ctx->queryMetaOrData = withMeta; + ctx->queryMeta = withMeta; ctx->withMeta = withMeta; ctx->idVersion = taosHashInit(100, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_NO_LOCK); if (ctx->idVersion == NULL) { @@ -475,7 +475,7 @@ int32_t getMetafromSnapShot(SSnapContext* ctx, void** pBuf, int32_t* contLen, in if (ctx->index >= taosArrayGetSize(ctx->idList)) { metaDebug("tmqsnap get meta end"); ctx->index = 0; - ctx->queryMetaOrData = false; // change to get data + ctx->queryMeta = false; // change to get data return 0; } diff --git a/source/dnode/vnode/src/sma/smaRollup.c b/source/dnode/vnode/src/sma/smaRollup.c index c75c675ec39946e4f23d005a8050d35b52eca762..ce987ca88ec930db76fcb560ff79f758e2bed1a2 100644 --- a/source/dnode/vnode/src/sma/smaRollup.c +++ b/source/dnode/vnode/src/sma/smaRollup.c @@ -168,7 +168,7 @@ static int32_t tdUpdateTbUidListImpl(SSma *pSma, tb_uid_t *suid, SArray *tbUids, for (int32_t i = 0; i < TSDB_RETENTION_L2; ++i) { if (pRSmaInfo->taskInfo[i]) { - if ((terrno = qUpdateQualifiedTableId(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { + if ((terrno = qUpdateTableListForStreamScanner(pRSmaInfo->taskInfo[i], tbUids, isAdd)) < 0) { tdReleaseRSmaInfo(pSma, pRSmaInfo); smaError("vgId:%d, update tbUidList failed for uid:%" PRIi64 " level %d since %s", SMA_VID(pSma), *suid, i, terrstr()); diff --git a/source/dnode/vnode/src/tq/tq.c b/source/dnode/vnode/src/tq/tq.c index 75c2bfcce842366543ec3558516e0533bd93414f..4a9bf07429a641ceedf1c131e261f72f8cee7716 100644 --- a/source/dnode/vnode/src/tq/tq.c +++ b/source/dnode/vnode/src/tq/tq.c @@ -18,6 +18,9 @@ // 0: not init // 1: already inited // 2: wait to be inited or cleaup +#define WAL_READ_TASKS_ID (-1) + +static int32_t tqInitialize(STQ* pTq); int32_t tqInit() { int8_t old; @@ -61,19 +64,19 @@ static void destroyTqHandle(void* data) { if (pData->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { taosMemoryFreeClear(pData->execHandle.execCol.qmsg); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__DB) { - tqCloseReader(pData->execHandle.pExecReader); + tqCloseReader(pData->execHandle.pTqReader); walCloseReader(pData->pWalReader); taosHashCleanup(pData->execHandle.execDb.pFilterOutTbUid); } else if (pData->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { walCloseReader(pData->pWalReader); - tqCloseReader(pData->execHandle.pExecReader); + tqCloseReader(pData->execHandle.pTqReader); } } static void tqPushEntryFree(void* data) { STqPushEntry* p = *(void**)data; if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__POLL_RSP) { - tDeleteSMqDataRsp(p->pDataRsp); + tDeleteMqDataRsp(p->pDataRsp); } else if (p->pDataRsp->head.mqMsgType == TMQ_MSG_TYPE__TAOSX_RSP) { tDeleteSTaosxRsp((STaosxRsp*)p->pDataRsp); } @@ -82,12 +85,18 @@ static void tqPushEntryFree(void* data) { taosMemoryFree(p); } +static bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { + return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && + pLeft->val.version <= pRight->val.version; +} + STQ* tqOpen(const char* path, SVnode* pVnode) { STQ* pTq = taosMemoryCalloc(1, sizeof(STQ)); if (pTq == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + pTq->path = taosStrdup(path); pTq->pVnode = pVnode; pTq->walLogLastVer = pVnode->pWal->vers.lastVer; @@ -102,25 +111,32 @@ STQ* tqOpen(const char* path, SVnode* pVnode) { pTq->pCheckInfo = taosHashInit(64, MurmurHash3_32, true, HASH_ENTRY_LOCK); taosHashSetFreeFp(pTq->pCheckInfo, (FDelete)tDeleteSTqCheckInfo); + tqInitialize(pTq); + return pTq; +} + +int32_t tqInitialize(STQ* pTq) { if (tqMetaOpen(pTq) < 0) { - return NULL; + return -1; } pTq->pOffsetStore = tqOffsetOpen(pTq); if (pTq->pOffsetStore == NULL) { - return NULL; + return -1; } - pTq->pStreamMeta = streamMetaOpen(path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); + pTq->pStreamMeta = streamMetaOpen(pTq->path, pTq, (FTaskExpand*)tqExpandTask, pTq->pVnode->config.vgId); if (pTq->pStreamMeta == NULL) { - return NULL; + return -1; } - if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pVnode->pWal)) < 0) { - return NULL; + // the version is kept in task's meta data + // todo check if this version is required or not + if (streamLoadTasks(pTq->pStreamMeta, walGetCommittedVer(pTq->pVnode->pWal)) < 0) { + return -1; } - return pTq; + return 0; } void tqClose(STQ* pTq) { @@ -138,120 +154,30 @@ void tqClose(STQ* pTq) { taosMemoryFree(pTq); } -int32_t tqSendMetaPollRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp) { - int32_t len = 0; - int32_t code = 0; - tEncodeSize(tEncodeSMqMetaRsp, pRsp, len, code); - if (code < 0) { - return -1; - } - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - - ((SMqRspHead*)buf)->mqMsgType = TMQ_MSG_TYPE__POLL_META_RSP; - ((SMqRspHead*)buf)->epoch = pReq->epoch; - ((SMqRspHead*)buf)->consumerId = pReq->consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - tEncodeSMqMetaRsp(&encoder, pRsp); - tEncoderClear(&encoder); - - SRpcMsg resp = { - .info = pMsg->info, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - tmsgSendRsp(&resp); - - tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type); - - return 0; -} - -static int32_t doSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, - int64_t consumerId, int32_t type) { - int32_t len = 0; - int32_t code = 0; - - if (type == TMQ_MSG_TYPE__POLL_RSP) { - tEncodeSize(tEncodeSMqDataRsp, pRsp, len, code); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); - } - - if (code < 0) { - return -1; - } - - int32_t tlen = sizeof(SMqRspHead) + len; - void* buf = rpcMallocCont(tlen); - if (buf == NULL) { - return -1; - } - - ((SMqRspHead*)buf)->mqMsgType = type; - ((SMqRspHead*)buf)->epoch = epoch; - ((SMqRspHead*)buf)->consumerId = consumerId; - - void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); - - SEncoder encoder = {0}; - tEncoderInit(&encoder, abuf, len); - - if (type == TMQ_MSG_TYPE__POLL_RSP) { - tEncodeSMqDataRsp(&encoder, pRsp); - } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { - tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); - } - - tEncoderClear(&encoder); - - SRpcMsg rsp = { - .info = *pRpcHandleInfo, - .pCont = buf, - .contLen = tlen, - .code = 0, - }; - - tmsgSendRsp(&rsp); - return 0; -} - -int32_t tqPushDataRsp(STQ* pTq, STqPushEntry* pPushEntry) { +int32_t tqPushDataRsp(STqPushEntry* pPushEntry, int32_t vgId) { SMqDataRsp* pRsp = pPushEntry->pDataRsp; SMqRspHead* pHeader = &pPushEntry->pDataRsp->head; - doSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType); + + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pPushEntry->pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + + tqDoSendDataRsp(&pPushEntry->info, pRsp, pHeader->epoch, pHeader->consumerId, pHeader->mqMsgType, sver, ever); char buf1[80] = {0}; char buf2[80] = {0}; tFormatOffset(buf1, tListLen(buf1), &pRsp->reqOffset); tFormatOffset(buf2, tListLen(buf2), &pRsp->rspOffset); tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) push rsp, block num: %d, req:%s, rsp:%s", - TD_VID(pTq->pVnode), pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); + vgId, pRsp->head.consumerId, pRsp->head.epoch, pRsp->blockNum, buf1, buf2); return 0; } -int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, int32_t type) { -#if 0 - A(taosArrayGetSize(pRsp->blockData) == pRsp->blockNum); - A(taosArrayGetSize(pRsp->blockDataLen) == pRsp->blockNum); +int32_t tqSendDataRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqDataRsp* pRsp, + int32_t type, int32_t vgId) { + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); - A(!pRsp->withSchema); - A(taosArrayGetSize(pRsp->blockSchema) == 0); - - if (pRsp->reqOffset.type == TMQ_OFFSET__LOG) { - A(pRsp->rspOffset.version > pRsp->reqOffset.version); - } -#endif - doSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type); + tqDoSendDataRsp(&pMsg->info, pRsp, pReq->epoch, pReq->consumerId, type, sver, ever); char buf1[80] = {0}; char buf2[80] = {0}; @@ -259,55 +185,54 @@ int32_t tqSendDataRsp(STQ* pTq, const SRpcMsg* pMsg, const SMqPollReq* pReq, con tFormatOffset(buf2, 80, &pRsp->rspOffset); tqDebug("vgId:%d consumer:0x%" PRIx64 " (epoch %d) send rsp, block num:%d, req:%s, rsp:%s, reqId:0x%" PRIx64, - TD_VID(pTq->pVnode), pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); + vgId, pReq->consumerId, pReq->epoch, pRsp->blockNum, buf1, buf2, pReq->reqId); return 0; } -static FORCE_INLINE bool tqOffsetLessOrEqual(const STqOffset* pLeft, const STqOffset* pRight) { - return pLeft->val.type == TMQ_OFFSET__LOG && pRight->val.type == TMQ_OFFSET__LOG && - pLeft->val.version <= pRight->val.version; -} - int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { - STqOffset offset = {0}; - int32_t vgId = TD_VID(pTq->pVnode); + SMqVgOffset vgOffset = {0}; + int32_t vgId = TD_VID(pTq->pVnode); SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeSTqOffset(&decoder, &offset) < 0) { + if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { return -1; } tDecoderClear(&decoder); - if (offset.val.type == TMQ_OFFSET__SNAPSHOT_DATA || offset.val.type == TMQ_OFFSET__SNAPSHOT_META) { + STqOffset* pOffset = &vgOffset.offset; + + if (pOffset->val.type == TMQ_OFFSET__SNAPSHOT_DATA || pOffset->val.type == TMQ_OFFSET__SNAPSHOT_META) { tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:snapshot) uid:%" PRId64 ", ts:%" PRId64, - offset.subKey, vgId, offset.val.uid, offset.val.ts); - } else if (offset.val.type == TMQ_OFFSET__LOG) { - tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, offset.subKey, vgId, - offset.val.version); - if (offset.val.version + 1 == sversion) { - offset.val.version += 1; + pOffset->subKey, vgId, pOffset->val.uid, pOffset->val.ts); + } else if (pOffset->val.type == TMQ_OFFSET__LOG) { + tqDebug("receive offset commit msg to %s on vgId:%d, offset(type:log) version:%" PRId64, pOffset->subKey, vgId, + pOffset->val.version); + if (pOffset->val.version + 1 == sversion) { + pOffset->val.version += 1; } } else { - tqError("invalid commit offset type:%d", offset.val.type); + tqError("invalid commit offset type:%d", pOffset->val.type); return -1; } - STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, offset.subKey); - if (pSavedOffset != NULL && tqOffsetLessOrEqual(&offset, pSavedOffset)) { + STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + if (pSavedOffset != NULL && tqOffsetLessOrEqual(pOffset, pSavedOffset)) { + tqDebug("not update the offset, vgId:%d sub:%s since committed:%" PRId64 " less than/equal to existed:%" PRId64, + vgId, pOffset->subKey, pOffset->val.version, pSavedOffset->val.version); return 0; // no need to update the offset value } // save the new offset value - if (tqOffsetWrite(pTq->pOffsetStore, &offset) < 0) { + if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { return -1; } - if (offset.val.type == TMQ_OFFSET__LOG) { - STqHandle* pHandle = taosHashGet(pTq->pHandle, offset.subKey, strlen(offset.subKey)); - if (pHandle && (walRefVer(pHandle->pRef, offset.val.version) < 0)) { + if (pOffset->val.type == TMQ_OFFSET__LOG) { + STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); + if (pHandle && (walRefVer(pHandle->pRef, pOffset->val.version) < 0)) { return -1; } } @@ -315,347 +240,101 @@ int32_t tqProcessOffsetCommitReq(STQ* pTq, int64_t sversion, char* msg, int32_t return 0; } -int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { - void* pIter = NULL; +int32_t tqProcessSeekReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { + SMqVgOffset vgOffset = {0}; + int32_t vgId = TD_VID(pTq->pVnode); - while (1) { - pIter = taosHashIterate(pTq->pCheckInfo, pIter); - if (pIter == NULL) { - break; - } - - STqCheckInfo* pCheck = (STqCheckInfo*)pIter; - - if (pCheck->ntbUid == tbUid) { - int32_t sz = taosArrayGetSize(pCheck->colIdList); - for (int32_t i = 0; i < sz; i++) { - int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); - if (forbidColId == colId) { - taosHashCancelIterate(pTq->pCheckInfo, pIter); - return -1; - } - } - } - } - - return 0; -} - -static int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq, int8_t subType) { - pRsp->reqOffset = pReq->reqOffset; - - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); - - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { + SDecoder decoder; + tDecoderInit(&decoder, (uint8_t*)msg, msgLen); + if (tDecodeMqVgOffset(&decoder, &vgOffset) < 0) { return -1; } - pRsp->withTbName = 0; -#if 0 - pRsp->withTbName = pReq->withTbName; - if (pRsp->withTbName) { - pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); - if (pRsp->blockTbName == NULL) { - // TODO free - return -1; - } - } -#endif - - pRsp->withSchema = false; - return 0; -} - -static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { - pRsp->reqOffset = pReq->reqOffset; - - pRsp->withTbName = 1; - pRsp->withSchema = 1; - pRsp->blockData = taosArrayInit(0, sizeof(void*)); - pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); - pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); - pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + tDecoderClear(&decoder); - if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { + STqOffset* pOffset = &vgOffset.offset; + if (pOffset->val.type != TMQ_OFFSET__LOG) { + tqError("vgId:%d, subKey:%s invalid seek offset type:%d", vgId, pOffset->subKey, pOffset->val.type); return -1; } - return 0; -} - -static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, bool* pBlockReturned) { - uint64_t consumerId = pRequest->consumerId; - STqOffsetVal reqOffset = pRequest->reqOffset; - STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); - int32_t vgId = TD_VID(pTq->pVnode); - - *pBlockReturned = false; - - // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. - if (pOffset != NULL) { - *pOffsetVal = pOffset->val; - - char formatBuf[80]; - tFormatOffset(formatBuf, 80, pOffsetVal); - tqDebug("tmq poll: consumer:0x%" PRIx64 - ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%" PRIx64, - consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); - return 0; - } else { - // no poll occurs in this vnode for this topic, let's seek to the right offset value. - if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { - if (pRequest->useSnapshot) { - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", - consumerId, pHandle->subKey, vgId); - - if (pHandle->fetchMeta) { - tqOffsetResetToMeta(pOffsetVal, 0); - } else { - tqOffsetResetToData(pOffsetVal, 0, 0); - } - } else { - pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); - if (pHandle->pRef == NULL) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; - } - - tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, - pHandle->subKey, vgId, dataRsp.rspOffset.version); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - tDeleteSMqDataRsp(&dataRsp); - - *pBlockReturned = true; - return code; - } else { - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); - int32_t code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - - *pBlockReturned = true; - return code; - } - } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { - tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 - " in vg %d, subkey %s, reset none failed", - pHandle->subKey, consumerId, vgId, pRequest->subKey); - terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; - return -1; - } - } - - return 0; -} - -#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) - -static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, - SRpcMsg* pMsg, STqOffsetVal* pOffset) { - int32_t code = 0; - uint64_t consumerId = pRequest->consumerId; - int32_t vgId = TD_VID(pTq->pVnode); - - SMqDataRsp dataRsp = {0}; - tqInitDataRsp(&dataRsp, pRequest, pHandle->execHandle.subType); - - // lock - taosWLockLatch(&pTq->lock); - - qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); - code = tqScanData(pTq, pHandle, &dataRsp, pOffset); - if (code != TSDB_CODE_SUCCESS) { - taosWUnLockLatch(&pTq->lock); - return code; + STqHandle* pHandle = taosHashGet(pTq->pHandle, pOffset->subKey, strlen(pOffset->subKey)); + if (pHandle == NULL) { + tqError("tmq seek: consumer:0x%" PRIx64 " vgId:%d subkey %s not found", vgOffset.consumerId, vgId, + pOffset->subKey); + terrno = TSDB_CODE_INVALID_MSG; + return -1; } - // till now, all data has been transferred to consumer, new data needs to push client once arrived. - if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && - dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { - code = tqRegisterPushEntry(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); - taosWUnLockLatch(&pTq->lock); - return code; + // 2. check consumer-vg assignment status + taosRLockLatch(&pTq->lock); + if (pHandle->consumerId != vgOffset.consumerId) { + tqDebug("ERROR tmq seek: consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + vgOffset.consumerId, vgId, pOffset->subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosRUnLockLatch(&pTq->lock); + return -1; } + taosRUnLockLatch(&pTq->lock); - taosWUnLockLatch(&pTq->lock); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP); - - // NOTE: this pHandle->consumerId may have been changed already. - tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, offset type:%d, uid/version:%" PRId64 - ", ts:%" PRId64 ", reqId:0x%" PRIx64, - consumerId, pHandle->subKey, vgId, dataRsp.blockNum, dataRsp.rspOffset.type, dataRsp.rspOffset.uid, - dataRsp.rspOffset.ts, pRequest->reqId); - - tDeleteSMqDataRsp(&dataRsp); - return code; -} - -static int32_t doPollDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { - int32_t code = -1; - STqOffsetVal offset = {0}; - SWalCkHead* pCkHead = NULL; - int32_t vgId = TD_VID(pTq->pVnode); - - STqOffsetVal reqOffset = pRequest->reqOffset; - uint64_t consumerId = pRequest->consumerId; - - // 1. reset the offset if needed - if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { - // handle the reset offset cases, according to the consumer's choice. - bool blockReturned = false; - code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); - if (code != 0) { - return code; + //3. check the offset info + STqOffset* pSavedOffset = tqOffsetRead(pTq->pOffsetStore, pOffset->subKey); + if (pSavedOffset != NULL) { + if (pSavedOffset->val.type != TMQ_OFFSET__LOG) { + tqError("invalid saved offset type, vgId:%d sub:%s", vgId, pOffset->subKey); + return 0; // no need to update the offset value } - // empty block returned, quit - if (blockReturned) { + if (pSavedOffset->val.version == pOffset->val.version) { + tqDebug("vgId:%d subKey:%s no need to seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, + pOffset->val.version, pSavedOffset->val.version); return 0; } - } else { // use the consumer specified offset - // the offset value can not be monotonious increase?? - offset = reqOffset; } - // this is a normal subscribe requirement - if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset); + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + if (pOffset->val.version < sver) { + pOffset->val.version = sver; + } else if (pOffset->val.version > ever) { + pOffset->val.version = ever; } - // todo handle the case where re-balance occurs. - // for taosx - SMqMetaRsp metaRsp = {0}; - STaosxRsp taosxRsp = {0}; - tqInitTaosxRsp(&taosxRsp, pRequest); - - if (offset.type != TMQ_OFFSET__LOG) { - if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, &offset) < 0) { - return -1; - } + // save the new offset value + tqDebug("vgId:%d sub:%s seek to %" PRId64 " prev offset:%" PRId64, vgId, pOffset->subKey, pOffset->val.version, + pSavedOffset->val.version); - if (metaRsp.metaRspLen > 0) { - code = tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp); - tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 - ",ts:%" PRId64, - consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts); - taosMemoryFree(metaRsp.metaRsp); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } + if (tqOffsetWrite(pTq->pOffsetStore, pOffset) < 0) { + tqError("failed to save offset, vgId:%d sub:%s seek to %" PRId64, vgId, pOffset->subKey, pOffset->val.version); + return -1; + } - if (taosxRsp.blockNum > 0) { - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } else { - offset = taosxRsp.rspOffset; - } + return 0; +} - tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 - ",version:%" PRId64, - consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid, - taosxRsp.rspOffset.version); - } +int32_t tqCheckColModifiable(STQ* pTq, int64_t tbUid, int32_t colId) { + void* pIter = NULL; - if (offset.type == TMQ_OFFSET__LOG) { - int64_t fetchVer = offset.version + 1; - pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); - if (pCkHead == NULL) { - tDeleteSTaosxRsp(&taosxRsp); - terrno = TSDB_CODE_OUT_OF_MEMORY; - return -1; + while (1) { + pIter = taosHashIterate(pTq->pCheckInfo, pIter); + if (pIter == NULL) { + break; } - walSetReaderCapacity(pHandle->pWalReader, 2048); - int totalRows = 0; - while (1) { - // todo refactor: this is not correct. - int32_t savedEpoch = atomic_load_32(&pHandle->epoch); - if (savedEpoch > pRequest->epoch) { - tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 - ", found new consumer epoch %d, discard req epoch %d", - consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch); - break; - } - - if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } - SWalCont* pHead = &pCkHead->head; - tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", consumerId, - pRequest->epoch, vgId, fetchVer, pHead->msgType); - - // process meta - if (pHead->msgType != TDMT_VND_SUBMIT) { - if (totalRows > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } + STqCheckInfo* pCheck = (STqCheckInfo*)pIter; - tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); - tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); - metaRsp.resMsgType = pHead->msgType; - metaRsp.metaRspLen = pHead->bodyLen; - metaRsp.metaRsp = pHead->body; - if (tqSendMetaPollRsp(pTq, pMsg, pRequest, &metaRsp) < 0) { - code = -1; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; + if (pCheck->ntbUid == tbUid) { + int32_t sz = taosArrayGetSize(pCheck->colIdList); + for (int32_t i = 0; i < sz; i++) { + int16_t forbidColId = *(int16_t*)taosArrayGet(pCheck->colIdList, i); + if (forbidColId == colId) { + taosHashCancelIterate(pTq->pCheckInfo, pIter); + return -1; } - code = 0; - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return code; - } - - // process data - SPackedData submit = { - .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), - .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), - .ver = pHead->version, - }; - - if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { - tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", consumerId, vgId, - pRequest->subKey); - taosMemoryFreeClear(pCkHead); - tDeleteSTaosxRsp(&taosxRsp); - return -1; - } - - if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { - tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); - code = tqSendDataRsp(pTq, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP); - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); - return code; - } else { - fetchVer++; } } } - tDeleteSTaosxRsp(&taosxRsp); - taosMemoryFreeClear(pCkHead); return 0; } @@ -706,7 +385,82 @@ int32_t tqProcessPollReq(STQ* pTq, SRpcMsg* pMsg) { tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey %s, recv poll req vgId:%d, req:%s, reqId:0x%" PRIx64, consumerId, req.epoch, pHandle->subKey, vgId, buf, req.reqId); - return doPollDataForMq(pTq, pHandle, &req, pMsg); + return tqExtractDataForMq(pTq, pHandle, &req, pMsg); +} + +int32_t tqProcessVgWalInfoReq(STQ* pTq, SRpcMsg* pMsg) { + SMqPollReq req = {0}; + if (tDeserializeSMqPollReq(pMsg->pCont, pMsg->contLen, &req) < 0) { + tqError("tDeserializeSMqPollReq %d failed", pMsg->contLen); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + int64_t consumerId = req.consumerId; + STqOffsetVal reqOffset = req.reqOffset; + int32_t vgId = TD_VID(pTq->pVnode); + + // 1. find handle + STqHandle* pHandle = taosHashGet(pTq->pHandle, req.subKey, strlen(req.subKey)); + if (pHandle == NULL) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s not found", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_MSG; + return -1; + } + + // 2. check re-balance status + taosRLockLatch(&pTq->lock); + if (pHandle->consumerId != consumerId) { + tqDebug("ERROR consumer:0x%" PRIx64 " vgId:%d, subkey %s, mismatch for saved handle consumer:0x%" PRIx64, + consumerId, vgId, req.subKey, pHandle->consumerId); + terrno = TSDB_CODE_TMQ_CONSUMER_MISMATCH; + taosRUnLockLatch(&pTq->lock); + return -1; + } + taosRUnLockLatch(&pTq->lock); + + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, &req); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, req.subKey); + if (pOffset != NULL) { + if (pOffset->val.type != TMQ_OFFSET__LOG) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s use snapshot, no valid wal info", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; + } + + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; + dataRsp.rspOffset.version = pOffset->val.version; + } else { + if (req.useSnapshot == true) { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s snapshot not support wal info", consumerId, vgId, req.subKey); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; + } + + dataRsp.rspOffset.type = TMQ_OFFSET__LOG; + + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + dataRsp.rspOffset.version = sver; + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + dataRsp.rspOffset.version = ever; + } else { + tqError("consumer:0x%" PRIx64 " vgId:%d subkey:%s invalid offset type:%d", consumerId, vgId, req.subKey, + reqOffset.type); + terrno = TSDB_CODE_INVALID_PARA; + tDeleteMqDataRsp(&dataRsp); + return -1; + } + } + + tqDoSendDataRsp(&pMsg->info, &dataRsp, req.epoch, req.consumerId, TMQ_MSG_TYPE__WALINFO_RSP, sver, ever); + return 0; } int32_t tqProcessDeleteSubReq(STQ* pTq, int64_t sversion, char* msg, int32_t msgLen) { @@ -821,13 +575,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg pHandle->pRef = pRef; SReadHandle handle = { - .meta = pVnode->pMeta, - .vnode = pVnode, - .initTableReader = true, - .initTqReader = true, - .version = ver, - }; - + .meta = pVnode->pMeta, .vnode = pVnode, .initTableReader = true, .initTqReader = true, .version = ver}; pHandle->snapshotVer = ver; if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { @@ -838,10 +586,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg &pHandle->execHandle.numOfCols, req.newConsumerId); void* scanner = NULL; qExtractStreamScanner(pHandle->execHandle.task, &scanner); - pHandle->execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); + pHandle->execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); } else if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { pHandle->pWalReader = walOpenReader(pVnode->pWal, NULL); - pHandle->execHandle.pExecReader = tqOpenReader(pVnode); + pHandle->execHandle.pTqReader = tqOpenReader(pVnode); pHandle->execHandle.execDb.pFilterOutTbUid = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), false, HASH_NO_LOCK); @@ -860,8 +608,8 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - pHandle->execHandle.pExecReader = tqOpenReader(pVnode); - tqReaderSetTbUidList(pHandle->execHandle.pExecReader, tbUidList); + pHandle->execHandle.pTqReader = tqOpenReader(pVnode); + tqReaderSetTbUidList(pHandle->execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); buildSnapContext(handle.meta, handle.version, req.suid, pHandle->execHandle.subType, pHandle->fetchMeta, @@ -897,7 +645,7 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg atomic_store_32(&pHandle->epoch, -1); // remove if it has been register in the push manager, and return one empty block to consumer - tqUnregisterPushEntry(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); + tqUnregisterPushHandle(pTq, req.subKey, (int32_t)strlen(req.subKey), pHandle->consumerId, true); atomic_store_64(&pHandle->consumerId, req.newConsumerId); atomic_add_fetch_32(&pHandle->epoch, 1); @@ -919,16 +667,10 @@ int32_t tqProcessSubscribeReq(STQ* pTq, int64_t sversion, char* msg, int32_t msg } int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { -#if 0 - if (pTask->taskLevel == TASK_LEVEL__AGG) { - A(taosArrayGetSize(pTask->childEpInfo) != 0); - } -#endif - int32_t vgId = TD_VID(pTq->pVnode); + pTask->id.idStr = createStreamTaskIdStr(pTask->id.streamId, pTask->id.taskId); pTask->refCnt = 1; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; - + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputQueue = streamQueueOpen(); pTask->outputQueue = streamQueueOpen(); @@ -939,11 +681,14 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; pTask->pMsgCb = &pTq->pVnode->msgCb; - pTask->startVer = ver; + pTask->pMeta = pTq->pStreamMeta; + pTask->chkInfo.version = ver; // expand executor if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + } else { + pTask->status.taskStatus = TASK_STATUS__RESTORE; } if (pTask->taskLevel == TASK_LEVEL__SOURCE) { @@ -953,14 +698,10 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { } SReadHandle handle = { - .meta = pTq->pVnode->pMeta, - .vnode = pTq->pVnode, - .initTqReader = 1, - .pStateBackend = pTask->pState, - }; - - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); - if (pTask->exec.executor == NULL) { + .meta = pTq->pVnode->pMeta, .vnode = pTq->pVnode, .initTqReader = 1, .pStateBackend = pTask->pState}; + + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &handle, vgId); + if (pTask->exec.pExecutor == NULL) { return -1; } @@ -969,14 +710,12 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { if (pTask->pState == NULL) { return -1; } - SReadHandle mgHandle = { - .vnode = NULL, - .numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo), - .pStateBackend = pTask->pState, - }; - - pTask->exec.executor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); - if (pTask->exec.executor == NULL) { + + int32_t numOfVgroups = (int32_t)taosArrayGetSize(pTask->childEpInfo); + SReadHandle mgHandle = { .vnode = NULL, .numOfVgroups = numOfVgroups, .pStateBackend = pTask->pState}; + + pTask->exec.pExecutor = qCreateStreamExecTaskInfo(pTask->exec.qmsg, &mgHandle, vgId); + if (pTask->exec.pExecutor == NULL) { return -1; } } @@ -997,16 +736,23 @@ int32_t tqExpandTask(STQ* pTq, SStreamTask* pTask, int64_t ver) { ver1 = info.skmVer; } - pTask->tbSink.pTSchema = - tBuildTSchema(pTask->tbSink.pSchemaWrapper->pSchema, pTask->tbSink.pSchemaWrapper->nCols, ver1); - if (pTask->tbSink.pTSchema == NULL) { + SSchemaWrapper* pschemaWrapper = pTask->tbSink.pSchemaWrapper; + pTask->tbSink.pTSchema = tBuildTSchema(pschemaWrapper->pSchema, pschemaWrapper->nCols, ver1); + if(pTask->tbSink.pTSchema == NULL) { return -1; } } + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { + pTask->exec.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); + } + streamSetupTrigger(pTask); - tqInfo("expand stream task on vg %d, task id %d, child id %d, level %d", vgId, pTask->taskId, pTask->selfChildId, - pTask->taskLevel); + tqInfo("vgId:%d expand stream task, s-task:%s, checkpoint ver:%" PRId64 " child id:%d, level:%d", vgId, pTask->id.idStr, + pTask->chkInfo.version, pTask->selfChildId, pTask->taskLevel); + + // next valid version will add one + pTask->chkInfo.version += 1; return 0; } @@ -1029,18 +775,24 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { .upstreamNodeId = req.upstreamNodeId, .upstreamTaskId = req.upstreamTaskId, }; + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask && atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL) { - rsp.status = 1; + if (pTask) { + rsp.status = (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL) ? 1 : 0; + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + + tqDebug("tq recv task check req(reqId:0x%" PRIx64 + ") %d at node %d task status:%d, check req from task %d at node %d, rsp status %d", + rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, pTask->status.taskStatus, rsp.upstreamTaskId, + rsp.upstreamNodeId, rsp.status); } else { rsp.status = 0; + tqDebug("tq recv task check(taskId:%d not built yet) req(reqId:0x%" PRIx64 + ") %d at node %d, check req from task %d at node %d, rsp status %d", + taskId, rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, + rsp.status); } - if (pTask) streamMetaReleaseTask(pTq->pStreamMeta, pTask); - - tqDebug("tq recv task check req(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", - rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); - SEncoder encoder; int32_t code; int32_t len; @@ -1058,13 +810,7 @@ int32_t tqProcessStreamTaskCheckReq(STQ* pTq, SRpcMsg* pMsg) { tEncodeSStreamTaskCheckRsp(&encoder, &rsp); tEncoderClear(&encoder); - SRpcMsg rspMsg = { - .code = 0, - .pCont = buf, - .contLen = sizeof(SMsgHead) + len, - .info = pMsg->info, - }; - + SRpcMsg rspMsg = { .code = 0, .pCont = buf, .contLen = sizeof(SMsgHead) + len, .info = pMsg->info }; tmsgSendRsp(&rspMsg); return 0; } @@ -1080,8 +826,8 @@ int32_t tqProcessStreamTaskCheckRsp(STQ* pTq, int64_t sversion, char* msg, int32 tDecoderClear(&decoder); return -1; } - tDecoderClear(&decoder); + tDecoderClear(&decoder); tqDebug("tq recv task check rsp(reqId:0x%" PRIx64 ") %d at node %d check req from task %d at node %d, status %d", rsp.reqId, rsp.downstreamTaskId, rsp.downstreamNodeId, rsp.upstreamTaskId, rsp.upstreamNodeId, rsp.status); @@ -1113,17 +859,20 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - code = tDecodeSStreamTask(&decoder, pTask); + code = tDecodeStreamTask(&decoder, pTask); if (code < 0) { tDecoderClear(&decoder); taosMemoryFree(pTask); return -1; } + tDecoderClear(&decoder); - // 2.save task - code = streamMetaAddTask(pTq->pStreamMeta, sversion, pTask); + // 2.save task, use the newest commit version as the initial start version of stream task. + code = streamMetaAddDeployedTask(pTq->pStreamMeta, sversion, pTask); if (code < 0) { + tqError("vgId:%d failed to add s-task:%s, total:%d", TD_VID(pTq->pVnode), pTask->id.idStr, + streamMetaGetNumOfTasks(pTq->pStreamMeta)); return -1; } @@ -1132,6 +881,8 @@ int32_t tqProcessTaskDeployReq(STQ* pTq, int64_t sversion, char* msg, int32_t ms streamTaskCheckDownstream(pTask, sversion); } + tqDebug("vgId:%d s-task:%s is deployed and add meta from mnd, status:%d, total:%d", TD_VID(pTq->pVnode), + pTask->id.idStr, pTask->status.taskStatus, streamMetaGetNumOfTasks(pTq->pStreamMeta)); return 0; } @@ -1147,7 +898,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { } // check param - int64_t fillVer1 = pTask->startVer; + int64_t fillVer1 = pTask->chkInfo.version; if (fillVer1 <= 0) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return -1; @@ -1156,7 +907,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { // do recovery step 1 streamSourceRecoverScanStep1(pTask); - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1171,7 +922,7 @@ int32_t tqProcessTaskRecover1Req(STQ* pTq, SRpcMsg* pMsg) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { return 0; } @@ -1213,7 +964,7 @@ int32_t tqProcessTaskRecover2Req(STQ* pTq, int64_t sversion, char* msg, int32_t return -1; } - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; } @@ -1332,7 +1083,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->taskId, ver); + qDebug("delete req enqueue stream task: %d, ver: %" PRId64, pTask->id.taskId, ver); if (!failed) { SStreamRefDataBlock* pRefBlock = taosAllocateQitem(sizeof(SStreamRefDataBlock), DEF_QITEM, 0); @@ -1341,8 +1092,8 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { pRefBlock->dataRef = pRef; atomic_add_fetch_32(pRefBlock->dataRef, 1); - if (streamTaskInput(pTask, (SStreamQueueItem*)pRefBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->taskId); + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pRefBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->id.taskId); atomic_sub_fetch_32(pRef, 1); taosFreeQitem(pRefBlock); @@ -1350,7 +1101,7 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + qError("stream task launch failed, task id %d", pTask->id.taskId); continue; } @@ -1376,13 +1127,13 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { taosArrayPush(pStreamBlock->blocks, &block); if (!failed) { - if (streamTaskInput(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { - qError("stream task input del failed, task id %d", pTask->taskId); + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pStreamBlock) < 0) { + qError("stream task input del failed, task id %d", pTask->id.taskId); continue; } if (streamSchedExec(pTask) < 0) { - qError("stream task launch failed, task id %d", pTask->taskId); + qError("stream task launch failed, task id %d", pTask->id.taskId); continue; } } else { @@ -1395,18 +1146,32 @@ int32_t tqProcessDelReq(STQ* pTq, void* pReq, int32_t len, int64_t ver) { return 0; } -int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { - void* pIter = NULL; - bool failed = false; - SStreamDataSubmit2* pSubmit = NULL; +static int32_t addSubmitBlockNLaunchTask(STqOffsetStore* pOffsetStore, SStreamTask* pTask, SStreamDataSubmit2* pSubmit, + const char* key, int64_t ver) { + doSaveTaskOffset(pOffsetStore, key, ver); + int32_t code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)pSubmit, ver); - pSubmit = streamDataSubmitNew(submit); + // remove the offset, if all functions are completed successfully. + if (code == TSDB_CODE_SUCCESS) { + tqOffsetDelete(pOffsetStore, key); + } + + return code; +} + +int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { +#if 0 + void* pIter = NULL; + SStreamDataSubmit2* pSubmit = streamDataSubmitNew(submit, STREAM_INPUT__DATA_SUBMIT); if (pSubmit == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; tqError("failed to create data submit for stream since out of memory"); - failed = true; + saveOffsetForAllTasks(pTq, submit.ver); + return -1; } + SArray* pInputQueueFullTasks = taosArrayInit(4, POINTER_BYTES); + while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); if (pIter == NULL) { @@ -1414,46 +1179,79 @@ int32_t tqProcessSubmitReq(STQ* pTq, SPackedData submit) { } SStreamTask* pTask = *(SStreamTask**)pIter; - if (pTask->taskLevel != TASK_LEVEL__SOURCE) continue; - if (pTask->taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { - tqDebug("skip push task %d, task status %d", pTask->taskId, pTask->taskStatus); + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { continue; } - tqDebug("data submit enqueue stream task: %d, ver: %" PRId64, pTask->taskId, submit.ver); + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("stream task:%d skip push data, not ready for processing, status %d", pTask->id.taskId, + pTask->status.taskStatus); + continue; + } - if (!failed) { - if (streamTaskInput(pTask, (SStreamQueueItem*)pSubmit) < 0) { - tqError("stream task input failed, task id %d", pTask->taskId); - continue; - } + // check if offset value exists + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); - if (streamSchedExec(pTask) < 0) { - tqError("stream task launch failed, task id %d", pTask->taskId); - continue; + if (tInputQueueIsFull(pTask)) { + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + + int64_t ver = submit.ver; + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, submit.ver); + } else { + ver = pOffset->val.version; } - } else { - streamTaskInputFail(pTask); + + tqDebug("s-task:%s input queue is full, discard submit block, ver:%" PRId64, pTask->id.idStr, ver); + taosArrayPush(pInputQueueFullTasks, &pTask); + continue; } - } - if (pSubmit) { - streamDataSubmitRefDec(pSubmit); - taosFreeQitem(pSubmit); + // check if offset value exists + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + ASSERT(pOffset == NULL); + + addSubmitBlockNLaunchTask(pTq->pOffsetStore, pTask, pSubmit, key, submit.ver); } - return failed ? -1 : 0; + streamDataSubmitDestroy(pSubmit); + taosFreeQitem(pSubmit); +#endif + + tqStartStreamTasks(pTq); + return 0; } int32_t tqProcessTaskRunReq(STQ* pTq, SRpcMsg* pMsg) { SStreamTaskRunReq* pReq = pMsg->pCont; - int32_t taskId = pReq->taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - if (pTask) { - streamProcessRunReq(pTask); + + int32_t taskId = pReq->taskId; + int32_t vgId = TD_VID(pTq->pVnode); + + if (taskId == WAL_READ_TASKS_ID) { // all tasks are extracted submit data from the wal + tqStreamTasksScanWal(pTq); + return 0; + } + + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + if (pTask != NULL) { + if (pTask->status.taskStatus == TASK_STATUS__NORMAL) { + tqDebug("vgId:%d s-task:%s start to process run req", vgId, pTask->id.idStr); + streamProcessRunReq(pTask); + } else if (pTask->status.taskStatus == TASK_STATUS__RESTORE) { + tqDebug("vgId:%d s-task:%s start to process block from wal, last chk point:%" PRId64, vgId, + pTask->id.idStr, pTask->chkInfo.version); + streamProcessRunReq(pTask); + } else { + tqDebug("vgId:%d s-task:%s ignore run req since not in ready state", vgId, pTask->id.idStr); + } + streamMetaReleaseTask(pTq->pStreamMeta, pTask); + tqStartStreamTasks(pTq); return 0; } else { + tqError("vgId:%d failed to found s-task, taskId:%d", vgId, taskId); return -1; } } @@ -1466,14 +1264,10 @@ int32_t tqProcessTaskDispatchReq(STQ* pTq, SRpcMsg* pMsg, bool exec) { SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msgBody, msgLen); tDecodeStreamDispatchReq(&decoder, &req); - int32_t taskId = req.taskId; - SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); + SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, req.taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchReq(pTask, &req, &rsp, exec); streamMetaReleaseTask(pTq->pStreamMeta, pTask); return 0; @@ -1486,7 +1280,7 @@ int32_t tqProcessTaskDispatchRsp(STQ* pTq, SRpcMsg* pMsg) { SStreamDispatchRsp* pRsp = POINTER_SHIFT(pMsg->pCont, sizeof(SMsgHead)); int32_t taskId = ntohl(pRsp->upstreamTaskId); SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); - tqDebug("recv dispatch rsp, code: %x", pMsg->code); + tqDebug("recv dispatch rsp, code:%x", pMsg->code); if (pTask) { streamProcessDispatchRsp(pTask, pRsp, pMsg->code); streamMetaReleaseTask(pTq->pStreamMeta, pTask); @@ -1514,10 +1308,7 @@ int32_t tqProcessTaskRetrieveReq(STQ* pTq, SRpcMsg* pMsg) { int32_t taskId = req.dstTaskId; SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessRetrieveReq(pTask, &req, &rsp); streamMetaReleaseTask(pTq->pStreamMeta, pTask); tDeleteStreamRetrieveReq(&req); @@ -1553,15 +1344,14 @@ int32_t vnodeEnqueueStreamMsg(SVnode* pVnode, SRpcMsg* pMsg) { SStreamTask* pTask = streamMetaAcquireTask(pTq->pStreamMeta, taskId); if (pTask) { - SRpcMsg rsp = { - .info = pMsg->info, - .code = 0, - }; + SRpcMsg rsp = { .info = pMsg->info, .code = 0 }; streamProcessDispatchReq(pTask, &req, &rsp, false); streamMetaReleaseTask(pTq->pStreamMeta, pTask); rpcFreeCont(pMsg->pCont); taosFreeQitem(pMsg); return 0; + } else { + tDeleteStreamDispatchReq(&req); } code = TSDB_CODE_STREAM_TASK_NOT_EXIST; @@ -1571,10 +1361,7 @@ FAIL: SMsgHead* pRspHead = rpcMallocCont(sizeof(SMsgHead) + sizeof(SStreamDispatchRsp)); if (pRspHead == NULL) { - SRpcMsg rsp = { - .code = TSDB_CODE_OUT_OF_MEMORY, - .info = pMsg->info, - }; + SRpcMsg rsp = { .code = TSDB_CODE_OUT_OF_MEMORY, .info = pMsg->info }; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1592,11 +1379,7 @@ FAIL: pRsp->inputStatus = TASK_OUTPUT_STATUS__NORMAL; SRpcMsg rsp = { - .code = code, - .info = pMsg->info, - .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), - .pCont = pRspHead, - }; + .code = code, .info = pMsg->info, .contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp), .pCont = pRspHead}; tqDebug("send dispatch error rsp, code: %x", code); tmsgSendRsp(&rsp); rpcFreeCont(pMsg->pCont); @@ -1605,3 +1388,45 @@ FAIL: } int32_t tqCheckLogInWal(STQ* pTq, int64_t sversion) { return sversion <= pTq->walLogLastVer; } + +int32_t tqStartStreamTasks(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + + SStreamMeta* pMeta = pTq->pStreamMeta; + taosWLockLatch(&pMeta->lock); + int32_t numOfTasks = taosHashGetSize(pTq->pStreamMeta->pTasks); + if (numOfTasks == 0) { + tqInfo("vgId:%d no stream tasks exists", vgId); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; + } + + pMeta->walScan += 1; + + if (pMeta->walScan > 1) { + tqDebug("vgId:%d wal read task has been launched, remain scan times:%d", vgId, pMeta->walScan); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return 0; + } + + SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); + if (pRunReq == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d failed restore stream tasks, code:%s", vgId, terrstr(terrno)); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + return -1; + } + + tqInfo("vgId:%d start wal scan stream tasks, tasks:%d", vgId, numOfTasks); + initOffsetForAllRestoreTasks(pTq); + + pRunReq->head.vgId = vgId; + pRunReq->streamId = 0; + pRunReq->taskId = WAL_READ_TASKS_ID; + + SRpcMsg msg = {.msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq)}; + tmsgPutToQueue(&pTq->pVnode->msgCb, STREAM_QUEUE, &msg); + taosWUnLockLatch(&pTq->pStreamMeta->lock); + + return 0; +} diff --git a/source/dnode/vnode/src/tq/tqCommit.c b/source/dnode/vnode/src/tq/tqCommit.c index 7fc66c49192eb011ce674bc7a2d396825ba1b435..0f5daa31ad412c8ed0773cca8e09cdeeb0fa80f5 100644 --- a/source/dnode/vnode/src/tq/tqCommit.c +++ b/source/dnode/vnode/src/tq/tqCommit.c @@ -16,10 +16,13 @@ #include "tq.h" int tqCommit(STQ* pTq) { +#if 0 + // stream meta commit does not be aligned to the vnode commit if (streamMetaCommit(pTq->pStreamMeta) < 0) { tqError("vgId:%d, failed to commit stream meta since %s", TD_VID(pTq->pVnode), terrstr()); return -1; } +#endif return tqOffsetCommitFile(pTq->pOffsetStore); } diff --git a/source/dnode/vnode/src/tq/tqMeta.c b/source/dnode/vnode/src/tq/tqMeta.c index 7b0cdab2f8ac7e49ef7f18fc5e971a0399969d65..cd8cefb30741d2d42891380ede286aab6df91f0a 100644 --- a/source/dnode/vnode/src/tq/tqMeta.c +++ b/source/dnode/vnode/src/tq/tqMeta.c @@ -320,15 +320,15 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { code = -1; goto end; } - handle.execHandle.pExecReader = qExtractReaderFromStreamScanner(scanner); - if (handle.execHandle.pExecReader == NULL) { + handle.execHandle.pTqReader = qExtractReaderFromStreamScanner(scanner); + if (handle.execHandle.pTqReader == NULL) { tqError("cannot extract exec reader for %s", handle.subKey); code = -1; goto end; } } else if (handle.execHandle.subType == TOPIC_SUB_TYPE__DB) { handle.pWalReader = walOpenReader(pTq->pVnode->pWal, NULL); - handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); + handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); buildSnapContext(reader.meta, reader.version, 0, handle.execHandle.subType, handle.fetchMeta, (SSnapContext**)(&reader.sContext)); @@ -343,8 +343,8 @@ int32_t tqMetaRestoreHandle(STQ* pTq) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); tqDebug("vgId:%d, idx %d, uid:%" PRId64, vgId, i, tbUid); } - handle.execHandle.pExecReader = tqOpenReader(pTq->pVnode); - tqReaderSetTbUidList(handle.execHandle.pExecReader, tbUidList); + handle.execHandle.pTqReader = tqOpenReader(pTq->pVnode); + tqReaderSetTbUidList(handle.execHandle.pTqReader, tbUidList); taosArrayDestroy(tbUidList); buildSnapContext(reader.meta, reader.version, handle.execHandle.execTb.suid, handle.execHandle.subType, diff --git a/source/dnode/vnode/src/tq/tqOffset.c b/source/dnode/vnode/src/tq/tqOffset.c index 66d1ac2c7e530e4b8c732deaaa83afc6191b5e44..e8051a140616d2c8296a42c0ca4dca11c3816cf9 100644 --- a/source/dnode/vnode/src/tq/tqOffset.c +++ b/source/dnode/vnode/src/tq/tqOffset.c @@ -128,31 +128,35 @@ int32_t tqOffsetDelete(STqOffsetStore* pStore, const char* subscribeKey) { } int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { - if (!pStore->needCommit) return 0; + if (!pStore->needCommit) { + return 0; + } + // TODO file name should be with a newer version char* fname = tqOffsetBuildFName(pStore->pTq->path, 0); TdFilePtr pFile = taosOpenFile(fname, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); if (pFile == NULL) { terrno = TAOS_SYSTEM_ERROR(errno); - - int32_t err = terrno; - const char* errStr = tstrerror(err); - int32_t sysErr = errno; - const char* sysErrStr = strerror(errno); - tqError("vgId:%d, cannot open file %s when commit offset since %s", pStore->pTq->pVnode->config.vgId, fname, - sysErrStr); + const char* err = strerror(errno); + tqError("vgId:%d, failed to open offset file %s, since %s", TD_VID(pStore->pTq->pVnode), fname, err); taosMemoryFree(fname); return -1; } + taosMemoryFree(fname); + void* pIter = NULL; while (1) { pIter = taosHashIterate(pStore->pHash, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + STqOffset* pOffset = (STqOffset*)pIter; int32_t bodyLen; int32_t code; tEncodeSize(tEncodeSTqOffset, pOffset, bodyLen, code); + if (code < 0) { taosHashCancelIterate(pStore->pHash, pIter); return -1; @@ -166,6 +170,7 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { SEncoder encoder; tEncoderInit(&encoder, abuf, bodyLen); tEncodeSTqOffset(&encoder, pOffset); + // write file int64_t writeLen; if ((writeLen = taosWriteFile(pFile, buf, totLen)) != totLen) { @@ -174,8 +179,10 @@ int32_t tqOffsetCommitFile(STqOffsetStore* pStore) { taosMemoryFree(buf); return -1; } + taosMemoryFree(buf); } + // close and rename file taosCloseFile(&pFile); pStore->needCommit = 0; diff --git a/source/dnode/vnode/src/tq/tqPush.c b/source/dnode/vnode/src/tq/tqPush.c index d55b533f8a95319e964a2519dacff60a1e5fab42..43463f67b7dc80be6e0175c69f051b99f84e8670 100644 --- a/source/dnode/vnode/src/tq/tqPush.c +++ b/source/dnode/vnode/src/tq/tqPush.c @@ -30,7 +30,7 @@ static int32_t tqLoopExecFromQueue(STQ* pTq, STqHandle* pHandle, SStreamDataSubm // update processed atomic_store_64(&pHandle->pushHandle.processedVer, pSubmit->ver); streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); if (pRsp->blockNum > 0) { *ppSubmit = pSubmit; return 0; @@ -58,7 +58,7 @@ int32_t tqExecFromInputQ(STQ* pTq, STqHandle* pHandle) { } while (pHandle->pushHandle.processedVer > pSubmit->ver + 1) { streamQueueProcessSuccess(&pHandle->pushHandle.inputQ); - streamDataSubmitRefDec(pSubmit); + streamDataSubmitDestroy(pSubmit); pSubmit = streamQueueNextItem(&pHandle->pushHandle.inputQ); if (pSubmit == NULL) break; } @@ -120,7 +120,7 @@ int32_t tqPreparePush(STQ* pTq, STqHandle* pHandle, int64_t reqId, const SRpcHan int32_t tqEnqueue(STqHandle* pHandle, SStreamDataSubmit* pSubmit) { int8_t inputStatus = atomic_load_8(&pHandle->pushHandle.inputStatus); if (inputStatus == TASK_INPUT_STATUS__NORMAL) { - SStreamDataSubmit* pSubmitClone = streamSubmitRefClone(pSubmit); + SStreamDataSubmit* pSubmitClone = streamSubmitBlockClone(pSubmit); if (pSubmitClone == NULL) { return -1; } @@ -212,28 +212,13 @@ typedef struct { } SItem; static void recordPushedEntry(SArray* cachedKey, void* pIter); +static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq); static void freeItem(void* param) { SItem* p = (SItem*)param; taosMemoryFree(p->pKey); } -static void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { - int32_t vgId = TD_VID(pTq->pVnode); - int32_t numOfKeys = (int32_t)taosArrayGetSize(pCachedKeys); - - for (int32_t i = 0; i < numOfKeys; i++) { - SItem* pItem = taosArrayGet(pCachedKeys, i); - if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { - tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*)pItem->pKey); - } - } - - if (numOfKeys > 0) { - tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); - } -} - static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int64_t ver, int32_t vgId, char* pData, int32_t dataLen, SArray* pCachedKey) { STqPushEntry* pPushEntry = *(STqPushEntry**)pIter; @@ -253,7 +238,7 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6 if (qStreamSetScanMemData(pTaskInfo, submit) != 0) { return; } - + qStreamSetOpen(pTaskInfo); // here start to scan submit block to extract the subscribed data int32_t totalRows = 0; @@ -278,7 +263,7 @@ static void doPushDataForEntry(void* pIter, STqExecHandle* pExec, STQ* pTq, int6 if (pRsp->blockNum > 0) { tqOffsetResetToLog(&pRsp->rspOffset, ver); - tqPushDataRsp(pTq, pPushEntry); + tqPushDataRsp(pPushEntry, vgId); recordPushedEntry(pCachedKey, pIter); } } @@ -338,24 +323,34 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v taosWUnLockLatch(&pTq->lock); } - // push data for stream processing - if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode)) { + tqDebug("handle submit, restore:%d, size:%d", pTq->pVnode->restored, (int)taosHashGetSize(pTq->pStreamMeta->pTasks)); + + // push data for stream processing: + // 1. the vnode has already been restored. + // 2. the vnode should be the leader. + // 3. the stream is not suspended yet. + if (!tsDisableStream && vnodeIsRoleLeader(pTq->pVnode) && pTq->pVnode->restored) { if (taosHashGetSize(pTq->pStreamMeta->pTasks) == 0) { return 0; } if (msgType == TDMT_VND_SUBMIT) { +#if 0 void* data = taosMemoryMalloc(len); if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry terrno = TSDB_CODE_OUT_OF_MEMORY; - tqError("failed to copy data for stream since out of memory"); + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", vgId); return -1; } memcpy(data, pReq, len); SPackedData submit = {.msgStr = data, .msgLen = len, .ver = ver}; - tqDebug("tq copy write msg %p %d %" PRId64 " from %p", data, len, ver, pReq); + tqDebug("vgId:%d tq copy submit msg:%p len:%d ver:%" PRId64 " from %p for stream", vgId, data, len, ver, pReq); + tqProcessSubmitReq(pTq, submit); +#endif + SPackedData submit = {0}; tqProcessSubmitReq(pTq, submit); } @@ -367,14 +362,7 @@ int32_t tqPushMsg(STQ* pTq, void* msg, int32_t msgLen, tmsg_t msgType, int64_t v return 0; } -void recordPushedEntry(SArray* cachedKey, void* pIter) { - size_t kLen = 0; - void* key = taosHashGetKey(pIter, &kLen); - SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; - taosArrayPush(cachedKey, &item); -} - -int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, +int32_t tqRegisterPushHandle(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, SRpcMsg* pRpcMsg, SMqDataRsp* pDataRsp, int32_t type) { uint64_t consumerId = pRequest->consumerId; int32_t vgId = TD_VID(pTq->pVnode); @@ -388,6 +376,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, return -1; } + pPushEntry->pHandle = pHandle; pPushEntry->info = pRpcMsg->info; memcpy(pPushEntry->subKey, pTqHandle->subKey, TSDB_SUBSCRIBE_KEY_LEN); @@ -400,6 +389,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, } SMqRspHead* pHead = &pPushEntry->pDataRsp->head; + pHead->consumerId = consumerId; pHead->epoch = pRequest->epoch; pHead->mqMsgType = type; @@ -411,7 +401,7 @@ int32_t tqRegisterPushEntry(STQ* pTq, void* pHandle, const SMqPollReq* pRequest, return 0; } -int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { +int32_t tqUnregisterPushHandle(STQ* pTq, const char* pKey, int32_t keyLen, uint64_t consumerId, bool rspConsumer) { int32_t vgId = TD_VID(pTq->pVnode); STqPushEntry** pEntry = taosHashGet(pTq->pPushMgr, pKey, keyLen); @@ -423,7 +413,7 @@ int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64 (*pEntry)->subKey, vgId, taosHashGetSize(pTq->pPushMgr) - 1); if (rspConsumer) { // rsp the old consumer with empty block. - tqPushDataRsp(pTq, *pEntry); + tqPushDataRsp(*pEntry, vgId); } taosHashRemove(pTq->pPushMgr, pKey, keyLen); @@ -431,3 +421,26 @@ int32_t tqUnregisterPushEntry(STQ* pTq, const char* pKey, int32_t keyLen, uint64 return 0; } + +void recordPushedEntry(SArray* cachedKey, void* pIter) { + size_t kLen = 0; + void* key = taosHashGetKey(pIter, &kLen); + SItem item = {.pKey = strndup(key, kLen), .keyLen = kLen}; + taosArrayPush(cachedKey, &item); +} + +void doRemovePushedEntry(SArray* pCachedKeys, STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + int32_t numOfKeys = (int32_t)taosArrayGetSize(pCachedKeys); + + for (int32_t i = 0; i < numOfKeys; i++) { + SItem* pItem = taosArrayGet(pCachedKeys, i); + if (taosHashRemove(pTq->pPushMgr, pItem->pKey, pItem->keyLen) != 0) { + tqError("vgId:%d, tq push hash remove key error, key: %s", vgId, (char*)pItem->pKey); + } + } + + if (numOfKeys > 0) { + tqDebug("vgId:%d, pushed %d items and remain:%d", vgId, numOfKeys, (int32_t)taosHashGetSize(pTq->pPushMgr)); + } +} diff --git a/source/dnode/vnode/src/tq/tqRead.c b/source/dnode/vnode/src/tq/tqRead.c index 90ff1f8a84800eb7c1c857b35e48f3cd34359476..25ab7209d2ca33c928ba5b648348bcc329189d1f 100644 --- a/source/dnode/vnode/src/tq/tqRead.c +++ b/source/dnode/vnode/src/tq/tqRead.c @@ -113,7 +113,7 @@ bool isValValidForTable(STqHandle* pHandle, SWalCont* pHead) { } SMetaReader mr = {0}; - metaReaderInit(&mr, pHandle->execHandle.pExecReader->pVnodeMeta, 0); + metaReaderInit(&mr, pHandle->execHandle.pTqReader->pVnodeMeta, 0); if (metaGetTableEntryByName(&mr, req.tbName) < 0) { metaReaderClear(&mr); @@ -262,8 +262,6 @@ STqReader* tqOpenReader(SVnode* pVnode) { } pReader->pVnodeMeta = pVnode->pMeta; - /*pReader->pMsg = NULL;*/ - pReader->ver = -1; pReader->pColIdList = NULL; pReader->cachedSchemaVer = 0; pReader->cachedSchemaSuid = 0; @@ -290,163 +288,112 @@ void tqCloseReader(STqReader* pReader) { } // free hash taosHashCleanup(pReader->tbIdHash); + tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); taosMemoryFree(pReader); } int32_t tqSeekVer(STqReader* pReader, int64_t ver, const char* id) { - // todo set the correct vgId - tqDebug("tmq poll: wal seek to version:%"PRId64" %s", ver, id); if (walReadSeekVer(pReader->pWalReader, ver) < 0) { - tqDebug("tmq poll: wal reader failed to seek to ver:%"PRId64" code:%s, %s", ver, tstrerror(terrno), id); return -1; - } else { - tqDebug("tmq poll: wal reader seek to ver:%"PRId64" %s", ver, id); - return 0; } + tqDebug("wal reader seek to ver:%"PRId64" %s", ver, id); + return 0; } -int32_t tqNextBlock(STqReader* pReader, SFetchRet* ret) { - bool fromProcessedMsg = pReader->msg2.msgStr != NULL; +int32_t extractSubmitMsgFromWal(SWalReader* pReader, SPackedData* pPackedData) { + if (walNextValidMsg(pReader) < 0) { + return -1; + } + + void* pBody = POINTER_SHIFT(pReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + int32_t len = pReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); + int64_t ver = pReader->pHead->head.version; + void* data = taosMemoryMalloc(len); + if (data == NULL) { + // todo: for all stream in this vnode, keep this offset in the offset files, and wait for a moment, and then retry + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("vgId:%d, failed to copy submit data for stream processing, since out of memory", 0); + return -1; + } + + memcpy(data, pBody, len); + *pPackedData = (SPackedData){.ver = ver, .msgLen = len, .msgStr = data}; + return 0; +} + +void tqNextBlock(STqReader* pReader, SFetchRet* ret) { while (1) { - if (!fromProcessedMsg) { + if (pReader->msg2.msgStr == NULL) { if (walNextValidMsg(pReader->pWalReader) < 0) { -// pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; - if(pReader->pWalReader->curInvalid == 0){ - pReader->ver = pReader->pWalReader->curVersion - pReader->pWalReader->curStopped; - }else{ - pReader->ver = walGetLastVer(pReader->pWalReader->pWal); - } - ret->offset.type = TMQ_OFFSET__LOG; - - ret->offset.version = pReader->ver; ret->fetchType = FETCH_TYPE__NONE; - tqDebug("return offset %" PRId64 ", no more valid msg in wal", ret->offset.version); - return -1; + return; } - void* body = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); + void* pBody = POINTER_SHIFT(pReader->pWalReader->pHead->head.body, sizeof(SSubmitReq2Msg)); int32_t bodyLen = pReader->pWalReader->pHead->head.bodyLen - sizeof(SSubmitReq2Msg); int64_t ver = pReader->pWalReader->pHead->head.version; - tqReaderSetSubmitReq2(pReader, body, bodyLen, ver); + tqReaderSetSubmitMsg(pReader, pBody, bodyLen, ver); } - while (tqNextDataBlock2(pReader)) { - // TODO mem free + while (tqNextDataBlock(pReader)) { memset(&ret->data, 0, sizeof(SSDataBlock)); int32_t code = tqRetrieveDataBlock2(&ret->data, pReader, NULL); if (code != 0 || ret->data.info.rows == 0) { continue; } - ret->fetchType = FETCH_TYPE__DATA; - tqDebug("return data rows %" PRId64, ret->data.info.rows); - return 0; - } - if (fromProcessedMsg) { - ret->offset.type = TMQ_OFFSET__LOG; - ret->offset.version = pReader->ver; - ret->fetchType = FETCH_TYPE__SEP; - tqDebug("return offset %" PRId64 ", processed finish", ret->offset.version); - return 0; + ret->fetchType = FETCH_TYPE__DATA; + return; } } } -#if 0 -int32_t tqReaderSetDataMsg(STqReader* pReader, const SSubmitReq* pMsg, int64_t ver) { - pReader->pMsg = pMsg; - -// if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; -// while (true) { -// if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) return -1; -// tqDebug("submitnext vgId:%d, block:%p, dataLen:%d, len:%d, uid:%"PRId64, pWalReader->pWal->cfg.vgId, pReader->pBlock, pReader->msgIter.dataLen, -// pReader->msgIter.len, pReader->msgIter.uid); -// if (pReader->pBlock == NULL) break; -// } - - if (tInitSubmitMsgIter(pMsg, &pReader->msgIter) < 0) return -1; - pReader->ver = ver; - memset(&pReader->blkIter, 0, sizeof(SSubmitBlkIter)); - return 0; -} -#endif - -int32_t tqReaderSetSubmitReq2(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { - ASSERT(pReader->msg2.msgStr == NULL && msgStr && msgLen && (ver >= 0)); - +int32_t tqReaderSetSubmitMsg(STqReader* pReader, void* msgStr, int32_t msgLen, int64_t ver) { pReader->msg2.msgStr = msgStr; pReader->msg2.msgLen = msgLen; pReader->msg2.ver = ver; - pReader->ver = ver; tqDebug("tq reader set msg %p %d", msgStr, msgLen); - - if (pReader->setMsg == 0) { - SDecoder decoder; - tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); - if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { - ASSERT(0); - } + SDecoder decoder; + tDecoderInit(&decoder, pReader->msg2.msgStr, pReader->msg2.msgLen); + if (tDecodeSSubmitReq2(&decoder, &pReader->submit) < 0) { tDecoderClear(&decoder); - pReader->setMsg = 1; + tqError("DecodeSSubmitReq2 error, msgLen:%d, ver:%"PRId64, msgLen, ver); + return -1; } + tDecoderClear(&decoder); return 0; } -#if 0 bool tqNextDataBlock(STqReader* pReader) { - if (pReader->pMsg == NULL) return false; - while (1) { - if (tGetSubmitMsgNext(&pReader->msgIter, &pReader->pBlock) < 0) { - return false; - } - if (pReader->pBlock == NULL) { - pReader->pMsg = NULL; - return false; - } - - if (pReader->tbIdHash == NULL) { - return true; - } - void* ret = taosHashGet(pReader->tbIdHash, &pReader->msgIter.uid, sizeof(int64_t)); - /*tqDebug("search uid %" PRId64, pHandle->msgIter.uid);*/ - if (ret != NULL) { - /*tqDebug("find uid %" PRId64, pHandle->msgIter.uid);*/ - return true; - } - } - return false; -} -#endif - -bool tqNextDataBlock2(STqReader* pReader) { if (pReader->msg2.msgStr == NULL) { return false; } - ASSERT(pReader->setMsg == 1); - - tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, - pReader->msg2.ver, pReader->nextBlk); - int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { - SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); - ASSERT(pSubmitTbData->uid); + tqDebug("tq reader next data block %p, %d %" PRId64 " %d", pReader->msg2.msgStr, pReader->msg2.msgLen, + pReader->msg2.ver, pReader->nextBlk); - if (pReader->tbIdHash == NULL) return true; + SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); + if (pReader->tbIdHash == NULL) { + return true; + } void* ret = taosHashGet(pReader->tbIdHash, &pSubmitTbData->uid, sizeof(int64_t)); if (ret != NULL) { + tqDebug("tq reader block found, ver:%"PRId64", uid:%"PRId64, pReader->msg2.ver, pSubmitTbData->uid); return true; + } else { + tqDebug("tq reader discard block, uid:%"PRId64", continue", pSubmitTbData->uid); } + pReader->nextBlk++; } tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->setMsg = 0; pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; @@ -455,7 +402,6 @@ bool tqNextDataBlock2(STqReader* pReader) { bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { if (pReader->msg2.msgStr == NULL) return false; - ASSERT(pReader->setMsg == 1); int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); while (pReader->nextBlk < blockSz) { @@ -470,7 +416,6 @@ bool tqNextDataBlockFilterOut2(STqReader* pReader, SHashObj* filterOutUids) { } tDestroySSubmitReq2(&pReader->submit, TSDB_MSG_FLG_DECODE); - pReader->setMsg = 0; pReader->nextBlk = 0; pReader->msg2.msgStr = NULL; @@ -506,67 +451,15 @@ int32_t tqMaskBlock(SSchemaWrapper* pDst, SSDataBlock* pBlock, const SSchemaWrap return 0; } -#if 0 -bool tqNextDataBlockFilterOut(STqReader* pHandle, SHashObj* filterOutUids) { - while (1) { - if (tGetSubmitMsgNext(&pHandle->msgIter, &pHandle->pBlock) < 0) { - return false; - } - if (pHandle->pBlock == NULL) return false; - - void* ret = taosHashGet(filterOutUids, &pHandle->msgIter.uid, sizeof(int64_t)); - if (ret == NULL) { - return true; - } - } - return false; -} - -int32_t tqScanSubmitSplit(SArray* pBlocks, SArray* schemas, STqReader* pReader) { - // - int32_t sversion = htonl(pReader->pBlock->sversion); - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("vgId:%d, cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 - "), version %d, possibly dropped table", - pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->msgIter.suid, sversion); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("vgId:%d, cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pWalReader->pWal->cfg.vgId, pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - } - return 0; -} -#endif - int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbData** pSubmitTbDataRet) { - int32_t blockSz = taosArrayGetSize(pReader->submit.aSubmitTbData); - ASSERT(pReader->nextBlk < blockSz); - - tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); - + tqDebug("tq reader retrieve data block %p, index:%d", pReader->msg2.msgStr, pReader->nextBlk); SSubmitTbData* pSubmitTbData = taosArrayGet(pReader->submit.aSubmitTbData, pReader->nextBlk); pReader->nextBlk++; - if (pSubmitTbDataRet) *pSubmitTbDataRet = pSubmitTbData; + if (pSubmitTbDataRet) { + *pSubmitTbDataRet = pSubmitTbData; + } + int32_t sversion = pSubmitTbData->sver; int64_t suid = pSubmitTbData->suid; int64_t uid = pSubmitTbData->uid; @@ -666,33 +559,27 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD int32_t targetIdx = 0; int32_t sourceIdx = 0; while (targetIdx < colActual) { - ASSERT(sourceIdx < numOfCols); - + if(sourceIdx >= numOfCols){ + tqError("tqRetrieveDataBlock2 sourceIdx:%d >= numOfCols:%d", sourceIdx, numOfCols); + goto FAIL; + } SColData* pCol = taosArrayGet(pCols, sourceIdx); SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, targetIdx); SColVal colVal; - ASSERT(pCol->nVal == numOfRows); + if(pCol->nVal != numOfRows){ + tqError("tqRetrieveDataBlock2 pCol->nVal:%d != numOfRows:%d", pCol->nVal, numOfRows); + goto FAIL; + } if (pCol->cid < pColData->info.colId) { sourceIdx++; } else if (pCol->cid == pColData->info.colId) { for (int32_t i = 0; i < pCol->nVal; i++) { tColDataGetValue(pCol, i, &colVal); -#if 0 - void* val = NULL; - if (IS_STR_DATA_TYPE(colVal.type)) { - val = colVal.value.pData; - } else { - val = &colVal.value.val; - } - if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { - goto FAIL; - } -#endif if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { - char val[65535 + 2]; + char val[65535 + 2] = {0}; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -726,8 +613,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD for (int32_t j = 0; j < colActual; j++) { SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, j); while (1) { - ASSERT(sourceIdx < pTschema->numOfCols); - SColVal colVal; tRowGet(pRow, pTschema, sourceIdx, &colVal); if (colVal.cid < pColData->info.colId) { @@ -736,7 +621,7 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD } else if (colVal.cid == pColData->info.colId) { if (IS_STR_DATA_TYPE(colVal.type)) { if (colVal.value.pData != NULL) { - char val[65535 + 2]; + char val[65535 + 2] = {0}; memcpy(varDataVal(val), colVal.value.pData, colVal.value.nData); varDataSetLen(val, colVal.value.nData); if (colDataAppend(pColData, i, val, !COL_VAL_IS_VALUE(&colVal)) < 0) { @@ -745,7 +630,6 @@ int32_t tqRetrieveDataBlock2(SSDataBlock* pBlock, STqReader* pReader, SSubmitTbD } else { colDataSetNULL(pColData, i); } - /*val = colVal.value.pData;*/ } else { if (colDataAppend(pColData, i, (void*)&colVal.value.val, !COL_VAL_IS_VALUE(&colVal)) < 0) { goto FAIL; @@ -771,253 +655,6 @@ FAIL: return -1; } -#if 0 -int32_t tqRetrieveDataBlock(SSDataBlock* pBlock, STqReader* pReader) { - // TODO: cache multiple schema - int32_t sversion = htonl(pReader->pBlock->sversion); - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - if (pReader->pSchema) taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", - pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - pReader->cachedSchemaVer = sversion; - pReader->cachedSchemaSuid = pReader->msgIter.suid; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colNumNeed = taosArrayGetSize(pReader->pColIdList); - - if (colNumNeed == 0) { - int32_t colMeta = 0; - while (colMeta < pSchemaWrapper->nCols) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - goto FAIL; - } - colMeta++; - } - } else { - if (colNumNeed > pSchemaWrapper->nCols) { - colNumNeed = pSchemaWrapper->nCols; - } - - int32_t colMeta = 0; - int32_t colNeed = 0; - while (colMeta < pSchemaWrapper->nCols && colNeed < colNumNeed) { - SSchema* pColSchema = &pSchemaWrapper->pSchema[colMeta]; - col_id_t colIdSchema = pColSchema->colId; - col_id_t colIdNeed = *(col_id_t*)taosArrayGet(pReader->pColIdList, colNeed); - if (colIdSchema < colIdNeed) { - colMeta++; - } else if (colIdSchema > colIdNeed) { - colNeed++; - } else { - SColumnInfoData colInfo = createColumnInfoData(pColSchema->type, pColSchema->bytes, pColSchema->colId); - int32_t code = blockDataAppendColInfo(pBlock, &colInfo); - if (code != TSDB_CODE_SUCCESS) { - goto FAIL; - } - colMeta++; - colNeed++; - } - } - } - - if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - int32_t colActual = blockDataGetNumOfCols(pBlock); - - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - int32_t curRow = 0; - - tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - - pBlock->info.id.uid = pReader->msgIter.uid; - pBlock->info.rows = pReader->msgIter.numOfRows; - pBlock->info.version = pReader->pMsg->version; - pBlock->info.dataLoad = 1; - - while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { - tdSTSRowIterReset(&iter, row); - // get all wanted col of that block - for (int32_t i = 0; i < colActual; i++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SCellVal sVal = {0}; - if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { - break; - } - if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType != TD_VTYPE_NORM) < 0) { - goto FAIL; - } - } - curRow++; - } - return 0; - -FAIL: - blockDataFreeRes(pBlock); - return -1; -} - -int32_t tqRetrieveTaosxBlock(STqReader* pReader, SArray* blocks, SArray* schemas) { - int32_t sversion = htonl(pReader->pBlock->sversion); - - if (pReader->cachedSchemaSuid == 0 || pReader->cachedSchemaVer != sversion || - pReader->cachedSchemaSuid != pReader->msgIter.suid) { - if (pReader->pSchema) taosMemoryFree(pReader->pSchema); - pReader->pSchema = metaGetTbTSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchema == NULL) { - tqWarn("cannot found tsschema for table: uid:%" PRId64 " (suid:%" PRId64 "), version %d, possibly dropped table", - pReader->msgIter.uid, pReader->msgIter.suid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - - if (pReader->pSchemaWrapper) tDeleteSSchemaWrapper(pReader->pSchemaWrapper); - pReader->pSchemaWrapper = metaGetTableSchema(pReader->pVnodeMeta, pReader->msgIter.uid, sversion, 1); - if (pReader->pSchemaWrapper == NULL) { - tqWarn("cannot found schema wrapper for table: suid:%" PRId64 ", version %d, possibly dropped table", - pReader->msgIter.uid, pReader->cachedSchemaVer); - pReader->cachedSchemaSuid = 0; - terrno = TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND; - return -1; - } - pReader->cachedSchemaVer = sversion; - pReader->cachedSchemaSuid = pReader->msgIter.suid; - } - - STSchema* pTschema = pReader->pSchema; - SSchemaWrapper* pSchemaWrapper = pReader->pSchemaWrapper; - - int32_t colAtMost = pSchemaWrapper->nCols; - - int32_t curRow = 0; - int32_t lastRow = 0; - - char* assigned = taosMemoryCalloc(1, pSchemaWrapper->nCols); - if (assigned == NULL) return -1; - - tInitSubmitBlkIter(&pReader->msgIter, pReader->pBlock, &pReader->blkIter); - STSRowIter iter = {0}; - tdSTSRowIterInit(&iter, pTschema); - STSRow* row; - - while ((row = tGetSubmitBlkNext(&pReader->blkIter)) != NULL) { - bool buildNew = false; - tdSTSRowIterReset(&iter, row); - - tqDebug("vgId:%d, row of block %d", pReader->pWalReader->pWal->cfg.vgId, curRow); - for (int32_t i = 0; i < colAtMost; i++) { - SCellVal sVal = {0}; - if (!tdSTSRowIterFetch(&iter, pSchemaWrapper->pSchema[i].colId, pSchemaWrapper->pSchema[i].type, &sVal)) { - break; - } - tqDebug("vgId:%d, %d col, type %d", pReader->pWalReader->pWal->cfg.vgId, i, sVal.valType); - if (curRow == 0) { - assigned[i] = sVal.valType != TD_VTYPE_NONE; - buildNew = true; - } else { - bool currentRowAssigned = sVal.valType != TD_VTYPE_NONE; - if (currentRowAssigned != assigned[i]) { - assigned[i] = currentRowAssigned; - buildNew = true; - } - } - } - - if (buildNew) { - if (taosArrayGetSize(blocks) > 0) { - SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; - lastRow = curRow; - } - SSDataBlock* pBlock = createDataBlock(); - SSchemaWrapper* pSW = taosMemoryCalloc(1, sizeof(SSchemaWrapper)); - if (tqMaskBlock(pSW, pBlock, pSchemaWrapper, assigned) < 0) { - blockDataDestroy(pBlock); - goto FAIL; - } - SSDataBlock block = {0}; - assignOneDataBlock(&block, pBlock); - blockDataDestroy(pBlock); - - tqDebug("vgId:%d, build new block, col %d", pReader->pWalReader->pWal->cfg.vgId, - (int32_t)taosArrayGetSize(block.pDataBlock)); - - taosArrayPush(blocks, &block); - taosArrayPush(schemas, &pSW); - } - - SSDataBlock* pBlock = taosArrayGetLast(blocks); - pBlock->info.id.uid = pReader->msgIter.uid; - pBlock->info.rows = 0; - pBlock->info.version = pReader->pMsg->version; - - tqDebug("vgId:%d, taosx scan, block num: %d", pReader->pWalReader->pWal->cfg.vgId, - (int32_t)taosArrayGetSize(blocks)); - - if (blockDataEnsureCapacity(pBlock, pReader->msgIter.numOfRows - curRow) < 0) { - terrno = TSDB_CODE_OUT_OF_MEMORY; - goto FAIL; - } - - tdSTSRowIterReset(&iter, row); - for (int32_t i = 0; i < taosArrayGetSize(pBlock->pDataBlock); i++) { - SColumnInfoData* pColData = taosArrayGet(pBlock->pDataBlock, i); - SCellVal sVal = {0}; - - if (!tdSTSRowIterFetch(&iter, pColData->info.colId, pColData->info.type, &sVal)) { - break; - } - - ASSERT(sVal.valType != TD_VTYPE_NONE); - - if (colDataSetVal(pColData, curRow, sVal.val, sVal.valType == TD_VTYPE_NULL) < 0) { - goto FAIL; - } - tqDebug("vgId:%d, row %d col %d append %d", pReader->pWalReader->pWal->cfg.vgId, curRow, i, - sVal.valType == TD_VTYPE_NULL); - } - curRow++; - } - SSDataBlock* pLastBlock = taosArrayGetLast(blocks); - pLastBlock->info.rows = curRow - lastRow; - - taosMemoryFree(assigned); - return 0; - -FAIL: - taosMemoryFree(assigned); - return -1; -} -#endif - int32_t tqRetrieveTaosxBlock2(STqReader* pReader, SArray* blocks, SArray* schemas, SSubmitTbData** pSubmitTbDataRet) { tqDebug("tq reader retrieve data block %p, %d", pReader->msg2.msgStr, pReader->nextBlk); @@ -1295,7 +932,7 @@ int tqReaderSetTbUidList(STqReader* pReader, const SArray* tbUidList) { return 0; } -int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { +int tqReaderAddTbUidList(STqReader* pReader, const SArray* pTableUidList) { if (pReader->tbIdHash == NULL) { pReader->tbIdHash = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_BIGINT), true, HASH_ENTRY_LOCK); if (pReader->tbIdHash == NULL) { @@ -1304,8 +941,9 @@ int tqReaderAddTbUidList(STqReader* pReader, const SArray* tbUidList) { } } - for (int i = 0; i < taosArrayGetSize(tbUidList); i++) { - int64_t* pKey = (int64_t*)taosArrayGet(tbUidList, i); + int32_t numOfTables = taosArrayGetSize(pTableUidList); + for (int i = 0; i < numOfTables; i++) { + int64_t* pKey = (int64_t*)taosArrayGet(pTableUidList, i); taosHashPut(pReader->tbIdHash, pKey, sizeof(int64_t), NULL, 0); } @@ -1321,30 +959,34 @@ int tqReaderRemoveTbUidList(STqReader* pReader, const SArray* tbUidList) { return 0; } +// todo update the table list in wal reader int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { - void* pIter = NULL; + void* pIter = NULL; + int32_t vgId = TD_VID(pTq->pVnode); + + // update the table list for each consumer handle while (1) { pIter = taosHashIterate(pTq->pHandle, pIter); if (pIter == NULL) { break; } - STqHandle* pExec = (STqHandle*)pIter; - if (pExec->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { - int32_t code = qUpdateQualifiedTableId(pExec->execHandle.task, tbUidList, isAdd); + STqHandle* pTqHandle = (STqHandle*)pIter; + if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + int32_t code = qUpdateTableListForStreamScanner(pTqHandle->execHandle.task, tbUidList, isAdd); if (code != 0) { - tqError("update qualified table error for %s", pExec->subKey); + tqError("update qualified table error for %s", pTqHandle->subKey); continue; } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__DB) { + } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__DB) { if (!isAdd) { int32_t sz = taosArrayGetSize(tbUidList); for (int32_t i = 0; i < sz; i++) { int64_t tbUid = *(int64_t*)taosArrayGet(tbUidList, i); - taosHashPut(pExec->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); + taosHashPut(pTqHandle->execHandle.execDb.pFilterOutTbUid, &tbUid, sizeof(int64_t), NULL, 0); } } - } else if (pExec->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { + } else if (pTqHandle->execHandle.subType == TOPIC_SUB_TYPE__TABLE) { if (isAdd) { SArray* qa = taosArrayInit(4, sizeof(tb_uid_t)); SMetaReader mr = {0}; @@ -1359,35 +1001,43 @@ int32_t tqUpdateTbUidList(STQ* pTq, const SArray* tbUidList, bool isAdd) { } tDecoderClear(&mr.coder); - - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pExec->execHandle.execTb.suid) { + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pTqHandle->execHandle.execTb.suid) { tqDebug("table uid %" PRId64 " does not add to tq handle", *id); continue; } + tqDebug("table uid %" PRId64 " add to tq handle", *id); taosArrayPush(qa, id); } + metaReaderClear(&mr); if (taosArrayGetSize(qa) > 0) { - tqReaderAddTbUidList(pExec->execHandle.pExecReader, qa); + tqReaderAddTbUidList(pTqHandle->execHandle.pTqReader, qa); } + taosArrayDestroy(qa); } else { - tqReaderRemoveTbUidList(pExec->execHandle.pExecReader, tbUidList); + tqReaderRemoveTbUidList(pTqHandle->execHandle.pTqReader, tbUidList); } } } + + // update the table list handle for each stream scanner/wal reader while (1) { pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - int32_t code = qUpdateQualifiedTableId(pTask->exec.executor, tbUidList, isAdd); + int32_t code = qUpdateTableListForStreamScanner(pTask->exec.pExecutor, tbUidList, isAdd); if (code != 0) { - tqError("update qualified table error for stream task %d", pTask->taskId); + tqError("vgId:%d, s-task:%s update qualified table error for stream task", vgId, pTask->id.idStr); continue; } } } + return 0; } diff --git a/source/dnode/vnode/src/tq/tqRestore.c b/source/dnode/vnode/src/tq/tqRestore.c new file mode 100644 index 0000000000000000000000000000000000000000..54fcc04b62e817f9d8192c40e91a7f3f50e5f862 --- /dev/null +++ b/source/dnode/vnode/src/tq/tqRestore.c @@ -0,0 +1,143 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +static int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle); +static int32_t transferToNormalTask(SStreamMeta* pStreamMeta, SArray* pTaskList); + +// this function should be executed by stream threads. +// there is a case that the WAL increases more fast than the restore procedure, and this restore procedure +// will not stop eventually. +int tqStreamTasksScanWal(STQ* pTq) { + int32_t vgId = TD_VID(pTq->pVnode); + SStreamMeta* pMeta = pTq->pStreamMeta; + int64_t st = taosGetTimestampMs(); + + while (1) { + tqInfo("vgId:%d continue check if data in wal are available", vgId); + + // check all restore tasks + bool allFull = true; + streamTaskReplayWal(pTq->pStreamMeta, pTq->pOffsetStore, &allFull); + + int32_t times = 0; + + if (allFull) { + taosWLockLatch(&pMeta->lock); + pMeta->walScan -= 1; + times = pMeta->walScan; + + if (pMeta->walScan <= 0) { + taosWUnLockLatch(&pMeta->lock); + break; + } + + taosWUnLockLatch(&pMeta->lock); + tqInfo("vgId:%d scan wal for stream tasks for %d times", vgId, times); + } + } + + double el = (taosGetTimestampMs() - st) / 1000.0; + tqInfo("vgId:%d scan wal for stream tasks completed, elapsed time:%.2f sec", vgId, el); + return 0; +} + +int32_t streamTaskReplayWal(SStreamMeta* pStreamMeta, STqOffsetStore* pOffsetStore, bool* pScanIdle) { + void* pIter = NULL; + int32_t vgId = pStreamMeta->vgId; + + *pScanIdle = true; + + bool allWalChecked = true; + tqDebug("vgId:%d start to check wal to extract new submit block", vgId); + + while (1) { + pIter = taosHashIterate(pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + int8_t status = pTask->status.taskStatus; + if (status == TASK_STATUS__RECOVER_PREPARE || status == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, + pTask->status.taskStatus); + continue; + } + + // check if offset value exists + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + if (tInputQueueIsFull(pTask)) { + tqDebug("vgId:%d s-task:%s input queue is full, do nothing", vgId, pTask->id.idStr); + continue; + } + + *pScanIdle = false; + + // check if offset value exists + STqOffset* pOffset = tqOffsetRead(pOffsetStore, key); + ASSERT(pOffset != NULL); + + // seek the stored version and extract data from WAL + int32_t code = walReadSeekVer(pTask->exec.pWalReader, pOffset->val.version); + if (code != TSDB_CODE_SUCCESS) { // no data in wal, quit + continue; + } + + // append the data for the stream + tqDebug("vgId:%d wal reader seek to ver:%" PRId64 " %s", vgId, pOffset->val.version, pTask->id.idStr); + + SPackedData packData = {0}; + code = extractSubmitMsgFromWal(pTask->exec.pWalReader, &packData); + if (code != TSDB_CODE_SUCCESS) { // failed, continue + continue; + } + + SStreamDataSubmit2* p = streamDataSubmitNew(packData, STREAM_INPUT__DATA_SUBMIT); + if (p == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + tqError("%s failed to create data submit for stream since out of memory", pTask->id.idStr); + continue; + } + + allWalChecked = false; + + tqDebug("s-task:%s submit data extracted from WAL", pTask->id.idStr); + code = tqAddInputBlockNLaunchTask(pTask, (SStreamQueueItem*)p, packData.ver); + if (code == TSDB_CODE_SUCCESS) { + pOffset->val.version = walReaderGetCurrentVer(pTask->exec.pWalReader); + tqDebug("s-task:%s set the ver:%" PRId64 " from WALReader after extract block from WAL", pTask->id.idStr, + pOffset->val.version); + } else { + // do nothing + } + + streamDataSubmitDestroy(p); + taosFreeQitem(p); + } + + if (allWalChecked) { + *pScanIdle = true; + } + return 0; +} + diff --git a/source/dnode/vnode/src/tq/tqScan.c b/source/dnode/vnode/src/tq/tqScan.c index 6528b7c8d2a82f79f97cd984cb9d918f26dbacc6..27db66f048df70ebeaa0e566a591f3ce1ffbcebd 100644 --- a/source/dnode/vnode/src/tq/tqScan.c +++ b/source/dnode/vnode/src/tq/tqScan.c @@ -38,7 +38,7 @@ int32_t tqAddBlockDataToRsp(const SSDataBlock* pBlock, SMqDataRsp* pRsp, int32_t } static int32_t tqAddBlockSchemaToRsp(const STqExecHandle* pExec, STaosxRsp* pRsp) { - SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pExecReader->pSchemaWrapper); + SSchemaWrapper* pSW = tCloneSSchemaWrapper(pExec->pTqReader->pSchemaWrapper); if (pSW == NULL) { return -1; } @@ -74,33 +74,23 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, vgId:%d, consumer:0x%" PRIx64, vgId, pHandle->consumerId); - if (pOffset->type == TMQ_OFFSET__LOG) { - pRsp->rspOffset = *pOffset; - return code; - } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer); - if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, vgId:%d, consumer:0x%" PRIx64, vgId, pHandle->consumerId); - pRsp->rspOffset = *pOffset; - return code; - } - } + tqError("prepare scan failed, return"); + return -1; } while (1) { SSDataBlock* pDataBlock = NULL; uint64_t ts = 0; - - tqDebug("vgId:%d, tmq task start to execute, consumer:0x%" PRIx64, vgId, pHandle->consumerId); - - code = qExecTask(task, &pDataBlock, &ts); - if (code != TSDB_CODE_SUCCESS) { - tqError("vgId:%d, task exec error since %s, consumer:0x%" PRIx64, vgId, terrstr(), pHandle->consumerId); - return code; + qStreamSetOpen(task); + tqDebug("consumer:0x%" PRIx64 " vgId:%d, tmq one task start execute", pHandle->consumerId, vgId); + if (qExecTask(task, &pDataBlock, &ts) != TSDB_CODE_SUCCESS) { + tqError("consumer:0x%" PRIx64 " vgId:%d, task exec error since %s", pHandle->consumerId, vgId, terrstr()); + return -1; } - // current scan should be stopped ASAP, since the re-balance occurs. + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq one task end executed, pDataBlock:%p", pHandle->consumerId, vgId, + pDataBlock); + // current scan should be stopped asap, since the rebalance occurs. if (pDataBlock == NULL) { break; } @@ -112,37 +102,16 @@ int32_t tqScanData(STQ* pTq, const STqHandle* pHandle, SMqDataRsp* pRsp, STqOffs } pRsp->blockNum++; - - tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, rows:%" PRId64 ", total blocks:%d", vgId, - pHandle->consumerId, pDataBlock->info.rows, pRsp->blockNum); - - if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - totalRows += pDataBlock->info.rows; - if (totalRows >= MAX_ROWS_TO_RETURN) { - break; - } + totalRows += pDataBlock->info.rows; + if (totalRows >= MAX_ROWS_TO_RETURN) { + break; } } + tqDebug("consumer:0x%" PRIx64 " vgId:%d tmq task executed finished, total blocks:%d, totalRows:%d", + pHandle->consumerId, vgId, pRsp->blockNum, totalRows); qStreamExtractOffset(task, &pRsp->rspOffset); - - if (pRsp->rspOffset.type == 0) { - code = TSDB_CODE_INVALID_PARA; - tqError("vgId:%d, expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, vgId, pRsp->rspOffset.type, - pRsp->rspOffset.ts, pRsp->rspOffset.uid, pRsp->rspOffset.version); - return code; - } - - if (pRsp->withTbName || pRsp->withSchema) { - code = TSDB_CODE_INVALID_PARA; - tqError("vgId:%d, get column should not with meta:%d,%d", vgId, pRsp->withTbName, pRsp->withSchema); - return code; - } - - tqDebug("vgId:%d, consumer:0x%" PRIx64 " tmq task executed, total blocks:%d, rows:%d", vgId, pHandle->consumerId, - pRsp->blockNum, totalRows); - - return code; + return 0; } int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMetaRsp* pMetaRsp, STqOffsetVal* pOffset) { @@ -150,18 +119,8 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta qTaskInfo_t task = pExec->task; if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - if (pOffset->type == TMQ_OFFSET__LOG) { - pRsp->rspOffset = *pOffset; - return 0; - } else { - tqOffsetResetToLog(pOffset, pHandle->snapshotVer); - if (qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType) < 0) { - tqDebug("prepare scan failed, return"); - pRsp->rspOffset = *pOffset; - return 0; - } - } + tqDebug("tqScanTaosx prepare scan failed, return"); + return -1; } int32_t rowCnt = 0; @@ -178,7 +137,7 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta if (pDataBlock != NULL && pDataBlock->info.rows > 0) { if (pRsp->withTbName) { if (pOffset->type == TMQ_OFFSET__LOG) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, 1) < 0) { continue; } @@ -207,42 +166,32 @@ int32_t tqScanTaosx(STQ* pTq, const STqHandle* pHandle, STaosxRsp* pRsp, SMqMeta } } - if (pDataBlock == NULL && pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { - if (qStreamExtractPrepareUid(task) != 0) { + // get meta + SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); + if (tmp->metaRspLen > 0) { + qStreamExtractOffset(task, &tmp->rspOffset); + *pMetaRsp = *tmp; + + tqDebug("tmqsnap task get meta"); + break; + } + + if (pDataBlock == NULL) { + qStreamExtractOffset(task, pOffset); + if (pOffset->type == TMQ_OFFSET__SNAPSHOT_DATA) { continue; } tqDebug("tmqsnap vgId: %d, tsdb consume over, switch to wal, ver %" PRId64, TD_VID(pTq->pVnode), pHandle->snapshotVer + 1); + qStreamExtractOffset(task, &pRsp->rspOffset); break; } if (pRsp->blockNum > 0) { tqDebug("tmqsnap task exec exited, get data"); + qStreamExtractOffset(task, &pRsp->rspOffset); break; } - - SMqMetaRsp* tmp = qStreamExtractMetaMsg(task); - if (tmp->rspOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { - tqOffsetResetToData(pOffset, tmp->rspOffset.uid, tmp->rspOffset.ts); - qStreamPrepareScan(task, pOffset, pHandle->execHandle.subType); - tmp->rspOffset.type = TMQ_OFFSET__SNAPSHOT_META; - tqDebug("tmqsnap task exec change to get data"); - continue; - } - - *pMetaRsp = *tmp; - tqDebug("tmqsnap task exec exited, get meta"); - - tqDebug("task exec exited"); - break; - } - - qStreamExtractOffset(task, &pRsp->rspOffset); - - if (pRsp->rspOffset.type == 0) { - tqError("expected rsp offset: type %d %" PRId64 " %" PRId64 " %" PRId64, pRsp->rspOffset.type, pRsp->rspOffset.ts, - pRsp->rspOffset.uid, pRsp->rspOffset.version); - return -1; } return 0; @@ -254,9 +203,9 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR SArray* pSchemas = taosArrayInit(0, sizeof(void*)); if (pExec->subType == TOPIC_SUB_TYPE__TABLE) { - STqReader* pReader = pExec->pExecReader; - tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); - while (tqNextDataBlock2(pReader)) { + STqReader* pReader = pExec->pTqReader; + tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); + while (tqNextDataBlock(pReader)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); SSubmitTbData* pSubmitTbDataRet = NULL; @@ -264,7 +213,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); @@ -313,8 +262,8 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR } } } else if (pExec->subType == TOPIC_SUB_TYPE__DB) { - STqReader* pReader = pExec->pExecReader; - tqReaderSetSubmitReq2(pReader, submit.msgStr, submit.msgLen, submit.ver); + STqReader* pReader = pExec->pTqReader; + tqReaderSetSubmitMsg(pReader, submit.msgStr, submit.msgLen, submit.ver); while (tqNextDataBlockFilterOut2(pReader, pExec->execDb.pFilterOutTbUid)) { taosArrayClear(pBlocks); taosArrayClear(pSchemas); @@ -323,7 +272,7 @@ int32_t tqTaosxScanLog(STQ* pTq, STqHandle* pHandle, SPackedData submit, STaosxR if (terrno == TSDB_CODE_TQ_TABLE_SCHEMA_NOT_FOUND) continue; } if (pRsp->withTbName) { - int64_t uid = pExec->pExecReader->lastBlkUid; + int64_t uid = pExec->pTqReader->lastBlkUid; if (tqAddTbNameToRsp(pTq, uid, pRsp, taosArrayGetSize(pBlocks)) < 0) { taosArrayDestroyEx(pBlocks, (FDelete)blockDataFreeRes); taosArrayDestroyP(pSchemas, (FDelete)tDeleteSSchemaWrapper); diff --git a/source/dnode/vnode/src/tq/tqSink.c b/source/dnode/vnode/src/tq/tqSink.c index 4645df5b67c121065b0844a7bca345205479e1b2..62b81305b7233edd48e7dab796975b1ab69a7ad3 100644 --- a/source/dnode/vnode/src/tq/tqSink.c +++ b/source/dnode/vnode/src/tq/tqSink.c @@ -87,7 +87,7 @@ void tqSinkToTablePipeline(SStreamTask* pTask, void* vnode, int64_t ver, void* d return; } - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + tqDebug("vgId:%d, s-task:%s write into table, block num: %d", TD_VID(pVnode), pTask->id.idStr, blockSz); for (int32_t i = 0; i < blockSz; i++) { bool createTb = true; SSDataBlock* pDataBlock = taosArrayGet(pBlocks, i); @@ -382,7 +382,7 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* int32_t blockSz = taosArrayGetSize(pBlocks); - tqDebug("vgId:%d, task %d write into table, block num: %d", TD_VID(pVnode), pTask->taskId, blockSz); + tqDebug("vgId:%d, s-task:%s write results blocks:%d into table", TD_VID(pVnode), pTask->id.idStr, blockSz); void* pBuf = NULL; SArray* tagArray = NULL; @@ -475,11 +475,9 @@ void tqSinkToTablePipeline2(SStreamTask* pTask, void* vnode, int64_t ver, void* } for (int32_t tagId = UD_TAG_COLUMN_INDEX, step = 1; tagId < size; tagId++, step++) { SColumnInfoData* pTagData = taosArrayGet(pDataBlock->pDataBlock, tagId); - STagVal tagVal = { - .cid = pTSchema->numOfCols + step, - .type = pTagData->info.type, - }; - void* pData = colDataGetData(pTagData, rowId); + + STagVal tagVal = {.cid = pTSchema->numOfCols + step, .type = pTagData->info.type}; + void* pData = colDataGetData(pTagData, rowId); if (colDataIsNull_s(pTagData, rowId)) { continue; } else if (IS_VAR_DATA_TYPE(pTagData->info.type)) { diff --git a/source/dnode/vnode/src/tq/tqUtil.c b/source/dnode/vnode/src/tq/tqUtil.c new file mode 100644 index 0000000000000000000000000000000000000000..c09a8a030e78907d7564333cba773b1883bdacae --- /dev/null +++ b/source/dnode/vnode/src/tq/tqUtil.c @@ -0,0 +1,527 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +#include "tq.h" + +#define IS_OFFSET_RESET_TYPE(_t) ((_t) < 0) + +static int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId); + +char* createStreamTaskIdStr(int64_t streamId, int32_t taskId) { + char buf[128] = {0}; + sprintf(buf, "0x%" PRIx64 "-%d", streamId, taskId); + return taosStrdup(buf); +} + +// stream_task:stream_id:task_id +void createStreamTaskOffsetKey(char* dst, uint64_t streamId, uint32_t taskId) { + int32_t n = 12; + char* p = dst; + + memcpy(p, "stream_task:", n); + p += n; + + int32_t inc = tintToHex(streamId, p); + p += inc; + + *(p++) = ':'; + tintToHex(taskId, p); +} + +int32_t tqAddInputBlockNLaunchTask(SStreamTask* pTask, SStreamQueueItem* pQueueItem, int64_t ver) { + int32_t code = tAppendDataToInputQueue(pTask, pQueueItem); + if (code < 0) { + tqError("s-task:%s failed to put into queue, too many, next start ver:%" PRId64, pTask->id.idStr, ver); + return -1; + } + + if (streamSchedExec(pTask) < 0) { + tqError("stream task:%d failed to be launched, code:%s", pTask->id.taskId, tstrerror(terrno)); + return -1; + } + + return TSDB_CODE_SUCCESS; +} + +void initOffsetForAllRestoreTasks(STQ* pTq) { + void* pIter = NULL; + + while(1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, since not ready, status %d", pTask->id.idStr, pTask->status.taskStatus); + continue; + } + + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, pTask->chkInfo.version); + } + } +} + +void saveOffsetForAllTasks(STQ* pTq, int64_t ver) { + void* pIter = NULL; + + while(1) { + pIter = taosHashIterate(pTq->pStreamMeta->pTasks, pIter); + if (pIter == NULL) { + break; + } + + SStreamTask* pTask = *(SStreamTask**)pIter; + if (pTask->taskLevel != TASK_LEVEL__SOURCE) { + continue; + } + + if (pTask->status.taskStatus == TASK_STATUS__RECOVER_PREPARE || pTask->status.taskStatus == TASK_STATUS__WAIT_DOWNSTREAM) { + tqDebug("s-task:%s skip push data, not ready for processing, status %d", pTask->id.idStr, + pTask->status.taskStatus); + continue; + } + + char key[128] = {0}; + createStreamTaskOffsetKey(key, pTask->id.streamId, pTask->id.taskId); + + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, key); + if (pOffset == NULL) { + doSaveTaskOffset(pTq->pOffsetStore, key, ver); + } + } +} + +void doSaveTaskOffset(STqOffsetStore* pOffsetStore, const char* pKey, int64_t ver) { + STqOffset offset = {0}; + tqOffsetResetToLog(&offset.val, ver); + + tstrncpy(offset.subKey, pKey, tListLen(offset.subKey)); + + // keep the offset info in the offset store + tqOffsetWrite(pOffsetStore, &offset); +} + +int32_t tqInitDataRsp(SMqDataRsp* pRsp, const SMqPollReq* pReq) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL) { + return -1; + } + + pRsp->withTbName = 0; + pRsp->withSchema = false; + return 0; +} + +static int32_t tqInitTaosxRsp(STaosxRsp* pRsp, const SMqPollReq* pReq) { + pRsp->reqOffset = pReq->reqOffset; + + pRsp->withTbName = 1; + pRsp->withSchema = 1; + pRsp->blockData = taosArrayInit(0, sizeof(void*)); + pRsp->blockDataLen = taosArrayInit(0, sizeof(int32_t)); + pRsp->blockTbName = taosArrayInit(0, sizeof(void*)); + pRsp->blockSchema = taosArrayInit(0, sizeof(void*)); + + if (pRsp->blockData == NULL || pRsp->blockDataLen == NULL || pRsp->blockTbName == NULL || pRsp->blockSchema == NULL) { + if (pRsp->blockData != NULL) { + pRsp->blockData = taosArrayDestroy(pRsp->blockData); + } + + if (pRsp->blockDataLen != NULL) { + pRsp->blockDataLen = taosArrayDestroy(pRsp->blockDataLen); + } + + if (pRsp->blockTbName != NULL) { + pRsp->blockTbName = taosArrayDestroy(pRsp->blockTbName); + } + + if (pRsp->blockSchema != NULL) { + pRsp->blockSchema = taosArrayDestroy(pRsp->blockSchema); + } + return -1; + } + + return 0; +} + +static int32_t extractResetOffsetVal(STqOffsetVal* pOffsetVal, STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, bool* pBlockReturned) { + uint64_t consumerId = pRequest->consumerId; + STqOffsetVal reqOffset = pRequest->reqOffset; + STqOffset* pOffset = tqOffsetRead(pTq->pOffsetStore, pRequest->subKey); + int32_t vgId = TD_VID(pTq->pVnode); + + *pBlockReturned = false; + + // In this vnode, data has been polled by consumer for this topic, so let's continue from the last offset value. + if (pOffset != NULL) { + *pOffsetVal = pOffset->val; + + char formatBuf[80]; + tFormatOffset(formatBuf, 80, pOffsetVal); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, existed offset found, offset reset to %s and continue. reqId:0x%"PRIx64, + consumerId, pHandle->subKey, vgId, formatBuf, pRequest->reqId); + return 0; + } else { + // no poll occurs in this vnode for this topic, let's seek to the right offset value. + if (reqOffset.type == TMQ_OFFSET__RESET_EARLIEAST) { + if (pRequest->useSnapshot) { + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey:%s, vgId:%d, (earliest) set offset to be snapshot", + consumerId, pHandle->subKey, vgId); + + if (pHandle->fetchMeta) { + tqOffsetResetToMeta(pOffsetVal, 0); + } else { + tqOffsetResetToData(pOffsetVal, 0, 0); + } + } else { + pHandle->pRef = walRefFirstVer(pTq->pVnode->pWal, pHandle->pRef); + if (pHandle->pRef == NULL) { + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + + // offset set to previous version when init + tqOffsetResetToLog(pOffsetVal, pHandle->pRef->refVer - 1); + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_LATEST) { + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest); + + tqOffsetResetToLog(&dataRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, (latest) offset reset to %" PRId64, consumerId, + pHandle->subKey, vgId, dataRsp.rspOffset.version); + int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, &dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); + tDeleteMqDataRsp(&dataRsp); + + *pBlockReturned = true; + return code; + } else { + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + tqOffsetResetToLog(&taosxRsp.rspOffset, walGetLastVer(pTq->pVnode->pWal)); + int32_t code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + + *pBlockReturned = true; + return code; + } + } else if (reqOffset.type == TMQ_OFFSET__RESET_NONE) { + tqError("tmq poll: subkey:%s, no offset committed for consumer:0x%" PRIx64 " in vg %d, subkey %s, reset none failed", + pHandle->subKey, consumerId, vgId, pRequest->subKey); + terrno = TSDB_CODE_TQ_NO_COMMITTED_OFFSET; + return -1; + } + } + + return 0; +} + +static int32_t extractDataAndRspForNormalSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, + SRpcMsg* pMsg, STqOffsetVal* pOffset) { + char buf[80] = {0}; + uint64_t consumerId = pRequest->consumerId; + int32_t vgId = TD_VID(pTq->pVnode); + + SMqDataRsp dataRsp = {0}; + tqInitDataRsp(&dataRsp, pRequest); + + // lock + taosWLockLatch(&pTq->lock); + + qSetTaskId(pHandle->execHandle.task, consumerId, pRequest->reqId); + + int code = tqScanData(pTq, pHandle, &dataRsp, pOffset); + if (code == 0) { + + // till now, all data has been transferred to consumer, new data needs to push client once arrived. + if (dataRsp.blockNum == 0 && dataRsp.reqOffset.type == TMQ_OFFSET__LOG && + dataRsp.reqOffset.version == dataRsp.rspOffset.version && pHandle->consumerId == pRequest->consumerId) { + code = tqRegisterPushHandle(pTq, pHandle, pRequest, pMsg, &dataRsp, TMQ_MSG_TYPE__POLL_RSP); + taosWUnLockLatch(&pTq->lock); + return code; + } + + // NOTE: this pHandle->consumerId may have been changed already. + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&dataRsp, TMQ_MSG_TYPE__POLL_RSP, vgId); + } + + tFormatOffset(buf, 80, &dataRsp.rspOffset); + tqDebug("tmq poll: consumer:0x%" PRIx64 ", subkey %s, vgId:%d, rsp block:%d, rsp offset type:%s, reqId:0x%" PRIx64 + " code:%d", + consumerId, pHandle->subKey, vgId, dataRsp.blockNum, buf, pRequest->reqId, code); + taosWUnLockLatch(&pTq->lock); + tDeleteMqDataRsp(&dataRsp); + + return code; +} + +static int32_t extractDataAndRspForDbStbSubscribe(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg, STqOffsetVal *offset) { + int code = 0; + int32_t vgId = TD_VID(pTq->pVnode); + SWalCkHead* pCkHead = NULL; + SMqMetaRsp metaRsp = {0}; + STaosxRsp taosxRsp = {0}; + tqInitTaosxRsp(&taosxRsp, pRequest); + + if (offset->type != TMQ_OFFSET__LOG) { + if (tqScanTaosx(pTq, pHandle, &taosxRsp, &metaRsp, offset) < 0) { + tDeleteSTaosxRsp(&taosxRsp); + return -1; + } + + if (metaRsp.metaRspLen > 0) { + code = tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId); + tqDebug("tmq poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send meta offset type:%d,uid:%" PRId64 ",ts:%" PRId64, + pRequest->consumerId, pHandle->subKey, vgId, metaRsp.rspOffset.type, metaRsp.rspOffset.uid, metaRsp.rspOffset.ts); + taosMemoryFree(metaRsp.metaRsp); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + tqDebug("taosx poll: consumer:0x%" PRIx64 " subkey:%s vgId:%d, send data blockNum:%d, offset type:%d,uid:%" PRId64 + ",ts:%" PRId64,pRequest->consumerId, pHandle->subKey, vgId, taosxRsp.blockNum, taosxRsp.rspOffset.type, taosxRsp.rspOffset.uid,taosxRsp.rspOffset.ts); + if (taosxRsp.blockNum > 0) { + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + return code; + }else { + *offset = taosxRsp.rspOffset; + } + } + + if (offset->type == TMQ_OFFSET__LOG) { + int64_t fetchVer = offset->version + 1; + pCkHead = taosMemoryMalloc(sizeof(SWalCkHead) + 2048); + if (pCkHead == NULL) { + tDeleteSTaosxRsp(&taosxRsp); + terrno = TSDB_CODE_OUT_OF_MEMORY; + return -1; + } + walSetReaderCapacity(pHandle->pWalReader, 2048); + int totalRows = 0; + while (1) { + int32_t savedEpoch = atomic_load_32(&pHandle->epoch); + if (savedEpoch > pRequest->epoch) { + tqWarn("tmq poll: consumer:0x%" PRIx64 " (epoch %d), subkey:%s vgId:%d offset %" PRId64 + ", found new consumer epoch %d, discard req epoch %d", + pRequest->consumerId, pRequest->epoch, pHandle->subKey, vgId, fetchVer, savedEpoch, pRequest->epoch); + break; + } + + if (tqFetchLog(pTq, pHandle, &fetchVer, &pCkHead, pRequest->reqId) < 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } + + SWalCont* pHead = &pCkHead->head; + tqDebug("tmq poll: consumer:0x%" PRIx64 " (epoch %d) iter log, vgId:%d offset %" PRId64 " msgType %d", pRequest->consumerId, + pRequest->epoch, vgId, fetchVer, pHead->msgType); + + // process meta + if (pHead->msgType != TDMT_VND_SUBMIT) { + if(totalRows > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer - 1); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } + + tqDebug("fetch meta msg, ver:%" PRId64 ", type:%s", pHead->version, TMSG_INFO(pHead->msgType)); + tqOffsetResetToLog(&metaRsp.rspOffset, fetchVer); + metaRsp.resMsgType = pHead->msgType; + metaRsp.metaRspLen = pHead->bodyLen; + metaRsp.metaRsp = pHead->body; + if (tqSendMetaPollRsp(pHandle, pMsg, pRequest, &metaRsp, vgId) < 0) { + code = -1; + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + code = 0; + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return code; + } + + // process data + SPackedData submit = { + .msgStr = POINTER_SHIFT(pHead->body, sizeof(SSubmitReq2Msg)), + .msgLen = pHead->bodyLen - sizeof(SSubmitReq2Msg), + .ver = pHead->version, + }; + + if (tqTaosxScanLog(pTq, pHandle, submit, &taosxRsp, &totalRows) < 0) { + tqError("tmq poll: tqTaosxScanLog error %" PRId64 ", in vgId:%d, subkey %s", pRequest->consumerId, vgId, + pRequest->subKey); + taosMemoryFreeClear(pCkHead); + tDeleteSTaosxRsp(&taosxRsp); + return -1; + } + + if (totalRows >= 4096 || taosxRsp.createTableNum > 0) { + tqOffsetResetToLog(&taosxRsp.rspOffset, fetchVer); + code = tqSendDataRsp(pHandle, pMsg, pRequest, (SMqDataRsp*)&taosxRsp, TMQ_MSG_TYPE__TAOSX_RSP, vgId); + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return code; + } else { + fetchVer++; + } + } + } + + tDeleteSTaosxRsp(&taosxRsp); + taosMemoryFreeClear(pCkHead); + return 0; +} + +int32_t tqExtractDataForMq(STQ* pTq, STqHandle* pHandle, const SMqPollReq* pRequest, SRpcMsg* pMsg) { + int32_t code = -1; + STqOffsetVal offset = {0}; + STqOffsetVal reqOffset = pRequest->reqOffset; + + // 1. reset the offset if needed + if (IS_OFFSET_RESET_TYPE(reqOffset.type)) { + // handle the reset offset cases, according to the consumer's choice. + bool blockReturned = false; + code = extractResetOffsetVal(&offset, pTq, pHandle, pRequest, pMsg, &blockReturned); + if (code != 0) { + return code; + } + + // empty block returned, quit + if (blockReturned) { + return 0; + } + } else { // use the consumer specified offset + // the offset value can not be monotonious increase?? + offset = reqOffset; + } + + // this is a normal subscribe requirement + if (pHandle->execHandle.subType == TOPIC_SUB_TYPE__COLUMN) { + return extractDataAndRspForNormalSubscribe(pTq, pHandle, pRequest, pMsg, &offset); + } + + // todo handle the case where re-balance occurs. + // for taosx + return extractDataAndRspForDbStbSubscribe(pTq, pHandle, pRequest, pMsg, &offset); +} + +static void initMqRspHead(SMqRspHead* pMsgHead, int32_t type, int32_t epoch, int64_t consumerId, int64_t sver, + int64_t ever) { + pMsgHead->consumerId = consumerId; + pMsgHead->epoch = epoch; + pMsgHead->mqMsgType = type; + pMsgHead->walsver = sver; + pMsgHead->walever = ever; +} + +int32_t tqSendMetaPollRsp(STqHandle* pHandle, const SRpcMsg* pMsg, const SMqPollReq* pReq, const SMqMetaRsp* pRsp, int32_t vgId) { + int32_t len = 0; + int32_t code = 0; + tEncodeSize(tEncodeMqMetaRsp, pRsp, len, code); + if (code < 0) { + return -1; + } + int32_t tlen = sizeof(SMqRspHead) + len; + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + int64_t sver = 0, ever = 0; + walReaderValidVersionRange(pHandle->execHandle.pTqReader->pWalReader, &sver, &ever); + initMqRspHead(buf, TMQ_MSG_TYPE__POLL_META_RSP, pReq->epoch, pReq->consumerId, sver, ever); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, abuf, len); + tEncodeMqMetaRsp(&encoder, pRsp); + tEncoderClear(&encoder); + + SRpcMsg resp = { .info = pMsg->info, .pCont = buf, .contLen = tlen, .code = 0 }; + + tmsgSendRsp(&resp); + tqDebug("vgId:%d, from consumer:0x%" PRIx64 " (epoch %d) send rsp, res msg type %d, offset type:%d", vgId, + pReq->consumerId, pReq->epoch, pRsp->resMsgType, pRsp->rspOffset.type); + + return 0; +} + +int32_t tqDoSendDataRsp(const SRpcHandleInfo* pRpcHandleInfo, const SMqDataRsp* pRsp, int32_t epoch, int64_t consumerId, + int32_t type, int64_t sver, int64_t ever) { + int32_t len = 0; + int32_t code = 0; + + if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + tEncodeSize(tEncodeMqDataRsp, pRsp, len, code); + } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + tEncodeSize(tEncodeSTaosxRsp, (STaosxRsp*)pRsp, len, code); + } + + if (code < 0) { + return -1; + } + + int32_t tlen = sizeof(SMqRspHead) + len; + void* buf = rpcMallocCont(tlen); + if (buf == NULL) { + return -1; + } + + SMqRspHead* pHead = (SMqRspHead*)buf; + initMqRspHead(pHead, type, epoch, consumerId, sver, ever); + + void* abuf = POINTER_SHIFT(buf, sizeof(SMqRspHead)); + + SEncoder encoder = {0}; + tEncoderInit(&encoder, abuf, len); + + if (type == TMQ_MSG_TYPE__POLL_RSP || type == TMQ_MSG_TYPE__WALINFO_RSP) { + tEncodeMqDataRsp(&encoder, pRsp); + } else if (type == TMQ_MSG_TYPE__TAOSX_RSP) { + tEncodeSTaosxRsp(&encoder, (STaosxRsp*)pRsp); + } + + tEncoderClear(&encoder); + SRpcMsg rsp = { .info = *pRpcHandleInfo, .pCont = buf, .contLen = tlen, .code = 0 }; + + tmsgSendRsp(&rsp); + return 0; +} \ No newline at end of file diff --git a/source/dnode/vnode/src/tsdb/tsdbCache.c b/source/dnode/vnode/src/tsdb/tsdbCache.c index 48d33712845503aa8b69911dff837a81968ea319..3c7edd931baf1fd2f8796368e4ac76bbb398e636 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCache.c +++ b/source/dnode/vnode/src/tsdb/tsdbCache.c @@ -35,7 +35,11 @@ _err: static void tsdbCloseBICache(STsdb *pTsdb) { SLRUCache *pCache = pTsdb->biCache; if (pCache) { + int32_t elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheEraseUnrefEntries(pCache); + elems = taosLRUCacheGetElems(pCache); + tsdbTrace("vgId:%d, elems: %d", TD_VID(pTsdb->pVnode), elems); taosLRUCacheCleanup(pCache); @@ -820,7 +824,12 @@ static int32_t getNextRowFromFS(void *iter, TSDBROW **ppRow, bool *pIgnoreEarlie * &state->blockIdx); */ state->pBlockIdx = taosArraySearch(state->aBlockIdx, state->pBlockIdxExp, tCmprBlockIdx, TD_EQ); - if (!state->pBlockIdx) { /* + if (!state->pBlockIdx) { + tsdbBICacheRelease(state->pTsdb->biCache, state->aBlockIdxHandle); + + state->aBlockIdxHandle = NULL; + state->aBlockIdx = NULL; + /* tsdbDataFReaderClose(state->pDataFReader); *state->pDataFReader = NULL; resetLastBlockLoadInfo(state->pLoadInfo);*/ @@ -1469,11 +1478,14 @@ static int32_t mergeLastRow(tb_uid_t uid, STsdb *pTsdb, bool *dup, SArray **ppCo hasRow = true; - code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + code = updateTSchema(sversion, pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; } - pTSchema = pr->pCurrSchema; int16_t nCol = pTSchema->numOfCols; TSKEY rowTs = TSDBROW_TS(pRow); @@ -1623,11 +1635,14 @@ static int32_t mergeLast(tb_uid_t uid, STsdb *pTsdb, SArray **ppLastArray, SCach hasRow = true; - code = updateTSchema(TSDBROW_SVERSION(pRow), pr, uid); - if (TSDB_CODE_SUCCESS != code) { - goto _err; + int32_t sversion = TSDBROW_SVERSION(pRow); + if (sversion != -1) { + code = updateTSchema(sversion, pr, uid); + if (TSDB_CODE_SUCCESS != code) { + goto _err; + } + pTSchema = pr->pCurrSchema; } - pTSchema = pr->pCurrSchema; int16_t nCol = pTSchema->numOfCols; TSKEY rowTs = TSDBROW_TS(pRow); @@ -1931,6 +1946,7 @@ int32_t tsdbCacheGetBlockIdx(SLRUCache *pCache, SDataFReader *pFileReader, LRUHa taosThreadMutexUnlock(&pTsdb->biMutex); } + tsdbTrace("bi cache:%p, ref", pCache); *handle = h; return code; @@ -1940,6 +1956,7 @@ int32_t tsdbBICacheRelease(SLRUCache *pCache, LRUHandle *h) { int32_t code = 0; taosLRUCacheRelease(pCache, h, false); + tsdbTrace("bi cache:%p, release", pCache); return code; } diff --git a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c index 95981c2f0888dd4b9b53b8726a2de3563bca79b8..d6e819d84ad3d8a6131dcf15f09afdc8ff4b7f64 100644 --- a/source/dnode/vnode/src/tsdb/tsdbCacheRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbCacheRead.c @@ -362,15 +362,6 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 p->ts = pCol->ts; p->colVal = pCol->colVal; singleTableLastTs = pCol->ts; - - // only set value for last row query - if (HASTYPE(pr->type, CACHESCAN_RETRIEVE_LAST_ROW)) { - if (taosArrayGetSize(pTableUidList) == 0) { - taosArrayPush(pTableUidList, &pKeyInfo->uid); - } else { - taosArraySet(pTableUidList, 0, &pKeyInfo->uid); - } - } } } else { SLastCol* p = taosArrayGet(pLastCols, slotId); @@ -417,6 +408,12 @@ int32_t tsdbRetrieveCacheRows(void* pReader, SSDataBlock* pResBlock, const int32 } } + if (taosArrayGetSize(pTableUidList) == 0) { + taosArrayPush(pTableUidList, &pKeyInfo->uid); + } else { + taosArraySet(pTableUidList, 0, &pKeyInfo->uid); + } + tsdbCacheRelease(lruCache, h); } diff --git a/source/dnode/vnode/src/tsdb/tsdbRead.c b/source/dnode/vnode/src/tsdb/tsdbRead.c index 480ae7ea64153df2a47313fd7bb9b6d208b40dc2..362dc51ad5a9efcf9752669729662d1973b613a3 100644 --- a/source/dnode/vnode/src/tsdb/tsdbRead.c +++ b/source/dnode/vnode/src/tsdb/tsdbRead.c @@ -831,6 +831,7 @@ static int32_t doLoadBlockIndex(STsdbReader* pReader, SDataFReader* pFileReader, // this block belongs to a table that is not queried. STableBlockScanInfo* pScanInfo = getTableBlockScanInfo(pReader->status.pTableMap, pBlockIdx->uid, pReader->idStr); if (pScanInfo == NULL) { + tsdbBICacheRelease(pFileReader->pTsdb->biCache, handle); return terrno; } @@ -1880,8 +1881,8 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ return pReader->pSchema; } - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); - if (pReader->pSchema == NULL) { + int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); + if (code != TSDB_CODE_SUCCESS || pReader->pSchema == NULL) { tsdbError("failed to get table schema, uid:%" PRIu64 ", it may have been dropped, ver:-1, %s", uid, pReader->idStr); } @@ -1889,9 +1890,15 @@ static FORCE_INLINE STSchema* getLatestTableSchema(STsdbReader* pReader, uint64_ } static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* pReader, uint64_t uid) { + int32_t code = 0; + // always set the newest schema version in pReader->pSchema if (pReader->pSchema == NULL) { - pReader->pSchema = metaGetTbTSchema(pReader->pTsdb->pVnode->pMeta, uid, -1, 1); + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, -1, &pReader->pSchema); + if (code != TSDB_CODE_SUCCESS) { + terrno = code; + return NULL; + } } if (pReader->pSchema && sversion == pReader->pSchema->version) { @@ -1904,7 +1911,7 @@ static FORCE_INLINE STSchema* doGetSchemaForTSRow(int32_t sversion, STsdbReader* } STSchema* ptr = NULL; - int32_t code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); + code = metaGetTbTSchemaEx(pReader->pTsdb->pVnode->pMeta, pReader->suid, uid, sversion, &ptr); if (code != TSDB_CODE_SUCCESS) { terrno = code; return NULL; @@ -1968,8 +1975,8 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* // DESC: mem -----> imem -----> last block -----> file block if (pReader->order == TSDB_ORDER_ASC) { if (minKey == key) { - init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + init = true; // todo check if pReader->pSchema is null or not + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1979,10 +1986,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -1999,7 +2006,7 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* tsdbRowMergerAdd(&merge, pRow, pSchema); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2013,7 +2020,11 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == k.ts) { init = true; STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + if (pSchema == NULL) { + return terrno; + } + + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2027,10 +2038,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2040,10 +2051,10 @@ static int32_t doMergeBufAndFileRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { if (init) { - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } else { init = true; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2088,13 +2099,13 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, pBlockScanInfo->lastKey = tsLastBlock; return TSDB_CODE_SUCCESS; } else { - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, tsLastBlock, &merge, &pReader->verRange, pReader->idStr); code = tsdbRowMergerGetRow(&merge, &pTSRow); @@ -2112,7 +2123,7 @@ static int32_t doMergeFileBlockAndLastBlock(SLastBlockReader* pLastBlockReader, } } } else { // not merge block data - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2164,7 +2175,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2172,7 +2183,7 @@ static int32_t mergeFileBlockAndLastBlock(STsdbReader* pReader, SLastBlockReader doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); doMergeRowsInLastBlock(pLastBlockReader, pBlockScanInfo, ts, &merge, &pReader->verRange, pReader->idStr); @@ -2215,8 +2226,17 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* int64_t key = hasDataInFileBlock(pBlockData, pDumpInfo) ? pBlockData->aTSKEY[pDumpInfo->rowIndex] : INT64_MIN; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return code; + } + + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return code; + } int64_t minKey = 0; if (ASCENDING_TRAVERSE(pReader->order)) { @@ -2263,7 +2283,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { init = true; TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2274,10 +2294,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2288,15 +2308,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - if (pSchema == NULL) { - return code; - } - - code = tsdbRowMergerInit(&merge, piRow, pSchema); + code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2315,10 +2330,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* return code; } - tsdbRowMerge(&merge, pRow); + tsdbRowMergerAdd(&merge, pRow, pSchema); } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2332,8 +2347,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* } else { if (minKey == k.ts) { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, pRow, pSchema); + code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2347,11 +2361,11 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == ik.ts) { if (init) { - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, piSchema); } else { init = true; - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); - code = tsdbRowMergerInit(&merge, piRow, pSchema); + // STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2366,10 +2380,10 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == tsLast) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); if (init) { - tsdbRowMerge(&merge, &fRow1); + tsdbRowMergerAdd(&merge, &fRow1, NULL); } else { init = true; - code = tsdbRowMergerInit(&merge, &fRow1, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow1, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2380,7 +2394,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (minKey == key) { TSDBROW fRow = tsdbRowFromBlockData(pBlockData, pDumpInfo->rowIndex); if (!init) { - code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -2388,7 +2402,7 @@ static int32_t doMergeMultiLevelRows(STsdbReader* pReader, STableBlockScanInfo* if (merge.pTSchema == NULL) { return code; } - tsdbRowMerge(&merge, &fRow); + tsdbRowMergerAdd(&merge, &fRow, NULL); } doMergeRowsInFileBlocks(pBlockData, pBlockScanInfo, pReader, &merge); } @@ -2573,7 +2587,7 @@ int32_t mergeRowsInFileBlocks(SBlockData* pBlockData, STableBlockScanInfo* pBloc SRow* pTSRow = NULL; SRowMerger merge = {0}; - int32_t code = tsdbRowMergerInit(&merge, &fRow, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, &fRow, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3240,8 +3254,8 @@ static int32_t readRowsCountFromFiles(STsdbReader* pReader) { int32_t code = TSDB_CODE_SUCCESS; while (1) { - bool hasNext = false; - int32_t code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); + bool hasNext = false; + code = filesetIteratorNext(&pReader->status.fileIter, pReader, &hasNext); if (code) { return code; } @@ -3513,8 +3527,8 @@ SVersionRange getQueryVerRange(SVnode* pVnode, SQueryTableDataCond* pCond, int8_ int64_t startVer = (pCond->startVersion == -1) ? 0 : pCond->startVersion; int64_t endVer = 0; - if (pCond->endVersion == - -1) { // user not specified end version, set current maximum version of vnode as the endVersion + if (pCond->endVersion == -1) { + // user not specified end version, set current maximum version of vnode as the endVersion endVer = pVnode->state.applied; } else { endVer = (pCond->endVersion > pVnode->state.applied) ? pVnode->state.applied : pCond->endVersion; @@ -3694,7 +3708,7 @@ int32_t doMergeRowsInBuf(SIterInfo* pIter, uint64_t uid, int64_t ts, SArray* pDe tsdbRowMergerAdd(pMerger, pRow, pTSchema); } else { // column format - tsdbRowMerge(pMerger, pRow); + tsdbRowMergerAdd(pMerger, pRow, NULL); } } @@ -3710,7 +3724,7 @@ static int32_t doMergeRowsInFileBlockImpl(SBlockData* pBlockData, int32_t rowInd } TSDBROW fRow = tsdbRowFromBlockData(pBlockData, rowIndex); - tsdbRowMerge(pMerger, &fRow); + tsdbRowMergerAdd(pMerger, &fRow, NULL); rowIndex += step; } @@ -3788,7 +3802,7 @@ int32_t doMergeRowsInLastBlock(SLastBlockReader* pLastBlockReader, STableBlockSc int64_t next1 = getCurrentKeyInLastBlock(pLastBlockReader); if (next1 == ts) { TSDBROW fRow1 = tMergeTreeGetRow(&pLastBlockReader->mergeTree); - tsdbRowMerge(pMerger, &fRow1); + tsdbRowMergerAdd(pMerger, &fRow1, NULL); } else { tsdbTrace("uid:%" PRIu64 " last del index:%d, del range:%d, lastKeyInStt:%" PRId64 ", %s", pScanInfo->uid, pScanInfo->lastBlockDelIndex, (int32_t)taosArrayGetSize(pScanInfo->delSkyline), pScanInfo->lastKeyInStt, @@ -3840,11 +3854,8 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, return terrno; } - if (pReader->pSchema == NULL) { - pReader->pSchema = pTSchema; - } - - code = tsdbRowMergerInit2(&merge, pReader->pSchema, ¤t, pTSchema); + STSchema* ps = (pReader->pSchema != NULL)? pReader->pSchema:pTSchema; + code = tsdbRowMergerInit(&merge, ps, ¤t, pTSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3856,12 +3867,12 @@ int32_t doMergeMemTableMultiRows(TSDBROW* pRow, uint64_t uid, SIterInfo* pIter, tsdbRowMergerAdd(&merge, pNextRow, pTSchema1); } else { // let's merge rows in file block - code = tsdbRowMergerInit(&merge, ¤t, pReader->pSchema); + code = tsdbRowMergerInit(&merge, NULL, ¤t, pReader->pSchema); if (code != TSDB_CODE_SUCCESS) { return code; } - tsdbRowMerge(&merge, pNextRow); + tsdbRowMergerAdd(&merge, pNextRow, NULL); } code = doMergeRowsInBuf(pIter, uid, TSDBROW_TS(¤t), pDelList, &merge, pReader); @@ -3885,14 +3896,20 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p SRow** pTSRow) { SRowMerger merge = {0}; - TSDBKEY k = TSDBROW_KEY(pRow); - TSDBKEY ik = TSDBROW_KEY(piRow); + TSDBKEY k = TSDBROW_KEY(pRow); + TSDBKEY ik = TSDBROW_KEY(piRow); + STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); + if (pSchema == NULL) { + return terrno; + } - if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + STSchema* piSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(piRow), pReader, pBlockScanInfo->uid); + if (piSchema == NULL) { + return terrno; + } - int32_t code = tsdbRowMergerInit2(&merge, pSchema, piRow, piSchema); + if (ASCENDING_TRAVERSE(pReader->order)) { // ascending order imem --> mem + int32_t code = tsdbRowMergerInit(&merge, pSchema, piRow, piSchema); if (code != TSDB_CODE_SUCCESS) { return code; } @@ -3911,9 +3928,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p } } else { - STSchema* pSchema = doGetSchemaForTSRow(TSDBROW_SVERSION(pRow), pReader, pBlockScanInfo->uid); - - int32_t code = tsdbRowMergerInit(&merge, pRow, pSchema); + int32_t code = tsdbRowMergerInit(&merge, NULL, pRow, pSchema); if (code != TSDB_CODE_SUCCESS || merge.pTSchema == NULL) { return code; } @@ -3924,7 +3939,7 @@ int32_t doMergeMemIMemRows(TSDBROW* pRow, TSDBROW* piRow, STableBlockScanInfo* p return code; } - tsdbRowMerge(&merge, piRow); + tsdbRowMergerAdd(&merge, piRow, piSchema); code = doMergeRowsInBuf(&pBlockScanInfo->iiter, pBlockScanInfo->uid, ik.ts, pBlockScanInfo->delSkyline, &merge, pReader); if (code != TSDB_CODE_SUCCESS) { @@ -4000,10 +4015,11 @@ int32_t doAppendRowFromTSRow(SSDataBlock* pBlock, STsdbReader* pReader, SRow* pT int64_t uid = pScanInfo->uid; int32_t code = TSDB_CODE_SUCCESS; - int32_t numOfCols = (int32_t)taosArrayGetSize(pBlock->pDataBlock); - SBlockLoadSuppInfo* pSupInfo = &pReader->suppInfo; STSchema* pSchema = doGetSchemaForTSRow(pTSRow->sver, pReader, uid); + if (pSchema == NULL) { + return terrno; + } SColVal colVal = {0}; int32_t i = 0, j = 0; @@ -5187,8 +5203,6 @@ int64_t tsdbGetNumOfRowsInMemTable(STsdbReader* pReader) { } int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int64_t* suid) { - int32_t sversion = 1; - SMetaReader mr = {0}; metaReaderInit(&mr, pVnode->pMeta, 0); int32_t code = metaGetTableEntryByUidCache(&mr, uid); @@ -5200,6 +5214,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 *suid = 0; + // only child table and ordinary table is allowed, super table is not allowed. if (mr.me.type == TSDB_CHILD_TABLE) { tDecoderClear(&mr.coder); *suid = mr.me.ctbEntry.suid; @@ -5209,9 +5224,7 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 metaReaderClear(&mr); return terrno; } - sversion = mr.me.stbEntry.schemaRow.version; - } else if (mr.me.type == TSDB_NORMAL_TABLE) { - sversion = mr.me.ntbEntry.schemaRow.version; + } else if (mr.me.type == TSDB_NORMAL_TABLE) { // do nothing } else { terrno = TSDB_CODE_INVALID_PARA; metaReaderClear(&mr); @@ -5219,9 +5232,10 @@ int32_t tsdbGetTableSchema(SVnode* pVnode, int64_t uid, STSchema** pSchema, int6 } metaReaderClear(&mr); - *pSchema = metaGetTbTSchema(pVnode->pMeta, uid, sversion, 1); - return TSDB_CODE_SUCCESS; + // get the newest table schema version + code = metaGetTbTSchemaEx(pVnode->pMeta, *suid, uid, -1, pSchema); + return code; } int32_t tsdbTakeReadSnap(STsdbReader* pReader, _query_reseek_func_t reseek, STsdbReadSnap** ppSnap) { diff --git a/source/dnode/vnode/src/tsdb/tsdbUtil.c b/source/dnode/vnode/src/tsdb/tsdbUtil.c index c323ae1532a3f8fe97fdcddaae020723bda88c94..8e778da877af5affc40e81e06c3ba7048f7ae439 100644 --- a/source/dnode/vnode/src/tsdb/tsdbUtil.c +++ b/source/dnode/vnode/src/tsdb/tsdbUtil.c @@ -638,13 +638,17 @@ SColVal *tsdbRowIterNext(STSDBRowIter *pIter) { // SRowMerger ====================================================== -int32_t tsdbRowMergerInit2(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { +int32_t tsdbRowMergerInit(SRowMerger *pMerger, STSchema *pResTSchema, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); SColVal *pColVal = &(SColVal){0}; STColumn *pTColumn; int32_t iCol, jCol = 0; + if (NULL == pResTSchema) { + pResTSchema = pTSchema; + } + pMerger->pTSchema = pResTSchema; pMerger->version = key.version; @@ -712,6 +716,9 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) STColumn *pTColumn; int32_t iCol, jCol = 1; + if (NULL == pTSchema) { + pTSchema = pMerger->pTSchema; + } ASSERT(((SColVal *)pMerger->pArray->pData)->value.val == key.ts); for (iCol = 1; iCol < pMerger->pTSchema->numOfCols && jCol < pTSchema->numOfCols; ++iCol) { @@ -727,23 +734,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) tsdbRowGetColVal(pRow, pTSchema, jCol++, pColVal); if (key.version > pMerger->version) { -#if 0 - if (!COL_VAL_IS_NONE(pColVal)) { - if ((!COL_VAL_IS_NULL(pColVal)) && IS_VAR_DATA_TYPE(pColVal->type)) { - SColVal *tColVal = taosArrayGet(pMerger->pArray, iCol); - code = tRealloc(&tColVal->value.pData, pColVal->value.nData); - if (code) return code; - - tColVal->value.nData = pColVal->value.nData; - if (pColVal->value.nData) { - memcpy(tColVal->value.pData, pColVal->value.pData, pColVal->value.nData); - } - tColVal->flag = 0; - } else { - taosArraySet(pMerger->pArray, iCol, pColVal); - } - } -#endif if (!COL_VAL_IS_NONE(pColVal)) { if (IS_VAR_DATA_TYPE(pColVal->type)) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -758,7 +748,6 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pTColVal->flag = 0; } else { tFree(pTColVal->value.pData); - pTColVal->value.pData = NULL; taosArraySet(pMerger->pArray, iCol, pColVal); } } else { @@ -789,7 +778,7 @@ int32_t tsdbRowMergerAdd(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) pMerger->version = key.version; return code; } - +/* int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -840,7 +829,7 @@ int32_t tsdbRowMergerInit(SRowMerger *pMerger, TSDBROW *pRow, STSchema *pTSchema _exit: return code; } - +*/ void tsdbRowMergerClear(SRowMerger *pMerger) { for (int32_t iCol = 1; iCol < pMerger->pTSchema->numOfCols; iCol++) { SColVal *pTColVal = taosArrayGet(pMerger->pArray, iCol); @@ -851,7 +840,7 @@ void tsdbRowMergerClear(SRowMerger *pMerger) { taosArrayDestroy(pMerger->pArray); } - +/* int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { int32_t code = 0; TSDBKEY key = TSDBROW_KEY(pRow); @@ -916,7 +905,7 @@ int32_t tsdbRowMerge(SRowMerger *pMerger, TSDBROW *pRow) { _exit: return code; } - +*/ int32_t tsdbRowMergerGetRow(SRowMerger *pMerger, SRow **ppRow) { return tRowBuild(pMerger->pArray, pMerger->pTSchema, ppRow); } @@ -1270,6 +1259,11 @@ void tBlockDataReset(SBlockData *pBlockData) { pBlockData->suid = 0; pBlockData->uid = 0; pBlockData->nRow = 0; + for (int32_t i = 0; i < pBlockData->nColData; i++) { + tColDataDestroy(&pBlockData->aColData[i]); + } + pBlockData->nColData = 0; + taosMemoryFreeClear(pBlockData->aColData); } void tBlockDataClear(SBlockData *pBlockData) { diff --git a/source/dnode/vnode/src/vnd/vnodeCfg.c b/source/dnode/vnode/src/vnd/vnodeCfg.c index c326c8bfacbd897fc15527edaef5cd8e77c40f5f..511ba9cc2400300991504aeb435cb7ec6a2e6475 100644 --- a/source/dnode/vnode/src/vnd/vnodeCfg.c +++ b/source/dnode/vnode/src/vnd/vnodeCfg.c @@ -57,6 +57,28 @@ int vnodeCheckCfg(const SVnodeCfg *pCfg) { return 0; } +const char* vnodeRoleToStr(ESyncRole role) { + switch (role) { + case TAOS_SYNC_ROLE_VOTER: + return "true"; + case TAOS_SYNC_ROLE_LEARNER: + return "false"; + default: + return "unknown"; + } +} + +const ESyncRole vnodeStrToRole(char* str) { + if(strcmp(str, "true") == 0){ + return TAOS_SYNC_ROLE_VOTER; + } + if(strcmp(str, "false") == 0){ + return TAOS_SYNC_ROLE_LEARNER; + } + + return TAOS_SYNC_ROLE_ERROR; +} + int vnodeEncodeConfig(const void *pObj, SJson *pJson) { const SVnodeCfg *pCfg = (SVnodeCfg *)pObj; @@ -128,9 +150,9 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "syncCfg.nodeInfo", nodeInfo) < 0) return -1; - vDebug("vgId:%d, encode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.myIndex); - for (int i = 0; i < pCfg->syncCfg.replicaNum; ++i) { + vDebug("vgId:%d, encode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); + for (int i = 0; i < pCfg->syncCfg.totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); SNodeInfo *pNode = (SNodeInfo *)&pCfg->syncCfg.nodeInfo[i]; if (info == NULL) return -1; @@ -138,6 +160,7 @@ int vnodeEncodeConfig(const void *pObj, SJson *pJson) { if (tjsonAddStringToObject(info, "nodeFqdn", pNode->nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pNode->nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pNode->clusterId) < 0) return -1; + if (tjsonAddStringToObject(info, "isReplica", vnodeRoleToStr(pNode->nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; vDebug("vgId:%d, encode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); @@ -251,10 +274,10 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "syncCfg.nodeInfo"); int arraySize = tjsonGetArraySize(nodeInfo); - if (arraySize != pCfg->syncCfg.replicaNum) return -1; + pCfg->syncCfg.totalReplicaNum = arraySize; - vDebug("vgId:%d, decode config, replicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, - pCfg->syncCfg.myIndex); + vDebug("vgId:%d, decode config, replicas:%d totalReplicas:%d selfIndex:%d", pCfg->vgId, pCfg->syncCfg.replicaNum, + pCfg->syncCfg.totalReplicaNum, pCfg->syncCfg.myIndex); for (int i = 0; i < arraySize; ++i) { SJson *info = tjsonGetArrayItem(nodeInfo, i); SNodeInfo *pNode = &pCfg->syncCfg.nodeInfo[i]; @@ -266,6 +289,15 @@ int vnodeDecodeConfig(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "clusterId", pNode->clusterId, code); if (code < 0) return -1; + char role[10] = {0}; + code = tjsonGetStringValue(info, "isReplica", role); + if (code < 0) return -1; + if(strlen(role) != 0){ + pNode->nodeRole = vnodeStrToRole(role); + } + else{ + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; + } vDebug("vgId:%d, decode config, replica:%d ep:%s:%u dnode:%d", pCfg->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); } diff --git a/source/dnode/vnode/src/vnd/vnodeCommit.c b/source/dnode/vnode/src/vnd/vnodeCommit.c index 09f1ca78776784673a92f2a23d65314bf14b360c..847125018c5f15baa660dc2f53677e17e18e0d7f 100644 --- a/source/dnode/vnode/src/vnd/vnodeCommit.c +++ b/source/dnode/vnode/src/vnd/vnodeCommit.c @@ -143,22 +143,13 @@ _exit: return code; } -void vnodeUpdCommitSched(SVnode *pVnode) { - int64_t randNum = taosRand(); - pVnode->commitSched.commitMs = taosGetMonoTimestampMs(); - pVnode->commitSched.maxWaitMs = tsVndCommitMaxIntervalMs + (randNum % tsVndCommitMaxIntervalMs); -} - -int vnodeShouldCommit(SVnode *pVnode) { - SVCommitSched *pSched = &pVnode->commitSched; - int64_t nowMs = taosGetMonoTimestampMs(); +int vnodeShouldCommit(SVnode *pVnode, bool atExit) { bool diskAvail = osDataSpaceAvailable(); bool needCommit = false; taosThreadMutexLock(&pVnode->mutex); if (pVnode->inUse && diskAvail) { - needCommit = - ((pVnode->inUse->size > pVnode->inUse->node.size) && (pSched->commitMs + SYNC_VND_COMMIT_MIN_MS < nowMs)); + needCommit = (pVnode->inUse->size > pVnode->inUse->node.size) || (pVnode->inUse->size > 0 && atExit); } taosThreadMutexUnlock(&pVnode->mutex); return needCommit; @@ -430,8 +421,6 @@ static int vnodeCommitImpl(SCommitInfo *pInfo) { vInfo("vgId:%d, start to commit, commitId:%" PRId64 " version:%" PRId64 " term: %" PRId64, TD_VID(pVnode), pInfo->info.state.commitID, pInfo->info.state.committed, pInfo->info.state.commitTerm); - vnodeUpdCommitSched(pVnode); - // persist wal before starting if (walPersist(pVnode->pWal) < 0) { vError("vgId:%d, failed to persist wal since %s", TD_VID(pVnode), terrstr()); diff --git a/source/dnode/vnode/src/vnd/vnodeOpen.c b/source/dnode/vnode/src/vnd/vnodeOpen.c index c7d155be0d50600e419940b03297d2bad6d29528..7d41edfdd95913eb45dd7581e8f105252b949845 100644 --- a/source/dnode/vnode/src/vnd/vnodeOpen.c +++ b/source/dnode/vnode/src/vnd/vnodeOpen.c @@ -76,20 +76,42 @@ int32_t vnodeAlterReplica(const char *path, SAlterVnodeReplicaReq *pReq, STfs *p } SSyncCfg *pCfg = &info.config.syncCfg; - pCfg->myIndex = pReq->selfIndex; - pCfg->replicaNum = pReq->replica; + + pCfg->replicaNum = 0; + pCfg->totalReplicaNum = 0; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); - vInfo("vgId:%d, save config while alter, replicas:%d selfIndex:%d", pReq->vgId, pCfg->replicaNum, pCfg->myIndex); for (int i = 0; i < pReq->replica; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; pNode->nodeId = pReq->replicas[i].id; pNode->nodePort = pReq->replicas[i].port; tstrncpy(pNode->nodeFqdn, pReq->replicas[i].fqdn, sizeof(pNode->nodeFqdn)); + pNode->nodeRole = TAOS_SYNC_ROLE_VOTER; (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + pCfg->replicaNum++; + } + if(pReq->selfIndex != -1){ + pCfg->myIndex = pReq->selfIndex; + } + for (int i = pCfg->replicaNum; i < pReq->replica + pReq->learnerReplica; ++i) { + SNodeInfo *pNode = &pCfg->nodeInfo[i]; + pNode->nodeId = pReq->learnerReplicas[pCfg->totalReplicaNum].id; + pNode->nodePort = pReq->learnerReplicas[pCfg->totalReplicaNum].port; + pNode->nodeRole = TAOS_SYNC_ROLE_LEARNER; + tstrncpy(pNode->nodeFqdn, pReq->learnerReplicas[pCfg->totalReplicaNum].fqdn, sizeof(pNode->nodeFqdn)); + (void)tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort); + vInfo("vgId:%d, replica:%d ep:%s:%u dnode:%d", pReq->vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId); + pCfg->totalReplicaNum++; + } + pCfg->totalReplicaNum += pReq->replica; + if(pReq->learnerSelfIndex != -1){ + pCfg->myIndex = pReq->replica + pReq->learnerSelfIndex; } + vInfo("vgId:%d, save config while alter, replicas:%d totalReplicas:%d selfIndex:%d", + pReq->vgId, pCfg->replicaNum, pCfg->totalReplicaNum, pCfg->myIndex); + info.config.syncCfg = *pCfg; ret = vnodeSaveInfo(dir, &info); if (ret < 0) { @@ -181,6 +203,7 @@ int32_t vnodeAlterHashRange(const char *srcPath, const char *dstPath, SAlterVnod SSyncCfg *pCfg = &info.config.syncCfg; pCfg->myIndex = 0; pCfg->replicaNum = 1; + pCfg->totalReplicaNum = 1; memset(&pCfg->nodeInfo, 0, sizeof(pCfg->nodeInfo)); vInfo("vgId:%d, alter vnode replicas to 1", pReq->srcVgId); @@ -248,7 +271,7 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { // save vnode info on dnode ep changed bool updated = false; SSyncCfg *pCfg = &info.config.syncCfg; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { updated = true; @@ -286,8 +309,6 @@ SVnode *vnodeOpen(const char *path, STfs *pTfs, SMsgCb msgCb) { taosThreadMutexInit(&pVnode->mutex, NULL); taosThreadCondInit(&pVnode->poolNotEmpty, NULL); - vnodeUpdCommitSched(pVnode); - int8_t rollback = vnodeShouldRollback(pVnode); // open buffer pool @@ -406,6 +427,14 @@ void vnodeClose(SVnode *pVnode) { // start the sync timer after the queue is ready int32_t vnodeStart(SVnode *pVnode) { return vnodeSyncStart(pVnode); } +int32_t vnodeIsCatchUp(SVnode *pVnode){ + return syncIsCatchUp(pVnode->sync); +} + +ESyncRole vnodeGetRole(SVnode *pVnode){ + return syncGetRole(pVnode->sync); +} + void vnodeStop(SVnode *pVnode) {} int64_t vnodeGetSyncHandle(SVnode *pVnode) { return pVnode->sync; } diff --git a/source/dnode/vnode/src/vnd/vnodeSvr.c b/source/dnode/vnode/src/vnd/vnodeSvr.c index 02d361ccdb0715f539dd2572e730cd1fd89bff81..dd576392f0c93b6ce5f2cff2d7d2ed8e9ed3c488 100644 --- a/source/dnode/vnode/src/vnd/vnodeSvr.c +++ b/source/dnode/vnode/src/vnd/vnodeSvr.c @@ -389,6 +389,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp goto _err; } break; + case TDMT_VND_TMQ_SEEK_TO_OFFSET: + if (tqProcessSeekReq(pVnode->pTq, version, pReq, pMsg->contLen - sizeof(SMsgHead)) < 0) { + goto _err; + } + break; case TDMT_VND_TMQ_ADD_CHECKINFO: if (tqProcessAddCheckInfoReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; @@ -400,7 +405,7 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp } break; case TDMT_STREAM_TASK_DEPLOY: { - if (tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { + if (pVnode->restored && tqProcessTaskDeployReq(pVnode->pTq, version, pReq, len) < 0) { goto _err; } } break; @@ -447,13 +452,11 @@ int32_t vnodeProcessWriteMsg(SVnode *pVnode, SRpcMsg *pMsg, int64_t version, SRp walApplyVer(pVnode->pWal, version); - /*vInfo("vgId:%d, push msg begin", pVnode->config.vgId);*/ if (tqPushMsg(pVnode->pTq, pMsg->pCont, pMsg->contLen, pMsg->msgType, version) < 0) { /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ vError("vgId:%d, failed to push msg to TQ since %s", TD_VID(pVnode), tstrerror(terrno)); return -1; } - /*vInfo("vgId:%d, push msg end", pVnode->config.vgId);*/ // commit if need if (needCommit) { @@ -541,12 +544,12 @@ int32_t vnodeProcessFetchMsg(SVnode *pVnode, SRpcMsg *pMsg, SQueueInfo *pInfo) { return vnodeGetBatchMeta(pVnode, pMsg); case TDMT_VND_TMQ_CONSUME: return tqProcessPollReq(pVnode->pTq, pMsg); + case TDMT_VND_TMQ_VG_WALINFO: + return tqProcessVgWalInfoReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_RUN: return tqProcessTaskRunReq(pVnode->pTq, pMsg); -#if 1 case TDMT_STREAM_TASK_DISPATCH: return tqProcessTaskDispatchReq(pVnode->pTq, pMsg, true); -#endif case TDMT_STREAM_TASK_CHECK: return tqProcessStreamTaskCheckReq(pVnode->pTq, pMsg); case TDMT_STREAM_TASK_DISPATCH_RSP: @@ -1451,7 +1454,8 @@ int32_t vnodeProcessCreateTSma(SVnode *pVnode, void *pCont, uint32_t contLen) { } static int32_t vnodeProcessAlterConfirmReq(SVnode *pVnode, int64_t version, void *pReq, int32_t len, SRpcMsg *pRsp) { - vInfo("vgId:%d, alter replica confim msg is processed", TD_VID(pVnode)); + vInfo("vgId:%d, vnode management handle msgType:alter-confirm, alter replica confim msg is processed", + TD_VID(pVnode)); pRsp->msgType = TDMT_VND_ALTER_CONFIRM_RSP; pRsp->code = TSDB_CODE_SUCCESS; pRsp->pCont = NULL; diff --git a/source/dnode/vnode/src/vnd/vnodeSync.c b/source/dnode/vnode/src/vnd/vnodeSync.c index d681f5b65e2228e6d2b16c88688ba672ad26564d..ba73c5fd4cfd1023bab861218a526898b342e072 100644 --- a/source/dnode/vnode/src/vnd/vnodeSync.c +++ b/source/dnode/vnode/src/vnd/vnodeSync.c @@ -112,9 +112,6 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak pVnode->blocked = true; pVnode->blockSec = taosGetTimestampSec(); pVnode->blockSeq = seq; -#if 0 - pVnode->blockInfo = pMsg->info; -#endif } taosThreadMutexUnlock(&pVnode->lock); @@ -129,8 +126,8 @@ static int32_t inline vnodeProposeMsg(SVnode *pVnode, SRpcMsg *pMsg, bool isWeak return code; } -void vnodeProposeCommitOnNeed(SVnode *pVnode) { - if (!vnodeShouldCommit(pVnode)) { +void vnodeProposeCommitOnNeed(SVnode *pVnode, bool atExit) { + if (!vnodeShouldCommit(pVnode, atExit)) { return; } @@ -145,18 +142,18 @@ void vnodeProposeCommitOnNeed(SVnode *pVnode) { rpcMsg.pCont = pHead; rpcMsg.info.noResp = 1; + vInfo("vgId:%d, propose vnode commit", pVnode->config.vgId); bool isWeak = false; - if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { - vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); - goto _out; - } - - vInfo("vgId:%d, proposed vnode commit", pVnode->config.vgId); -_out: - vnodeUpdCommitSched(pVnode); - rpcFreeCont(rpcMsg.pCont); - rpcMsg.pCont = NULL; + if (!atExit) { + if (vnodeProposeMsg(pVnode, &rpcMsg, isWeak) < 0) { + vTrace("vgId:%d, failed to propose vnode commit since %s", pVnode->config.vgId, terrstr()); + } + rpcFreeCont(rpcMsg.pCont); + rpcMsg.pCont = NULL; + } else { + tmsgPutToQueue(&pVnode->msgCb, WRITE_QUEUE, &rpcMsg); + } } #if BATCH_ENABLE @@ -236,7 +233,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - vnodeProposeCommitOnNeed(pVnode); + bool atExit = false; + vnodeProposeCommitOnNeed(pVnode, atExit); code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { @@ -288,7 +286,8 @@ void vnodeProposeWriteMsg(SQueueInfo *pInfo, STaosQall *qall, int32_t numOfMsgs) continue; } - vnodeProposeCommitOnNeed(pVnode); + bool atExit = false; + vnodeProposeCommitOnNeed(pVnode, atExit); code = vnodePreProcessWriteMsg(pVnode, pMsg); if (code != 0) { @@ -548,7 +547,10 @@ static void vnodeRestoreFinish(const SSyncFSM *pFsm, const SyncIndex commitIdx) walApplyVer(pVnode->pWal, commitIdx); pVnode->restored = true; - vInfo("vgId:%d, sync restore finished", pVnode->config.vgId); + vInfo("vgId:%d, sync restore finished, start to restore stream tasks by replay wal", pVnode->config.vgId); + + // start to restore all stream tasks + tqStartStreamTasks(pVnode->pTq); } static void vnodeBecomeFollower(const SSyncFSM *pFsm) { @@ -564,6 +566,19 @@ static void vnodeBecomeFollower(const SSyncFSM *pFsm) { taosThreadMutexUnlock(&pVnode->lock); } +static void vnodeBecomeLearner(const SSyncFSM *pFsm) { + SVnode *pVnode = pFsm->data; + vInfo("vgId:%d, become learner", pVnode->config.vgId); + + taosThreadMutexLock(&pVnode->lock); + if (pVnode->blocked) { + pVnode->blocked = false; + vDebug("vgId:%d, become learner and post block", pVnode->config.vgId); + tsem_post(&pVnode->syncSem); + } + taosThreadMutexUnlock(&pVnode->lock); +} + static void vnodeBecomeLeader(const SSyncFSM *pFsm) { SVnode *pVnode = pFsm->data; vDebug("vgId:%d, become leader", pVnode->config.vgId); @@ -605,6 +620,7 @@ static SSyncFSM *vnodeSyncMakeFsm(SVnode *pVnode) { pFsm->FpApplyQueueItems = vnodeApplyQueueItems; pFsm->FpBecomeLeaderCb = vnodeBecomeLeader; pFsm->FpBecomeFollowerCb = vnodeBecomeFollower; + pFsm->FpBecomeLearnerCb = vnodeBecomeLearner; pFsm->FpReConfigCb = NULL; pFsm->FpSnapshotStartRead = vnodeSnapshotStartRead; pFsm->FpSnapshotStopRead = vnodeSnapshotStopRead; @@ -637,7 +653,7 @@ int32_t vnodeSyncOpen(SVnode *pVnode, char *path) { SSyncCfg *pCfg = &syncInfo.syncCfg; vInfo("vgId:%d, start to open sync, replica:%d selfIndex:%d", pVnode->config.vgId, pCfg->replicaNum, pCfg->myIndex); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo *pNode = &pCfg->nodeInfo[i]; vInfo("vgId:%d, index:%d ep:%s:%u dnode:%d cluster:%" PRId64, pVnode->config.vgId, i, pNode->nodeFqdn, pNode->nodePort, pNode->nodeId, pNode->clusterId); diff --git a/source/libs/catalog/CMakeLists.txt b/source/libs/catalog/CMakeLists.txt index 632034d6b60ef0454ae53ba799534934608b9b07..bc56ff4c7f766862f2f70057d09af56d2a9f8895 100644 --- a/source/libs/catalog/CMakeLists.txt +++ b/source/libs/catalog/CMakeLists.txt @@ -8,9 +8,9 @@ target_include_directories( target_link_libraries( catalog - PRIVATE os util transport qcom + PRIVATE os util transport qcom nodes ) if(${BUILD_TEST}) ADD_SUBDIRECTORY(test) -endif(${BUILD_TEST}) \ No newline at end of file +endif(${BUILD_TEST}) diff --git a/source/libs/catalog/inc/catalogInt.h b/source/libs/catalog/inc/catalogInt.h index c4e1fd3078b18d74df3e89a4a9ea2d82f26cef0e..85a130d29373a390a4ca5281985879af9969e08a 100644 --- a/source/libs/catalog/inc/catalogInt.h +++ b/source/libs/catalog/inc/catalogInt.h @@ -66,8 +66,8 @@ typedef enum { } CTG_CACHE_ITEM; #define CTG_CI_FLAG_LEVEL_GLOBAL (1) -#define CTG_CI_FLAG_LEVEL_CLUSTER (1<<1) -#define CTG_CI_FLAG_LEVEL_DB (1<<2) +#define CTG_CI_FLAG_LEVEL_CLUSTER (1 << 1) +#define CTG_CI_FLAG_LEVEL_DB (1 << 2) enum { CTG_READ = 1, @@ -132,6 +132,19 @@ typedef struct SCtgCacheStat { uint64_t cacheNHit[CTG_CI_MAX_VALUE]; } SCtgCacheStat; +typedef struct SCtgAuthReq { + SRequestConnInfo* pConn; + SUserAuthInfo* pRawReq; + SGetUserAuthRsp authInfo; + AUTH_TYPE singleType; + bool onlyCache; +} SCtgAuthReq; + +typedef struct SCtgAuthRsp { + SUserAuthRes* pRawRes; + bool metaNotExists; +} SCtgAuthRsp; + typedef struct SCtgTbCacheInfo { bool inCache; uint64_t dbId; @@ -224,13 +237,13 @@ typedef struct SCtgVgCache { } SCtgVgCache; typedef struct SCtgDBCache { - SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads - uint64_t dbId; - int8_t deleted; - SCtgVgCache vgCache; - SHashObj* tbCache; // key:tbname, value:SCtgTbCache - SHashObj* stbCache; // key:suid, value:char* - uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; + SRWLatch dbLock; // RC between destroy tbCache/stbCache and all reads + uint64_t dbId; + int8_t deleted; + SCtgVgCache vgCache; + SHashObj* tbCache; // key:tbname, value:SCtgTbCache + SHashObj* stbCache; // key:suid, value:char* + uint64_t dbCacheNum[CTG_CI_MAX_VALUE]; } SCtgDBCache; typedef struct SCtgRentSlot { @@ -248,12 +261,8 @@ typedef struct SCtgRentMgmt { } SCtgRentMgmt; typedef struct SCtgUserAuth { - int32_t version; - SRWLatch lock; - bool superUser; - SHashObj* createdDbs; - SHashObj* readDbs; - SHashObj* writeDbs; + SRWLatch lock; + SGetUserAuthRsp userAuth; } SCtgUserAuth; typedef struct SCatalog { @@ -505,11 +514,11 @@ typedef struct SCtgOperation { } SCtgOperation; typedef struct SCtgCacheItemInfo { - char *name; + char* name; int32_t flag; } SCtgCacheItemInfo; -#define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) +#define CTG_AUTH_READ(_t) ((_t) == AUTH_TYPE_READ || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_AUTH_WRITE(_t) ((_t) == AUTH_TYPE_WRITE || (_t) == AUTH_TYPE_READ_OR_WRITE) #define CTG_QUEUE_INC() atomic_add_fetch_64(&gCtgMgmt.queue.qRemainNum, 1) @@ -519,87 +528,89 @@ typedef struct SCtgCacheItemInfo { #define CTG_STAT_DEC(_item, _n) atomic_sub_fetch_64(&(_item), _n) #define CTG_STAT_GET(_item) atomic_load_64(&(_item)) -#define CTG_DB_NUM_INC(_item) dbCache->dbCacheNum[_item] += 1 -#define CTG_DB_NUM_DEC(_item) dbCache->dbCacheNum[_item] -= 1 -#define CTG_DB_NUM_SET(_item) dbCache->dbCacheNum[_item] = 1 +#define CTG_DB_NUM_INC(_item) dbCache->dbCacheNum[_item] += 1 +#define CTG_DB_NUM_DEC(_item) dbCache->dbCacheNum[_item] -= 1 +#define CTG_DB_NUM_SET(_item) dbCache->dbCacheNum[_item] = 1 #define CTG_DB_NUM_RESET(_item) dbCache->dbCacheNum[_item] = 0 -#define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) -#define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) -#define CTG_STAT_NUM_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) -#define CTG_STAT_NUM_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) -#define CTG_STAT_HIT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) -#define CTG_STAT_HIT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) +#define CTG_STAT_API_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.api.item, n)) +#define CTG_STAT_RT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.runtime.item, n)) +#define CTG_STAT_NUM_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) +#define CTG_STAT_NUM_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheNum[item], n)) +#define CTG_STAT_HIT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) +#define CTG_STAT_HIT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheHit[item], n)) #define CTG_STAT_NHIT_INC(item, n) (CTG_STAT_INC(gCtgMgmt.statInfo.cache.cacheNHit[item], n)) #define CTG_STAT_NHIT_DEC(item, n) (CTG_STAT_DEC(gCtgMgmt.statInfo.cache.cacheNHit[item], n)) -#define CTG_CACHE_NUM_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNum[item], n)) -#define CTG_CACHE_NUM_DEC(item, n) (CTG_STAT_DEC(pCtg->cacheStat.cacheNum[item], n)) -#define CTG_CACHE_HIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheHit[item], n)) +#define CTG_CACHE_NUM_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNum[item], n)) +#define CTG_CACHE_NUM_DEC(item, n) (CTG_STAT_DEC(pCtg->cacheStat.cacheNum[item], n)) +#define CTG_CACHE_HIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheHit[item], n)) #define CTG_CACHE_NHIT_INC(item, n) (CTG_STAT_INC(pCtg->cacheStat.cacheNHit[item], n)) -#define CTG_META_NUM_INC(type) do { \ - switch (type) { \ - case TSDB_SUPER_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_STABLE_META); \ - break; \ - case TSDB_CHILD_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_CTABLE_META); \ - break; \ - case TSDB_NORMAL_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_NTABLE_META); \ - break; \ - case TSDB_SYSTEM_TABLE: \ - CTG_DB_NUM_INC(CTG_CI_SYSTABLE_META); \ - break; \ - default: \ - CTG_DB_NUM_INC(CTG_CI_OTHERTABLE_META); \ - break; \ - } \ -} while (0) - -#define CTG_META_NUM_DEC(type) do { \ - switch (type) { \ - case TSDB_SUPER_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_STABLE_META); \ - break; \ - case TSDB_CHILD_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_CTABLE_META); \ - break; \ - case TSDB_NORMAL_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_NTABLE_META); \ - break; \ - case TSDB_SYSTEM_TABLE: \ - CTG_DB_NUM_DEC(CTG_CI_SYSTABLE_META); \ - break; \ - default: \ - CTG_DB_NUM_DEC(CTG_CI_OTHERTABLE_META); \ - break; \ - } \ -} while (0) - -#define CTG_META_HIT_INC(type) do { \ - switch (type) { \ - case TSDB_SUPER_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_STABLE_META, 1); \ - break; \ - case TSDB_CHILD_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_CTABLE_META, 1); \ - break; \ - case TSDB_NORMAL_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_NTABLE_META, 1); \ - break; \ - case TSDB_SYSTEM_TABLE: \ - CTG_CACHE_HIT_INC(CTG_CI_SYSTABLE_META, 1); \ - break; \ - default: \ - CTG_CACHE_HIT_INC(CTG_CI_OTHERTABLE_META, 1); \ - break; \ - } \ -} while (0) +#define CTG_META_NUM_INC(type) \ + do { \ + switch (type) { \ + case TSDB_SUPER_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_STABLE_META); \ + break; \ + case TSDB_CHILD_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_CTABLE_META); \ + break; \ + case TSDB_NORMAL_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_NTABLE_META); \ + break; \ + case TSDB_SYSTEM_TABLE: \ + CTG_DB_NUM_INC(CTG_CI_SYSTABLE_META); \ + break; \ + default: \ + CTG_DB_NUM_INC(CTG_CI_OTHERTABLE_META); \ + break; \ + } \ + } while (0) -#define CTG_META_NHIT_INC() CTG_CACHE_NHIT_INC(CTG_CI_OTHERTABLE_META, 1) +#define CTG_META_NUM_DEC(type) \ + do { \ + switch (type) { \ + case TSDB_SUPER_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_STABLE_META); \ + break; \ + case TSDB_CHILD_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_CTABLE_META); \ + break; \ + case TSDB_NORMAL_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_NTABLE_META); \ + break; \ + case TSDB_SYSTEM_TABLE: \ + CTG_DB_NUM_DEC(CTG_CI_SYSTABLE_META); \ + break; \ + default: \ + CTG_DB_NUM_DEC(CTG_CI_OTHERTABLE_META); \ + break; \ + } \ + } while (0) +#define CTG_META_HIT_INC(type) \ + do { \ + switch (type) { \ + case TSDB_SUPER_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_STABLE_META, 1); \ + break; \ + case TSDB_CHILD_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_CTABLE_META, 1); \ + break; \ + case TSDB_NORMAL_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_NTABLE_META, 1); \ + break; \ + case TSDB_SYSTEM_TABLE: \ + CTG_CACHE_HIT_INC(CTG_CI_SYSTABLE_META, 1); \ + break; \ + default: \ + CTG_CACHE_HIT_INC(CTG_CI_OTHERTABLE_META, 1); \ + break; \ + } \ + } while (0) + +#define CTG_META_NHIT_INC() CTG_CACHE_NHIT_INC(CTG_CI_OTHERTABLE_META, 1) #define CTG_IS_META_NULL(type) ((type) == META_TYPE_NULL_TABLE) #define CTG_IS_META_CTABLE(type) ((type) == META_TYPE_CTABLE) @@ -659,7 +670,6 @@ typedef struct SCtgCacheItemInfo { #define ctgTaskDebug(param, ...) qDebug("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) #define ctgTaskTrace(param, ...) qTrace("QID:%" PRIx64 " CTG:%p " param, pTask->pJob->queryId, pCtg, __VA_ARGS__) - #define CTG_LOCK_DEBUG(...) \ do { \ if (gCTGDebug.lockEnable) { \ @@ -681,38 +691,38 @@ typedef struct SCtgCacheItemInfo { #define TD_RWLATCH_WRITE_FLAG_COPY 0x40000000 -#define CTG_LOCK(type, _lock) \ - do { \ - if (CTG_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ - } else { \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ - } \ +#define CTG_LOCK(type, _lock) \ + do { \ + if (CTG_READ == (type)) { \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before read lock"); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG RLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value after read lock"); \ + } else { \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value before write lock"); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG WLOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value after write lock"); \ + } \ } while (0) -#define CTG_UNLOCK(type, _lock) \ - do { \ - if (CTG_READ == (type)) { \ - ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosRUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ - } else { \ - ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - taosWUnLockLatch(_lock); \ - CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ - ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ - } \ +#define CTG_UNLOCK(type, _lock) \ + do { \ + if (CTG_READ == (type)) { \ + ASSERTS(atomic_load_32((_lock)) > 0, "invalid lock value before read unlock"); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosRUnLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG RULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after read unlock"); \ + } else { \ + ASSERTS(atomic_load_32((_lock)) == TD_RWLATCH_WRITE_FLAG_COPY, "invalid lock value before write unlock"); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d B", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + taosWUnLockLatch(_lock); \ + CTG_LOCK_DEBUG("CTG WULOCK%p:%d, %s:%d E", (_lock), atomic_load_32(_lock), __FILE__, __LINE__); \ + ASSERTS(atomic_load_32((_lock)) >= 0, "invalid lock value after write unlock"); \ + } \ } while (0) #define CTG_ERR_RET(c) \ @@ -806,7 +816,7 @@ int32_t ctgTbMetaExistInCache(SCatalog* pCtg, char* dbFName, char* tbName, int32 int32_t ctgReadTbMetaFromCache(SCatalog* pCtg, SCtgTbMetaCtx* ctx, STableMeta** pTableMeta); int32_t ctgReadTbVerFromCache(SCatalog* pCtg, SName* pTableName, int32_t* sver, int32_t* tver, int32_t* tbType, uint64_t* suid, char* stbName); -int32_t ctgChkAuthFromCache(SCatalog* pCtg, char* user, char* dbFName, AUTH_TYPE type, bool* inCache, bool* pass); +int32_t ctgChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo* pReq, bool* inCache, SCtgAuthRsp* pRes); int32_t ctgDropDbCacheEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId); int32_t ctgDropDbVgroupEnqueue(SCatalog* pCtg, const char* dbFName, bool syncReq); int32_t ctgDropStbMetaEnqueue(SCatalog* pCtg, const char* dbFName, int64_t dbId, const char* stbName, uint64_t suid, @@ -879,7 +889,8 @@ void ctgFreeHandleImpl(SCatalog* pCtg); int32_t ctgGetVgInfoFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, const SName* pTableName, SVgroupInfo* pVgroup); int32_t ctgGetVgInfosFromHashValue(SCatalog* pCtg, SCtgTaskReq* tReq, SDBVgInfo* dbInfo, SCtgTbHashsCtx* pCtx, char* dbFName, SArray* pNames, bool update); -int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, int32_t* vgId); +int32_t ctgGetVgIdsFromHashValue(SCatalog* pCtg, SDBVgInfo* dbInfo, char* dbFName, const char* pTbs[], int32_t tbNum, + int32_t* vgId); void ctgResetTbMetaTask(SCtgTask* pTask); void ctgFreeDbCache(SCtgDBCache* dbCache); int32_t ctgStbVersionSortCompare(const void* key1, const void* key2); @@ -900,20 +911,24 @@ void ctgFreeQNode(SCtgQNode* node); void ctgClearHandle(SCatalog* pCtg); void ctgFreeTbCacheImpl(SCtgTbCache* pCache); int32_t ctgRemoveTbMeta(SCatalog* pCtg, SName* pTableName); -int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, bool* exists); +int32_t ctgGetTbHashVgroup(SCatalog* pCtg, SRequestConnInfo* pConn, const SName* pTableName, SVgroupInfo* pVgroup, + bool* exists); SName* ctgGetFetchName(SArray* pNames, SCtgFetch* pFetch); -int32_t ctgdGetOneHandle(SCatalog **pHandle); +int32_t ctgdGetOneHandle(SCatalog** pHandle); int ctgVgInfoComp(const void* lp, const void* rp); int32_t ctgMakeVgArray(SDBVgInfo* dbInfo); -int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb); -int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName); -void ctgReleaseVgMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache); -void ctgReleaseTbMetaToCache(SCatalog *pCtg, SCtgDBCache *dbCache, SCtgTbCache *pCache); -void ctgGetGlobalCacheStat(SCtgCacheStat *pStat); - -extern SCatalogMgmt gCtgMgmt; -extern SCtgDebug gCTGDebug; -extern SCtgAsyncFps gCtgAsyncFps[]; +int32_t ctgAcquireVgMetaFromCache(SCatalog* pCtg, const char* dbFName, const char* tbName, SCtgDBCache** pDb, + SCtgTbCache** pTb); +int32_t ctgCopyTbMeta(SCatalog* pCtg, SCtgTbMetaCtx* ctx, SCtgDBCache** pDb, SCtgTbCache** pTb, STableMeta** pTableMeta, + char* dbFName); +void ctgReleaseVgMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); +void ctgReleaseTbMetaToCache(SCatalog* pCtg, SCtgDBCache* dbCache, SCtgTbCache* pCache); +void ctgGetGlobalCacheStat(SCtgCacheStat* pStat); +int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res); + +extern SCatalogMgmt gCtgMgmt; +extern SCtgDebug gCTGDebug; +extern SCtgAsyncFps gCtgAsyncFps[]; extern SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE]; #ifdef __cplusplus diff --git a/source/libs/catalog/src/catalog.c b/source/libs/catalog/src/catalog.c index 8a56b0889b679b750e7077662dbbc4361e478310..bddc6c01a7cc8bf782db98987651ff1546707ce9 100644 --- a/source/libs/catalog/src/catalog.c +++ b/source/libs/catalog/src/catalog.c @@ -319,14 +319,13 @@ _return: CTG_RET(code); } -int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass, bool* exists) { +int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pReq, SUserAuthRes* pRes, bool* exists) { bool inCache = false; int32_t code = 0; + SCtgAuthRsp rsp = {0}; + rsp.pRawRes = pRes; - *pass = false; - - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, (char*)user, (char*)dbFName, type, &inCache, pass)); + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pReq, &inCache, &rsp)); if (inCache) { if (exists) { @@ -339,30 +338,22 @@ int32_t ctgChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, co return TSDB_CODE_SUCCESS; } - SGetUserAuthRsp authRsp = {0}; - CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, user, &authRsp, NULL)); - - if (authRsp.superAuth) { - *pass = true; - goto _return; - } - - if (authRsp.createdDbs && taosHashGet(authRsp.createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - goto _return; - } + SCtgAuthReq req = {0}; + req.pRawReq = pReq; + req.pConn = pConn; + req.onlyCache = exists ? true : false; + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pReq->user, &req.authInfo, NULL)); - if (CTG_AUTH_READ(type) && authRsp.readDbs && taosHashGet(authRsp.readDbs, dbFName, strlen(dbFName))) { - *pass = true; - } else if (CTG_AUTH_WRITE(type) && authRsp.writeDbs && taosHashGet(authRsp.writeDbs, dbFName, strlen(dbFName))) { - *pass = true; + CTG_ERR_JRET(ctgChkSetAuthRes(pCtg, &req, &rsp)); + if (rsp.metaNotExists && exists) { + *exists = false; } _return: - ctgUpdateUserEnqueue(pCtg, &authRsp, false); + ctgUpdateUserEnqueue(pCtg, &req.authInfo, false); - return TSDB_CODE_SUCCESS; + CTG_RET(code); } int32_t ctgGetTbType(SCatalog* pCtg, SRequestConnInfo* pConn, SName* pTableName, int32_t* tbType) { @@ -1373,7 +1364,7 @@ int32_t catalogGetExpiredUsers(SCatalog* pCtg, SUserAuthVersion** users, uint32_ void* key = taosHashGetKey(pAuth, &len); strncpy((*users)[i].user, key, len); (*users)[i].user[len] = 0; - (*users)[i].version = pAuth->version; + (*users)[i].version = pAuth->userAuth.version; ++i; if (i >= *num) { taosHashCancelIterate(pCtg->userCache, pAuth); @@ -1457,32 +1448,30 @@ _return: CTG_API_LEAVE(code); } -int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass) { +int32_t catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == pConn || NULL == user || NULL == dbFName || NULL == pass) { + if (NULL == pCtg || NULL == pConn || NULL == pAuth || NULL == pRes) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, user, dbFName, type, pass, NULL)); + CTG_ERR_JRET(ctgChkAuth(pCtg, pConn, pAuth, pRes, NULL)); _return: CTG_API_LEAVE(code); } -int32_t catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass, bool* exists) { +int32_t catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists) { CTG_API_ENTER(); - if (NULL == pCtg || NULL == user || NULL == dbFName || NULL == pass || NULL == exists) { + if (NULL == pCtg || NULL == pAuth || NULL == pRes || NULL == exists) { CTG_API_LEAVE(TSDB_CODE_CTG_INVALID_INPUT); } int32_t code = 0; - CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, user, dbFName, type, pass, exists)); + CTG_ERR_JRET(ctgChkAuth(pCtg, NULL, pAuth, pRes, exists)); _return: diff --git a/source/libs/catalog/src/ctgAsync.c b/source/libs/catalog/src/ctgAsync.c index 6c5666193753aea698559937c660022a5650bbbb..f2a354997da4c199d58bfe141f29d4f48270f65e 100644 --- a/source/libs/catalog/src/ctgAsync.c +++ b/source/libs/catalog/src/ctgAsync.c @@ -1550,45 +1550,20 @@ _return: int32_t ctgHandleGetUserRsp(SCtgTaskReq* tReq, int32_t reqType, const SDataBuf* pMsg, int32_t rspCode) { int32_t code = 0; SCtgTask* pTask = tReq->pTask; - SCtgUserCtx* ctx = (SCtgUserCtx*)pTask->taskCtx; SCatalog* pCtg = pTask->pJob->pCtg; - bool pass = false; SGetUserAuthRsp* pOut = (SGetUserAuthRsp*)pTask->msgCtx.out; CTG_ERR_JRET(ctgProcessRspMsg(pTask->msgCtx.out, reqType, pMsg->pData, pMsg->len, rspCode, pTask->msgCtx.target)); - if (pOut->superAuth) { - pass = true; - goto _return; - } + ctgUpdateUserEnqueue(pCtg, pOut, true); + taosMemoryFreeClear(pTask->msgCtx.out); - if (pOut->createdDbs && taosHashGet(pOut->createdDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { - pass = true; - goto _return; - } + CTG_ERR_JRET((*gCtgAsyncFps[pTask->type].launchFp)(pTask)); - if (CTG_AUTH_READ(ctx->user.type) && pOut->readDbs && - taosHashGet(pOut->readDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { - pass = true; - } else if (CTG_AUTH_WRITE(ctx->user.type) && pOut->writeDbs && - taosHashGet(pOut->writeDbs, ctx->user.dbFName, strlen(ctx->user.dbFName))) { - pass = true; - } + return TSDB_CODE_SUCCESS; _return: - if (TSDB_CODE_SUCCESS == code) { - pTask->res = taosMemoryCalloc(1, sizeof(bool)); - if (NULL == pTask->res) { - code = TSDB_CODE_OUT_OF_MEMORY; - } else { - *(bool*)pTask->res = pass; - } - } - - ctgUpdateUserEnqueue(pCtg, pOut, false); - taosMemoryFreeClear(pTask->msgCtx.out); - ctgHandleTaskEnd(pTask, code); CTG_RET(code); @@ -2081,31 +2056,39 @@ int32_t ctgLaunchGetUdfTask(SCtgTask* pTask) { } int32_t ctgLaunchGetUserTask(SCtgTask* pTask) { + int32_t code = 0; SCatalog* pCtg = pTask->pJob->pCtg; SRequestConnInfo* pConn = &pTask->pJob->conn; SCtgUserCtx* pCtx = (SCtgUserCtx*)pTask->taskCtx; bool inCache = false; - bool pass = false; + SCtgAuthRsp rsp = {0}; SCtgJob* pJob = pTask->pJob; SCtgMsgCtx* pMsgCtx = CTG_GET_TASK_MSGCTX(pTask, -1); if (NULL == pMsgCtx->pBatchs) { pMsgCtx->pBatchs = pJob->pBatchs; } - CTG_ERR_RET(ctgChkAuthFromCache(pCtg, pCtx->user.user, pCtx->user.dbFName, pCtx->user.type, &inCache, &pass)); + rsp.pRawRes = taosMemoryCalloc(1, sizeof(SUserAuthRes)); + if (NULL == rsp.pRawRes) { + CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); + } + + CTG_ERR_RET(ctgChkAuthFromCache(pCtg, &pCtx->user, &inCache, &rsp)); if (inCache) { - pTask->res = taosMemoryCalloc(1, sizeof(bool)); - if (NULL == pTask->res) { - CTG_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); - } - *(bool*)pTask->res = pass; + pTask->res = rsp.pRawRes; CTG_ERR_RET(ctgHandleTaskEnd(pTask, 0)); return TSDB_CODE_SUCCESS; } - CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); + taosMemoryFreeClear(rsp.pRawRes); + if (rsp.metaNotExists) { + CTG_ERR_RET(ctgLaunchSubTask(pTask, CTG_TASK_GET_TB_META, ctgGetTbCfgCb, &pCtx->user.tbName)); + } else { + CTG_ERR_RET(ctgGetUserDbAuthFromMnode(pCtg, pConn, pCtx->user.user, NULL, pTask)); + } + return TSDB_CODE_SUCCESS; } @@ -2155,6 +2138,20 @@ _return: CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); } + +int32_t ctgGetUserCb(SCtgTask* pTask) { + int32_t code = 0; + + CTG_ERR_JRET(pTask->subRes.code); + + CTG_RET(ctgLaunchGetUserTask(pTask)); + +_return: + + CTG_RET(ctgHandleTaskEnd(pTask, pTask->subRes.code)); +} + + int32_t ctgCompDbVgTasks(SCtgTask* pTask, void* param, bool* equal) { SCtgDbVgCtx* ctx = pTask->taskCtx; diff --git a/source/libs/catalog/src/ctgCache.c b/source/libs/catalog/src/ctgCache.c index b0321588651a73da880fa062c9b4e87baa478955..592b6e9c727602ebc6a8d8dc118409a5c1d01e73 100644 --- a/source/libs/catalog/src/ctgCache.c +++ b/source/libs/catalog/src/ctgCache.c @@ -32,27 +32,26 @@ SCtgOperation gCtgCacheOperation[CTG_OP_MAX] = {{CTG_OP_UPDATE_VGROUP, "update v {CTG_OP_CLEAR_CACHE, "clear cache", ctgOpClearCache}}; SCtgCacheItemInfo gCtgStatItem[CTG_CI_MAX_VALUE] = { - {"Cluster ", CTG_CI_FLAG_LEVEL_GLOBAL}, //CTG_CI_CLUSTER - {"Dnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_DNODE, - {"Qnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_QNODE, - {"DB ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_DB, - {"DbVgroup ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_DB_VGROUP, - {"DbCfg ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_DB_CFG, - {"DbInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_DB_INFO, - {"StbMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_STABLE_META, - {"NtbMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_NTABLE_META, - {"CtbMeta ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_CTABLE_META, - {"SysTblMeta", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_SYSTABLE_META, - {"OthTblMeta", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_OTHERTABLE_META, - {"TblSMA ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_SMA, - {"TblCfg ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_TBL_CFG, - {"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, //CTG_CI_INDEX_INFO, - {"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_USER, - {"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, //CTG_CI_UDF, - {"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} //CTG_CI_SVR_VER, + {"Cluster ", CTG_CI_FLAG_LEVEL_GLOBAL}, // CTG_CI_CLUSTER + {"Dnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_DNODE, + {"Qnode ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_QNODE, + {"DB ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_DB, + {"DbVgroup ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_DB_VGROUP, + {"DbCfg ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_DB_CFG, + {"DbInfo ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_DB_INFO, + {"StbMeta ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_STABLE_META, + {"NtbMeta ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_NTABLE_META, + {"CtbMeta ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_CTABLE_META, + {"SysTblMeta", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_SYSTABLE_META, + {"OthTblMeta", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_OTHERTABLE_META, + {"TblSMA ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_TBL_SMA, + {"TblCfg ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_TBL_CFG, + {"IndexInfo ", CTG_CI_FLAG_LEVEL_DB}, // CTG_CI_INDEX_INFO, + {"User ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_USER, + {"UDF ", CTG_CI_FLAG_LEVEL_CLUSTER}, // CTG_CI_UDF, + {"SvrVer ", CTG_CI_FLAG_LEVEL_CLUSTER} // CTG_CI_SVR_VER, }; - int32_t ctgRLockVgInfo(SCatalog *pCtg, SCtgDBCache *dbCache, bool *inCache) { CTG_LOCK(CTG_READ, &dbCache->vgCache.vgLock); @@ -263,10 +262,11 @@ _return: return TSDB_CODE_SUCCESS; } -int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, SCtgTbCache **pTb) { +int32_t ctgAcquireVgMetaFromCache(SCatalog *pCtg, const char *dbFName, const char *tbName, SCtgDBCache **pDb, + SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; SCtgTbCache *tbCache = NULL; - bool vgInCache = false; + bool vgInCache = false; ctgAcquireDBCache(pCtg, dbFName, &dbCache); if (NULL == dbCache) { @@ -331,7 +331,6 @@ _return: return TSDB_CODE_SUCCESS; } - /* int32_t ctgAcquireStbMetaFromCache(SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgDBCache **pDb, SCtgTbCache **pTb) { SCtgDBCache *dbCache = NULL; @@ -385,9 +384,10 @@ _return: } */ -int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, SCtgTbCache **pTb) { +int32_t ctgAcquireStbMetaFromCache(SCtgDBCache *dbCache, SCatalog *pCtg, char *dbFName, uint64_t suid, + SCtgTbCache **pTb) { SCtgTbCache *pCache = NULL; - char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); + char *stName = taosHashAcquire(dbCache->stbCache, &suid, sizeof(suid)); if (NULL == stName) { ctgDebug("stb 0x%" PRIx64 " not in cache, dbFName:%s", suid, dbFName); goto _return; @@ -484,10 +484,11 @@ int32_t ctgTbMetaExistInCache(SCatalog *pCtg, char *dbFName, char *tbName, int32 return TSDB_CODE_SUCCESS; } -int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, char* dbFName) { +int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCtgTbCache **pTb, STableMeta **pTableMeta, + char *dbFName) { SCtgDBCache *dbCache = *pDb; SCtgTbCache *tbCache = *pTb; - STableMeta *tbMeta = tbCache->pMeta; + STableMeta *tbMeta = tbCache->pMeta; ctx->tbInfo.inCache = true; ctx->tbInfo.dbId = dbCache->dbId; ctx->tbInfo.suid = tbMeta->suid; @@ -516,12 +517,12 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt memcpy(*pTableMeta, tbMeta, metaSize); - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); CTG_UNLOCK(CTG_READ, &tbCache->metaLock); taosHashRelease(dbCache->tbCache, tbCache); *pTb = NULL; - + ctgDebug("Got ctb %s meta from cache, will continue to get its stb meta, type:%d, dbFName:%s", ctx->pName->tname, ctx->tbInfo.tbType, dbFName); @@ -553,7 +554,6 @@ int32_t ctgCopyTbMeta(SCatalog *pCtg, SCtgTbMetaCtx *ctx, SCtgDBCache **pDb, SCt return TSDB_CODE_SUCCESS; } - int32_t ctgReadTbMetaFromCache(SCatalog *pCtg, SCtgTbMetaCtx *ctx, STableMeta **pTableMeta) { int32_t code = 0; SCtgDBCache *dbCache = NULL; @@ -623,17 +623,17 @@ int32_t ctgReadTbVerFromCache(SCatalog *pCtg, SName *pTableName, int32_t *sver, // PROCESS FOR CHILD TABLE - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); if (tbCache) { CTG_UNLOCK(CTG_READ, &tbCache->metaLock); taosHashRelease(dbCache->tbCache, tbCache); } - + ctgDebug("Got ctb %s ver from cache, will continue to get its stb ver, dbFName:%s", pTableName->tname, dbFName); ctgAcquireStbMetaFromCache(dbCache, pCtg, dbFName, *suid, &tbCache); if (NULL == tbCache) { - //ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); + // ctgReleaseTbMetaToCache(pCtg, dbCache, tbCache); ctgDebug("stb 0x%" PRIx64 " meta not in cache", *suid); return TSDB_CODE_SUCCESS; } @@ -703,55 +703,40 @@ _return: CTG_RET(code); } -int32_t ctgChkAuthFromCache(SCatalog *pCtg, char *user, char *dbFName, AUTH_TYPE type, bool *inCache, bool *pass) { - char *p = strchr(dbFName, '.'); - if (p) { - ++p; - } else { - p = dbFName; - } - - if (IS_SYS_DBNAME(p)) { +int32_t ctgChkAuthFromCache(SCatalog *pCtg, SUserAuthInfo *pReq, bool *inCache, SCtgAuthRsp *pRes) { + if (IS_SYS_DBNAME(pReq->tbName.dbname)) { *inCache = true; - *pass = true; - ctgDebug("sysdb %s, pass", dbFName); + pRes->pRawRes->pass = true; + ctgDebug("sysdb %s, pass", pReq->tbName.dbname); return TSDB_CODE_SUCCESS; } - SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, user, strlen(user)); + SCtgUserAuth *pUser = (SCtgUserAuth *)taosHashGet(pCtg->userCache, pReq->user, strlen(pReq->user)); if (NULL == pUser) { - ctgDebug("user not in cache, user:%s", user); + ctgDebug("user not in cache, user:%s", pReq->user); goto _return; } *inCache = true; - ctgDebug("Got user from cache, user:%s", user); + ctgDebug("Got user from cache, user:%s", pReq->user); CTG_CACHE_HIT_INC(CTG_CI_USER, 1); - if (pUser->superUser) { - *pass = true; - return TSDB_CODE_SUCCESS; - } + SCtgAuthReq req = {0}; + req.pRawReq = pReq; + req.onlyCache = true; CTG_LOCK(CTG_READ, &pUser->lock); - if (pUser->createdDbs && taosHashGet(pUser->createdDbs, dbFName, strlen(dbFName))) { - *pass = true; - CTG_UNLOCK(CTG_READ, &pUser->lock); - return TSDB_CODE_SUCCESS; - } - - if (pUser->readDbs && taosHashGet(pUser->readDbs, dbFName, strlen(dbFName)) && CTG_AUTH_READ(type)) { - *pass = true; - } + memcpy(&req.authInfo, &pUser->userAuth, sizeof(pUser->userAuth)); + int32_t code = ctgChkSetAuthRes(pCtg, &req, pRes); + CTG_UNLOCK(CTG_READ, &pUser->lock); + CTG_ERR_JRET(code); - if (pUser->writeDbs && taosHashGet(pUser->writeDbs, dbFName, strlen(dbFName)) && CTG_AUTH_WRITE(type)) { - *pass = true; + if (pRes->metaNotExists) { + goto _return; } - CTG_UNLOCK(CTG_READ, &pUser->lock); - - return TSDB_CODE_SUCCESS; + CTG_RET(code); _return: @@ -1639,7 +1624,7 @@ int32_t ctgWriteTbIndexToCache(SCatalog *pCtg, SCtgDBCache *dbCache, char *dbFNa } CTG_DB_NUM_INC(CTG_CI_TBL_SMA); - + *index = NULL; ctgDebug("table %s index updated to cache, ver:%d, num:%d", tbName, pIndex->version, (int32_t)taosArrayGetSize(pIndex->pIndex)); @@ -1725,9 +1710,9 @@ void ctgFreeAllInstance(void) { taosHashClear(gCtgMgmt.pCluster); } -int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { - int32_t* key = (int32_t*)lp; - SVgroupInfo* pVg = (SVgroupInfo*)rp; +int32_t ctgVgInfoIdComp(void const *lp, void const *rp) { + int32_t *key = (int32_t *)lp; + SVgroupInfo *pVg = (SVgroupInfo *)rp; if (*key < pVg->vgId) { return -1; @@ -1738,7 +1723,6 @@ int32_t ctgVgInfoIdComp(void const* lp, void const* rp) { return 0; } - int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { int32_t code = 0; SCtgUpdateVgMsg *msg = operation->data; @@ -1805,10 +1789,10 @@ int32_t ctgOpUpdateVgroup(SCtgCacheOperation *operation) { dbCache = NULL; - //if (!IS_SYS_DBNAME(dbFName)) { - tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); - CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), - ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); + // if (!IS_SYS_DBNAME(dbFName)) { + tstrncpy(vgVersion.dbFName, dbFName, sizeof(vgVersion.dbFName)); + CTG_ERR_JRET(ctgMetaRentUpdate(&msg->pCtg->dbRent, &vgVersion, vgVersion.dbId, sizeof(SDbVgVersion), + ctgDbVgVersionSortCompare, ctgDbVgVersionSearchCompare)); //} _return: @@ -2054,11 +2038,7 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { if (NULL == pUser) { SCtgUserAuth userAuth = {0}; - userAuth.version = msg->userAuth.version; - userAuth.superUser = msg->userAuth.superAuth; - userAuth.createdDbs = msg->userAuth.createdDbs; - userAuth.readDbs = msg->userAuth.readDbs; - userAuth.writeDbs = msg->userAuth.writeDbs; + memcpy(&userAuth.userAuth, &msg->userAuth, sizeof(msg->userAuth)); if (taosHashPut(pCtg->userCache, msg->userAuth.user, strlen(msg->userAuth.user), &userAuth, sizeof(userAuth))) { ctgError("taosHashPut user %s to cache failed", msg->userAuth.user); @@ -2072,22 +2052,32 @@ int32_t ctgOpUpdateUser(SCtgCacheOperation *operation) { return TSDB_CODE_SUCCESS; } - pUser->version = msg->userAuth.version; - CTG_LOCK(CTG_WRITE, &pUser->lock); - taosHashCleanup(pUser->createdDbs); - pUser->createdDbs = msg->userAuth.createdDbs; + taosHashCleanup(pUser->userAuth.createdDbs); + pUser->userAuth.createdDbs = msg->userAuth.createdDbs; msg->userAuth.createdDbs = NULL; - taosHashCleanup(pUser->readDbs); - pUser->readDbs = msg->userAuth.readDbs; + taosHashCleanup(pUser->userAuth.readDbs); + pUser->userAuth.readDbs = msg->userAuth.readDbs; msg->userAuth.readDbs = NULL; - taosHashCleanup(pUser->writeDbs); - pUser->writeDbs = msg->userAuth.writeDbs; + taosHashCleanup(pUser->userAuth.writeDbs); + pUser->userAuth.writeDbs = msg->userAuth.writeDbs; msg->userAuth.writeDbs = NULL; + taosHashCleanup(pUser->userAuth.readTbs); + pUser->userAuth.readTbs = msg->userAuth.readTbs; + msg->userAuth.readTbs = NULL; + + taosHashCleanup(pUser->userAuth.writeTbs); + pUser->userAuth.writeTbs = msg->userAuth.writeTbs; + msg->userAuth.writeTbs = NULL; + + taosHashCleanup(pUser->userAuth.useDbs); + pUser->userAuth.useDbs = msg->userAuth.useDbs; + msg->userAuth.useDbs = NULL; + CTG_UNLOCK(CTG_WRITE, &pUser->lock); _return: @@ -2095,6 +2085,9 @@ _return: taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); + taosHashCleanup(msg->userAuth.readTbs); + taosHashCleanup(msg->userAuth.writeTbs); + taosHashCleanup(msg->userAuth.useDbs); taosMemoryFreeClear(msg); @@ -2287,6 +2280,9 @@ void ctgFreeCacheOperationData(SCtgCacheOperation *op) { taosHashCleanup(msg->userAuth.createdDbs); taosHashCleanup(msg->userAuth.readDbs); taosHashCleanup(msg->userAuth.writeDbs); + taosHashCleanup(msg->userAuth.readTbs); + taosHashCleanup(msg->userAuth.writeTbs); + taosHashCleanup(msg->userAuth.useDbs); taosMemoryFreeClear(op->data); break; } @@ -2526,7 +2522,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArrayPush(ctx->pResList, &(SMetaRes){0}); CTG_META_NHIT_INC(); - + continue; } @@ -2544,7 +2540,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe STableMeta *tbMeta = pCache->pMeta; CTG_META_HIT_INC(tbMeta->tableType); - + SCtgTbMetaCtx nctx = {0}; nctx.flag = flag; nctx.tbInfo.inCache = true; @@ -2626,7 +2622,7 @@ int32_t ctgGetTbMetasFromCache(SCatalog *pCtg, SRequestConnInfo *pConn, SCtgTbMe ctgAddFetch(&ctx->pFetchs, dbIdx, i, fetchIdx, baseResIdx + i, flag); taosArrayPush(ctx->pResList, &(SMetaRes){0}); taosMemoryFreeClear(pTableMeta); - + CTG_META_NHIT_INC(); continue; } diff --git a/source/libs/catalog/src/ctgUtil.c b/source/libs/catalog/src/ctgUtil.c index fd6ee6c1b9f36f7fbe8d8e9a6b754957334028e6..b2b2b5a87e4eea144de7c0302756e70dca391cc0 100644 --- a/source/libs/catalog/src/ctgUtil.c +++ b/source/libs/catalog/src/ctgUtil.c @@ -174,9 +174,12 @@ void ctgFreeSMetaData(SMetaData* pData) { } void ctgFreeSCtgUserAuth(SCtgUserAuth* userCache) { - taosHashCleanup(userCache->createdDbs); - taosHashCleanup(userCache->readDbs); - taosHashCleanup(userCache->writeDbs); + taosHashCleanup(userCache->userAuth.createdDbs); + taosHashCleanup(userCache->userAuth.readDbs); + taosHashCleanup(userCache->userAuth.writeDbs); + taosHashCleanup(userCache->userAuth.readTbs); + taosHashCleanup(userCache->userAuth.writeTbs); + taosHashCleanup(userCache->userAuth.useDbs); } void ctgFreeMetaRent(SCtgRentMgmt* mgmt) { @@ -419,6 +422,9 @@ void ctgFreeMsgCtx(SCtgMsgCtx* pCtx) { taosHashCleanup(pOut->createdDbs); taosHashCleanup(pOut->readDbs); taosHashCleanup(pOut->writeDbs); + taosHashCleanup(pOut->readTbs); + taosHashCleanup(pOut->writeTbs); + taosHashCleanup(pOut->useDbs); taosMemoryFreeClear(pCtx->out); break; } @@ -1325,6 +1331,143 @@ static void* ctgCloneDnodeList(void* pSrc) { return taosArrayDup((const SArray*) static void ctgFreeDnodeList(void* p) { taosArrayDestroy((SArray*)((SMetaRes*)p)->pRes); } +int32_t ctgChkSetTbAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { + int32_t code = 0; + STableMeta* pMeta = NULL; + SGetUserAuthRsp* pInfo = &req->authInfo; + SHashObj* pTbs = (AUTH_TYPE_READ == req->singleType) ? pInfo->readTbs : pInfo->writeTbs; + + char tbFullName[TSDB_TABLE_FNAME_LEN]; + tNameExtractFullName(&req->pRawReq->tbName, tbFullName); + char* pCond = taosHashGet(pTbs, tbFullName, strlen(tbFullName)); + if (pCond) { + if (strlen(pCond) > 1) { + CTG_ERR_RET(nodesStringToNode(pCond, &res->pRawRes->pCond)); + } + + res->pRawRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + res->pRawRes->pass = false; + + // CTG_ERR_RET(catalogGetCachedTableMeta(pCtg, &req->pRawReq->tbName, &pMeta)); + // if (NULL == pMeta) { + // if (req->onlyCache) { + // res->metaNotExists = true; + // ctgDebug("db %s tb %s meta not in cache for auth", req->pRawReq->tbName.dbname, req->pRawReq->tbName.tname); + // return TSDB_CODE_SUCCESS; + // } + + // CTG_ERR_RET(catalogGetTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, &pMeta)); + // } + + // if (TSDB_SUPER_TABLE == pMeta->tableType || TSDB_NORMAL_TABLE == pMeta->tableType) { + // res->pRawRes->pass = false; + // goto _return; + // } + + // if (TSDB_CHILD_TABLE == pMeta->tableType) { + // res->pRawRes->pass = true; + + // /* + // char stbName[TSDB_TABLE_NAME_LEN] = {0}; + // CTG_ERR_JRET(ctgGetCachedStbNameFromSuid(pCtg, pMeta->suid, stbName)); + // if (0 == stbName[0]) { + // if (req->onlyCache) { + // res->notExists = true; + // return TSDB_CODE_SUCCESS; + // } + + // CTG_ERR_RET(catalogRefreshTableMeta(pCtg, req->pConn, &req->pRawReq->tbName, 0)); + // } + // */ + // } + +_return: + + taosMemoryFree(pMeta); + + CTG_RET(code); +} + +int32_t ctgChkSetAuthRes(SCatalog* pCtg, SCtgAuthReq* req, SCtgAuthRsp* res) { + int32_t code = 0; + SUserAuthInfo* pReq = req->pRawReq; + SUserAuthRes* pRes = res->pRawRes; + SGetUserAuthRsp* pInfo = &req->authInfo; + + pRes->pass = false; + pRes->pCond = NULL; + + if (!pInfo->enable) { + pRes->pass = false; + return TSDB_CODE_SUCCESS; + } + + if (pInfo->superAuth) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + char dbFName[TSDB_DB_FNAME_LEN]; + tNameGetFullDbName(&pReq->tbName, dbFName); + + if (pInfo->createdDbs && taosHashGet(pInfo->createdDbs, dbFName, strlen(dbFName))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + switch (pReq->type) { + case AUTH_TYPE_READ: { + if (pInfo->readTbs && taosHashGetSize(pInfo->readTbs) > 0) { + req->singleType = AUTH_TYPE_READ; + CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); + if (pRes->pass) { + return TSDB_CODE_SUCCESS; + } + } + + if (pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + break; + } + case AUTH_TYPE_WRITE: { + if (pInfo->writeTbs && taosHashGetSize(pInfo->writeTbs) > 0) { + req->singleType = AUTH_TYPE_WRITE; + CTG_ERR_RET(ctgChkSetTbAuthRes(pCtg, req, res)); + if (pRes->pass) { + return TSDB_CODE_SUCCESS; + } + } + + if (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + break; + } + case AUTH_TYPE_READ_OR_WRITE: { + if ((pInfo->readDbs && taosHashGet(pInfo->readDbs, dbFName, strlen(dbFName))) || + (pInfo->writeDbs && taosHashGet(pInfo->writeDbs, dbFName, strlen(dbFName))) || + (pInfo->useDbs && taosHashGet(pInfo->useDbs, dbFName, strlen(dbFName)))) { + pRes->pass = true; + return TSDB_CODE_SUCCESS; + } + + break; + } + default: + break; + } + + return TSDB_CODE_SUCCESS; +} + #if 0 static int32_t ctgCloneMetaDataArray(SArray* pSrc, __array_item_dup_fn_t copyFunc, SArray** pDst) { if (NULL == pSrc) { diff --git a/source/libs/catalog/test/CMakeLists.txt b/source/libs/catalog/test/CMakeLists.txt index dbbb24ce0b5e8b3ac37b212d88b49d2766bf78ca..de4d08835cf0ebd149e36f252330dfb73940f8df 100644 --- a/source/libs/catalog/test/CMakeLists.txt +++ b/source/libs/catalog/test/CMakeLists.txt @@ -9,7 +9,7 @@ IF(NOT TD_DARWIN) ADD_EXECUTABLE(catalogTest ${SOURCE_LIST}) TARGET_LINK_LIBRARIES( catalogTest - PUBLIC os util common catalog transport gtest qcom taos_static + PUBLIC os util common nodes catalog transport gtest qcom taos_static ) TARGET_INCLUDE_DIRECTORIES( diff --git a/source/libs/catalog/test/catalogTests.cpp b/source/libs/catalog/test/catalogTests.cpp index 5e543384ac2747005ddb40602ca778253ea1eacd..2cba433e849f89b161b7b3e2c3e58d9694aacdb0 100644 --- a/source/libs/catalog/test/catalogTests.cpp +++ b/source/libs/catalog/test/catalogTests.cpp @@ -27,8 +27,8 @@ #ifdef WINDOWS #define TD_USE_WINSOCK #endif -#include "catalogInt.h" #include "catalog.h" +#include "catalogInt.h" #include "os.h" #include "stub.h" #include "taos.h" @@ -48,7 +48,7 @@ void ctgTestSetRspCTableMeta(); void ctgTestSetRspSTableMeta(); void ctgTestSetRspMultiSTableMeta(); -extern int32_t clientConnRefPool; +extern int32_t clientConnRefPool; enum { CTGT_RSP_VGINFO = 1, @@ -298,7 +298,7 @@ void ctgTestBuildSTableMetaRsp(STableMetaRsp *rspMsg) { void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + SUseDbRsp usedbRsp = {0}; strcpy(usedbRsp.db, ctgTestDbname); usedbRsp.vgVersion = ctgTestVgVersion; @@ -343,7 +343,7 @@ void ctgTestRspDbVgroups(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); strcpy(metaRsp.tbName, ctgTestTablename); @@ -384,13 +384,13 @@ void ctgTestRspTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * void ctgTestRspTableMetaNotExist(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + pRsp->code = CTG_ERR_CODE_TABLE_NOT_EXIST; } void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); strcpy(metaRsp.tbName, ctgTestCurrentCTableName ? ctgTestCurrentCTableName : ctgTestCTablename); @@ -438,7 +438,7 @@ void ctgTestRspCTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + STableMetaRsp metaRsp = {0}; strcpy(metaRsp.dbFName, ctgTestDbname); strcpy(metaRsp.tbName, ctgTestCurrentSTableName ? ctgTestCurrentSTableName : ctgTestSTablename); @@ -450,7 +450,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg metaRsp.sversion = ctgTestSVersion; metaRsp.tversion = ctgTestTVersion; metaRsp.suid = ctgTestSuid; - metaRsp.tuid = ctgTestSuid+1; + metaRsp.tuid = ctgTestSuid + 1; metaRsp.vgId = 0; metaRsp.pSchemas = (SSchema *)taosMemoryMalloc((metaRsp.numOfTags + metaRsp.numOfColumns) * sizeof(SSchema)); @@ -486,7 +486,7 @@ void ctgTestRspSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; STableMetaRsp metaRsp = {0}; @@ -536,16 +536,14 @@ void ctgTestRspMultiSTableMeta(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRp tFreeSTableMetaRsp(&metaRsp); } - void ctgTestRspErrIndexInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + pRsp->code = TSDB_CODE_MND_DB_INDEX_NOT_EXIST; pRsp->contLen = 0; pRsp->pCont = NULL; } - void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); @@ -553,6 +551,7 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p strcpy(userRsp.user, ctgTestUsername); userRsp.version = 1; userRsp.superAuth = 1; + userRsp.enable = 1; int32_t contLen = tSerializeSGetUserAuthRsp(NULL, 0, &userRsp); void *pReq = rpcMallocCont(contLen); @@ -565,7 +564,7 @@ void ctgTestRspUserAuth(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; STableCfgRsp tblRsp = {0}; @@ -600,7 +599,7 @@ void ctgTestRspTableCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; STableIndexRsp tblRsp = {0}; @@ -611,10 +610,10 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg STableIndexInfo info = {0}; for (int32_t i = 0; i < ctgTestIndexNum; ++i) { info.interval = 1 + i; - info.expr = (char*)taosMemoryCalloc(1, 10); + info.expr = (char *)taosMemoryCalloc(1, 10); taosArrayPush(tblRsp.pIndex, &info); } - + int32_t contLen = tSerializeSTableIndexRsp(NULL, 0, &tblRsp); void *pReq = rpcMallocCont(contLen); tSerializeSTableIndexRsp(pReq, contLen, &tblRsp); @@ -628,12 +627,12 @@ void ctgTestRspTableIndex(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg void ctgTestRspDBCfg(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { rpcFreeCont(pMsg->pCont); - + static int32_t idx = 1; SDbCfgRsp dbRsp = {0}; dbRsp.numOfVgroups = ctgTestVgNum; - + int32_t contLen = tSerializeSDbCfgRsp(NULL, 0, &dbRsp); void *pReq = rpcMallocCont(contLen); tSerializeSDbCfgRsp(pReq, contLen, &dbRsp); @@ -651,7 +650,7 @@ void ctgTestRspQnodeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg * for (int32_t i = 0; i < ctgTestQnodeNum; ++i) { SQueryNodeLoad nodeLoad = {0}; nodeLoad.addr.nodeId = i; - + (void)taosArrayPush(qlistRsp.qnodeList, &nodeLoad); } @@ -672,12 +671,17 @@ void ctgTestRspUdfInfo(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pR SRetrieveFuncRsp funcRsp = {0}; funcRsp.numOfFuncs = 1; funcRsp.pFuncInfos = taosArrayInit(1, sizeof(SFuncInfo)); + funcRsp.pFuncExtraInfos = taosArrayInit(1, sizeof(SFuncExtraInfo)); SFuncInfo funcInfo = {0}; strcpy(funcInfo.name, "func1"); funcInfo.funcType = ctgTestFuncType; - - (void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo); + (void)taosArrayPush(funcRsp.pFuncInfos, &funcInfo); + SFuncExtraInfo extraInfo = {0}; + extraInfo.funcVersion = 0; + extraInfo.funcCreatedTime = taosGetTimestampMs(); + (void)taosArrayPush(funcRsp.pFuncExtraInfos, &extraInfo); + int32_t rspLen = tSerializeSRetrieveFuncRsp(NULL, 0, &funcRsp); void *pReq = rpcMallocCont(rspLen); tSerializeSRetrieveFuncRsp(pReq, rspLen, &funcRsp); @@ -694,7 +698,7 @@ void ctgTestRspSvrVer(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRs SServerVerRsp verRsp = {0}; strcpy(verRsp.ver, "1.0"); - + int32_t rspLen = tSerializeSServerVerRsp(NULL, 0, &verRsp); void *pReq = rpcMallocCont(rspLen); tSerializeSServerVerRsp(pReq, rspLen, &verRsp); @@ -713,9 +717,9 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p epSet.numOfEps = 1; tstrncpy(epSet.eps[0].fqdn, "localhost", TSDB_FQDN_LEN); epSet.eps[0].port = 6030; - + (void)taosArrayPush(dRsp.dnodeList, &epSet); - + int32_t rspLen = tSerializeSDnodeListRsp(NULL, 0, &dRsp); void *pReq = rpcMallocCont(rspLen); tSerializeSDnodeListRsp(pReq, rspLen, &dRsp); @@ -727,8 +731,6 @@ void ctgTestRspDndeList(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *p tFreeSDnodeListRsp(&dRsp); } - - void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (pMsg->msgType) { case TDMT_MND_USE_DB: @@ -745,7 +747,7 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) ctgTestRspDBCfg(shandle, pEpSet, pMsg, pRsp); break; case TDMT_MND_QNODE_LIST: - ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp); + ctgTestRspQnodeList(shandle, pEpSet, pMsg, pRsp); break; case TDMT_MND_RETRIEVE_FUNC: ctgTestRspUdfInfo(shandle, pEpSet, pMsg, pRsp); @@ -763,7 +765,6 @@ void ctgTestRspAuto(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) return; } - void ctgTestRspByIdx(void *shandle, SEpSet *pEpSet, SRpcMsg *pMsg, SRpcMsg *pRsp) { switch (ctgTestRspFunc[ctgTestRspIdx]) { case CTGT_RSP_VGINFO: @@ -1058,7 +1059,7 @@ void ctgTestSetRspDbVgroupsAndMultiSuperMeta() { void *ctgTestGetDbVgroupThread(void *param) { struct SCatalog *pCtg = (struct SCatalog *)param; int32_t code = 0; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SArray *vgList = NULL; int32_t n = 0; @@ -1204,7 +1205,6 @@ void *ctgTestSetCtableMetaThread(void *param) { return NULL; } - void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) { TAOS_ROW row; int num_fields = taos_num_fields(result); @@ -1220,23 +1220,22 @@ void ctgTestFetchRows(TAOS_RES *result, int32_t *rows) { } } -void ctgTestExecQuery(TAOS * taos, char* sql, bool fetch, int32_t *rows) { +void ctgTestExecQuery(TAOS *taos, char *sql, bool fetch, int32_t *rows) { TAOS_RES *result = taos_query(taos, sql); - int code = taos_errno(result); + int code = taos_errno(result); ASSERT_EQ(code, 0); if (fetch) { ctgTestFetchRows(result, rows); } - + taos_free_result(result); } - TEST(tableMeta, normalTable) { struct SCatalog *pCtg = NULL; SVgroupInfo vgInfo = {0}; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -1372,7 +1371,7 @@ TEST(tableMeta, normalTable) { TEST(tableMeta, childTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1484,7 +1483,7 @@ TEST(tableMeta, childTableCase) { TEST(tableMeta, superTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1626,7 +1625,7 @@ TEST(tableMeta, superTableCase) { TEST(tableMeta, rmStbMeta) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1696,7 +1695,7 @@ TEST(tableMeta, rmStbMeta) { TEST(tableMeta, updateStbMeta) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; @@ -1785,7 +1784,7 @@ TEST(tableMeta, updateStbMeta) { TEST(getIndexInfo, notExists) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -1809,13 +1808,13 @@ TEST(getIndexInfo, notExists) { SIndexInfo info; code = catalogGetIndexMeta(pCtg, mockPointer, "index1", &info); ASSERT_TRUE(code != 0); - + catalogDestroy(); } TEST(refreshGetMeta, normal2normal) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -1894,7 +1893,7 @@ TEST(refreshGetMeta, normal2normal) { TEST(refreshGetMeta, normal2notexist) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -1964,7 +1963,7 @@ TEST(refreshGetMeta, normal2notexist) { TEST(refreshGetMeta, normal2child) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2045,10 +2044,9 @@ TEST(refreshGetMeta, normal2child) { ctgTestCurrentSTableName = NULL; } - TEST(refreshGetMeta, stable2child) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2134,7 +2132,7 @@ TEST(refreshGetMeta, stable2child) { TEST(refreshGetMeta, stable2stable) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2220,7 +2218,7 @@ TEST(refreshGetMeta, stable2stable) { TEST(refreshGetMeta, child2stable) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SArray *vgList = NULL; @@ -2306,7 +2304,7 @@ TEST(refreshGetMeta, child2stable) { TEST(tableDistVgroup, normalTable) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; @@ -2343,7 +2341,7 @@ TEST(tableDistVgroup, normalTable) { TEST(tableDistVgroup, childTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; @@ -2381,7 +2379,7 @@ TEST(tableDistVgroup, childTableCase) { TEST(tableDistVgroup, superTableCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo *vgInfo = NULL; SArray *vgList = NULL; @@ -2430,7 +2428,7 @@ TEST(tableDistVgroup, superTableCase) { TEST(dbVgroup, getSetDbVgroupCase) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2520,7 +2518,7 @@ TEST(dbVgroup, getSetDbVgroupCase) { TEST(multiThread, getSetRmSameDbVgroup) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2572,7 +2570,7 @@ TEST(multiThread, getSetRmSameDbVgroup) { TEST(multiThread, getSetRmDiffDbVgroup) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2624,7 +2622,7 @@ TEST(multiThread, getSetRmDiffDbVgroup) { TEST(multiThread, ctableMeta) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2675,7 +2673,7 @@ TEST(multiThread, ctableMeta) { TEST(rentTest, allRent) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; SVgroupInfo vgInfo = {0}; SVgroupInfo *pvgInfo = NULL; @@ -2754,7 +2752,7 @@ TEST(rentTest, allRent) { TEST(apiTest, catalogRefreshDBVgInfo_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2781,7 +2779,7 @@ TEST(apiTest, catalogRefreshDBVgInfo_test) { TEST(apiTest, catalogChkAuth_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2800,15 +2798,19 @@ TEST(apiTest, catalogChkAuth_test) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - bool pass = false; + SUserAuthInfo authInfo = {0}; + SUserAuthRes authRes = {0}; + strcpy(authInfo.user, ctgTestUsername); + toName(1, ctgTestDbname, ctgTestSTablename, &authInfo.tbName); + authInfo.type = AUTH_TYPE_READ; bool exists = false; - code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); ASSERT_EQ(code, 0); ASSERT_EQ(exists, false); - - code = catalogChkAuth(pCtg, mockPointer, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass); + + code = catalogChkAuth(pCtg, mockPointer, &authInfo, &authRes); ASSERT_EQ(code, 0); - ASSERT_EQ(pass, true); + ASSERT_EQ(authRes.pass, true); while (true) { uint64_t n = 0; @@ -2820,9 +2822,9 @@ TEST(apiTest, catalogChkAuth_test) { } } - code = catalogChkAuthFromCache(pCtg, ctgTestUsername, ctgTestDbname, AUTH_TYPE_READ, &pass, &exists); + code = catalogChkAuthFromCache(pCtg, &authInfo, &authRes, &exists); ASSERT_EQ(code, 0); - ASSERT_EQ(pass, true); + ASSERT_EQ(authRes.pass, true); ASSERT_EQ(exists, true); catalogDestroy(); @@ -2830,7 +2832,7 @@ TEST(apiTest, catalogChkAuth_test) { TEST(apiTest, catalogRefreshGetTableCfg_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2868,7 +2870,7 @@ TEST(apiTest, catalogRefreshGetTableCfg_test) { TEST(apiTest, catalogGetTableIndex_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2904,7 +2906,7 @@ TEST(apiTest, catalogGetTableIndex_test) { TEST(apiTest, catalogGetDBCfg_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2937,7 +2939,7 @@ TEST(apiTest, catalogGetDBCfg_test) { TEST(apiTest, catalogGetQnodeList_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2962,17 +2964,16 @@ TEST(apiTest, catalogGetQnodeList_test) { ASSERT_EQ(taosArrayGetSize(qnodeList), ctgTestQnodeNum); for (int32_t i = 0; i < ctgTestQnodeNum; ++i) { - SQueryNodeLoad * pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i); + SQueryNodeLoad *pLoad = (SQueryNodeLoad *)taosArrayGet(qnodeList, i); ASSERT_EQ(pLoad->addr.nodeId, i); } catalogDestroy(); } - TEST(apiTest, catalogGetUdfInfo_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -2999,10 +3000,9 @@ TEST(apiTest, catalogGetUdfInfo_test) { catalogDestroy(); } - TEST(apiTest, catalogGetServerVersion_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -3021,7 +3021,7 @@ TEST(apiTest, catalogGetServerVersion_test) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - char* ver = NULL; + char *ver = NULL; code = catalogGetServerVersion(pCtg, mockPointer, &ver); ASSERT_EQ(code, 0); ASSERT_TRUE(0 == strcmp(ver, "1.0")); @@ -3031,7 +3031,7 @@ TEST(apiTest, catalogGetServerVersion_test) { TEST(apiTest, catalogUpdateTableIndex_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -3061,10 +3061,9 @@ TEST(apiTest, catalogUpdateTableIndex_test) { catalogDestroy(); } - TEST(apiTest, catalogGetDnodeList_test) { struct SCatalog *pCtg = NULL; - SRequestConnInfo connInfo = {0}; + SRequestConnInfo connInfo = {0}; SRequestConnInfo *mockPointer = (SRequestConnInfo *)&connInfo; ctgTestInitLogFile(); @@ -3083,7 +3082,7 @@ TEST(apiTest, catalogGetDnodeList_test) { code = catalogGetHandle(ctgTestClusterId, &pCtg); ASSERT_EQ(code, 0); - SArray* pList = NULL; + SArray *pList = NULL; code = catalogGetDnodeList(pCtg, mockPointer, &pList); ASSERT_EQ(code, 0); ASSERT_EQ(taosArrayGetSize(pList), 1); @@ -3095,23 +3094,23 @@ TEST(apiTest, catalogGetDnodeList_test) { #ifdef INTEGRATION_TEST TEST(intTest, autoCreateTableTest) { - struct SCatalog *pCtg = NULL; + struct SCatalog *pCtg = NULL; - TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); - ASSERT_TRUE(NULL != taos); + TAOS *taos = taos_connect("localhost", "root", "taosdata", NULL, 0); + ASSERT_TRUE(NULL != taos); ctgdEnableDebug("api", true); ctgdEnableDebug("meta", true); ctgdEnableDebug("cache", true); ctgdEnableDebug("lock", true); - + ctgTestExecQuery(taos, "drop database if exists db1", false, NULL); ctgTestExecQuery(taos, "create database db1", false, NULL); ctgTestExecQuery(taos, "create stable db1.st1 (ts timestamp, f1 int) tags(tg1 int)", false, NULL); ctgTestExecQuery(taos, "insert into db1.tb1 using db1.st1 tags(1) values(now, 1)", false, NULL); ctgdGetOneHandle(&pCtg); - + while (true) { uint32_t n = ctgdGetClusterCacheNum(pCtg, CTG_DBG_META_NUM); if (2 != n) { @@ -3137,15 +3136,14 @@ TEST(intTest, autoCreateTableTest) { ctgTestExecQuery(taos, "alter table db1.st1 add column f2 double", false, NULL); ctgdEnableDebug("stopUpdate", false); - + ctgTestExecQuery(taos, "insert into db1.tb1 (ts, f1) values(now, 4)", false, NULL); - + taos_close(taos); } #endif - int main(int argc, char **argv) { testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); diff --git a/source/libs/executor/inc/dataSinkInt.h b/source/libs/executor/inc/dataSinkInt.h index 220e9c0b6ced01edce3a36a4297f2c76151a4092..57a771b275b219b776dbee34193e2ba644e73a1f 100644 --- a/source/libs/executor/inc/dataSinkInt.h +++ b/source/libs/executor/inc/dataSinkInt.h @@ -29,7 +29,6 @@ struct SDataSinkHandle; typedef struct SDataSinkManager { SDataSinkMgtCfg cfg; - TdThreadMutex mutex; } SDataSinkManager; typedef int32_t (*FPutDataBlock)(struct SDataSinkHandle* pHandle, const SInputData* pInput, bool* pContinue); diff --git a/source/libs/executor/inc/executil.h b/source/libs/executor/inc/executil.h index 9b8f034e448643ddfcfd7c9eaa1a46f181078718..2e92f9e396eb2a0b76acfa1066c3829c22e71071 100644 --- a/source/libs/executor/inc/executil.h +++ b/source/libs/executor/inc/executil.h @@ -108,6 +108,7 @@ uint64_t tableListGetSize(const STableListInfo* pTableList); uint64_t tableListGetSuid(const STableListInfo* pTableList); STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index); int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t startIndex); +void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type); size_t getResultRowSize(struct SqlFunctionCtx* pCtx, int32_t numOfOutput); void initResultRowInfo(SResultRowInfo* pResultRowInfo); diff --git a/source/libs/executor/inc/executorimpl.h b/source/libs/executor/inc/executorimpl.h index 54a9ff0942ee1b5865b42b15ff4228709719fe5b..43ee54f65d8ab29cea92c453624b0b64fa449350 100644 --- a/source/libs/executor/inc/executorimpl.h +++ b/source/libs/executor/inc/executorimpl.h @@ -127,14 +127,9 @@ enum { }; typedef struct { - // TODO remove prepareStatus - STqOffsetVal prepareStatus; // for tmq - STqOffsetVal lastStatus; // for tmq - SMqMetaRsp metaRsp; // for tmq fetching meta - int8_t returned; - int64_t snapshotVer; - // const SSubmitReq* pReq; - + STqOffsetVal currentOffset; // for tmq + SMqMetaRsp metaRsp; // for tmq fetching meta + int64_t snapshotVer; SPackedData submit; SSchemaWrapper* schema; char tbName[TSDB_TABLE_NAME_LEN]; @@ -144,6 +139,8 @@ typedef struct { int64_t fillHistoryVer1; int64_t fillHistoryVer2; SStreamState* pState; + int64_t dataVersion; + int64_t checkPointId; } SStreamTaskInfo; typedef struct { @@ -191,7 +188,6 @@ enum { OP_OPENED = 0x1, OP_RES_TO_RETURN = 0x5, OP_EXEC_DONE = 0x9, - OP_EXEC_RECV = 0x11, }; typedef struct SOperatorFpSet { @@ -370,6 +366,7 @@ typedef struct STagScanInfo { int32_t curPos; SReadHandle readHandle; STableListInfo* pTableListInfo; + SLimitNode* pSlimit; } STagScanInfo; typedef enum EStreamScanMode { @@ -431,7 +428,6 @@ typedef struct STimeWindowAggSupp { } STimeWindowAggSupp; typedef struct SStreamScanInfo { - uint64_t tableUid; // queried super table uid SExprInfo* pPseudoExpr; int32_t numOfPseudoExpr; SExprSupp tbnameCalSup; @@ -502,32 +498,12 @@ typedef struct STableCountScanSupp { char stbNameFilter[TSDB_TABLE_NAME_LEN]; } STableCountScanSupp; -typedef struct STableCountScanOperatorInfo { - SReadHandle readHandle; - SSDataBlock* pRes; - - STableCountScanSupp supp; - - int32_t currGrpIdx; - SArray* stbUidList; // when group by db_name and/or stable_name -} STableCountScanOperatorInfo; - typedef struct SOptrBasicInfo { SResultRowInfo resultRowInfo; SSDataBlock* pRes; bool mergeResultBlock; } SOptrBasicInfo; -typedef struct SAggOperatorInfo { - SOptrBasicInfo binfo; - SAggSupporter aggSup; - STableQueryInfo* current; - uint64_t groupId; - SGroupResInfo groupResInfo; - SExprSupp scalarExprSup; - bool groupKeyOptimized; -} SAggOperatorInfo; - typedef struct SIntervalAggOperatorInfo { SOptrBasicInfo binfo; // basic info SAggSupporter aggSup; // aggregate supporter @@ -580,6 +556,7 @@ typedef struct SStreamIntervalOperatorInfo { uint64_t numOfDatapack; SArray* pUpdated; SSHashObj* pUpdatedMap; + int64_t dataVersion; } SStreamIntervalOperatorInfo; typedef struct SDataGroupInfo { @@ -629,6 +606,7 @@ typedef struct SStreamSessionAggOperatorInfo { bool ignoreExpiredDataSaved; SArray* pUpdated; SSHashObj* pStUpdated; + int64_t dataVersion; } SStreamSessionAggOperatorInfo; typedef struct SStreamStateAggOperatorInfo { @@ -647,6 +625,7 @@ typedef struct SStreamStateAggOperatorInfo { bool ignoreExpiredDataSaved; SArray* pUpdated; SSHashObj* pSeUpdated; + int64_t dataVersion; } SStreamStateAggOperatorInfo; typedef struct SStreamPartitionOperatorInfo { @@ -847,14 +826,13 @@ void setTaskKilled(SExecTaskInfo* pTaskInfo, int32_t rspCode); void doDestroyTask(SExecTaskInfo* pTaskInfo); void setTaskStatus(SExecTaskInfo* pTaskInfo, int8_t status); -char* buildTaskId(uint64_t taskId, uint64_t queryId); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst); SArray* getTableListInfo(const SExecTaskInfo* pTaskInfo); int32_t createExecTaskInfo(SSubplan* pPlan, SExecTaskInfo** pTaskInfo, SReadHandle* pHandle, uint64_t taskId, int32_t vgId, char* sql, EOPTR_EXEC_MODEL model); -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo, - SReadHandle* readHandle); +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle); int32_t getOperatorExplainExecInfo(SOperatorInfo* operatorInfo, SArray* pExecInfoList); STimeWindow getActiveTimeWindow(SDiskbasedBuf* pBuf, SResultRowInfo* pResultRowInfo, int64_t ts, SInterval* pInterval, diff --git a/source/libs/executor/inc/tfill.h b/source/libs/executor/inc/tfill.h index 726f0df1e85877de7e00babad2a14148734a5953..5fd75f9b99d004be68b369172668697bfa6fa9a6 100644 --- a/source/libs/executor/inc/tfill.h +++ b/source/libs/executor/inc/tfill.h @@ -30,6 +30,7 @@ extern "C" { struct SSDataBlock; typedef struct SFillColInfo { + int32_t numOfFillExpr; SExprInfo* pExpr; bool notFillCol; // denote if this column needs fill operation SVariant fillVal; diff --git a/source/libs/executor/src/aggregateoperator.c b/source/libs/executor/src/aggregateoperator.c index a26e3ace7b5a715cd3bcdda315ae9587c2d033cd..ec8060348d4b5dd465be4e6356c73994aa54df7c 100644 --- a/source/libs/executor/src/aggregateoperator.c +++ b/source/libs/executor/src/aggregateoperator.c @@ -40,6 +40,16 @@ typedef struct { int32_t startOffset; } SFunctionCtxStatus; +typedef struct SAggOperatorInfo { + SOptrBasicInfo binfo; + SAggSupporter aggSup; + STableQueryInfo* current; + uint64_t groupId; + SGroupResInfo groupResInfo; + SExprSupp scalarExprSup; + bool groupKeyOptimized; +} SAggOperatorInfo; + static void destroyAggOperatorInfo(void* param); static void setExecutionContext(SOperatorInfo* pOperator, int32_t numOfOutput, uint64_t groupId); @@ -162,9 +172,9 @@ int32_t doOpenAggregateOptr(SOperatorInfo* pOperator) { int32_t scanFlag = MAIN_SCAN; bool hasValidBlock = false; - bool blockAllocated = false; while (1) { + bool blockAllocated = false; SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { if (!hasValidBlock) { @@ -560,4 +570,4 @@ void functionCtxRestore(SqlFunctionCtx* pCtx, SFunctionCtxStatus* pStatus) { pCtx->input.colDataSMAIsSet = pStatus->hasAgg; pCtx->input.numOfRows = pStatus->numOfRows; pCtx->input.startRowIndex = pStatus->startOffset; -} \ No newline at end of file +} diff --git a/source/libs/executor/src/cachescanoperator.c b/source/libs/executor/src/cachescanoperator.c index f6fc332b3793ee25830cacf585c4d7d6b698f959..61f024ebb70bcbbf2b2127b41d3ac5f22ec74693 100644 --- a/source/libs/executor/src/cachescanoperator.c +++ b/source/libs/executor/src/cachescanoperator.c @@ -45,13 +45,13 @@ static void destroyCacheScanOperator(void* param); static int32_t extractCacheScanSlotId(const SArray* pColMatchInfo, SExecTaskInfo* pTaskInfo, int32_t** pSlotIds); static int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pColMatchInfo); -#define SCAN_ROW_TYPE(_t) ((_t)? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) +#define SCAN_ROW_TYPE(_t) ((_t) ? CACHESCAN_RETRIEVE_LAST : CACHESCAN_RETRIEVE_LAST_ROW) SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { - int32_t code = TSDB_CODE_SUCCESS; + int32_t code = TSDB_CODE_SUCCESS; SCacheRowsScanInfo* pInfo = taosMemoryCalloc(1, sizeof(SCacheRowsScanInfo)); - SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); + SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { code = TSDB_CODE_OUT_OF_MEMORY; tableListDestroy(pTableListInfo); @@ -91,7 +91,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe uint64_t suid = tableListGetSuid(pTableListInfo); code = tsdbCacherowsReaderOpen(pInfo->readHandle.vnode, pInfo->retrieveType, pList, totalTables, - taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, pTaskInfo->id.str); + taosArrayGetSize(pInfo->matchInfo.pList), suid, &pInfo->pLastrowReader, + pTaskInfo->id.str); if (code != TSDB_CODE_SUCCESS) { goto _error; } @@ -114,7 +115,8 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe p->pCtx = createSqlFunctionCtx(p->pExprInfo, p->numOfExprs, &p->rowEntryInfoOffset); } - setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); + setOperatorInfo(pOperator, "CachedRowScanOperator", QUERY_NODE_PHYSICAL_PLAN_LAST_ROW_SCAN, false, OP_NOT_OPENED, + pInfo, pTaskInfo); pOperator->exprSupp.numOfExprs = taosArrayGetSize(pInfo->pRes->pDataBlock); pOperator->fpSet = @@ -123,7 +125,7 @@ SOperatorInfo* createCacherowsScanOperator(SLastRowScanPhysiNode* pScanNode, SRe pOperator->cost.openCost = 0; return pOperator; - _error: +_error: pTaskInfo->code = code; destroyCacheScanOperator(pInfo); taosMemoryFree(pOperator); @@ -136,8 +138,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } SCacheRowsScanInfo* pInfo = pOperator->info; - SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - STableListInfo* pTableList = pInfo->pTableList; + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STableListInfo* pTableList = pInfo->pTableList; uint64_t suid = tableListGetSuid(pTableList); int32_t size = tableListGetSize(pTableList); @@ -194,8 +196,8 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pRes->info.rows = 1; SExprSupp* pSup = &pInfo->pseudoExprSup; - int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, - pRes->info.rows, GET_TASKID(pTaskInfo), NULL); + int32_t code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pRes, + pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -217,7 +219,7 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { } STableKeyInfo* pList = NULL; - int32_t num = 0; + int32_t num = 0; int32_t code = tableListGetGroupList(pTableList, pInfo->currentGroupIndex, &pList, &num); if (code != TSDB_CODE_SUCCESS) { @@ -251,11 +253,9 @@ SSDataBlock* doScanCache(SOperatorInfo* pOperator) { pInfo->pRes->info.id.groupId = pKeyInfo->groupId; if (taosArrayGetSize(pInfo->pUidList) > 0) { - ASSERT((pInfo->retrieveType & CACHESCAN_RETRIEVE_LAST_ROW) == CACHESCAN_RETRIEVE_LAST_ROW); - pInfo->pRes->info.id.uid = *(tb_uid_t*)taosArrayGet(pInfo->pUidList, 0); - code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, pInfo->pRes->info.rows, - GET_TASKID(pTaskInfo), NULL); + code = addTagPseudoColumnData(&pInfo->readHandle, pSup->pExprInfo, pSup->numOfExprs, pInfo->pRes, + pInfo->pRes->info.rows, GET_TASKID(pTaskInfo), NULL); if (code != TSDB_CODE_SUCCESS) { pTaskInfo->code = code; return NULL; @@ -325,7 +325,7 @@ int32_t removeRedundantTsCol(SLastRowScanPhysiNode* pScanNode, SColMatchInfo* pC return TSDB_CODE_SUCCESS; } - size_t size = taosArrayGetSize(pColMatchInfo->pList); + size_t size = taosArrayGetSize(pColMatchInfo->pList); SArray* pMatchInfo = taosArrayInit(size, sizeof(SColMatchItem)); for (int32_t i = 0; i < size; ++i) { diff --git a/source/libs/executor/src/dataDeleter.c b/source/libs/executor/src/dataDeleter.c index dc3bf83a914eca1a93304b1487b7acdeca59c1ba..d693faf7f14e90399c4c2393f48c8da26ac1ff39 100644 --- a/source/libs/executor/src/dataDeleter.c +++ b/source/libs/executor/src/dataDeleter.c @@ -240,6 +240,7 @@ int32_t createDataDeleter(SDataSinkManager* pManager, const SDataSinkNode* pData SDataDeleterHandle* deleter = taosMemoryCalloc(1, sizeof(SDataDeleterHandle)); if (NULL == deleter) { + taosMemoryFree(pParam); code = TSDB_CODE_OUT_OF_MEMORY; goto _end; } diff --git a/source/libs/executor/src/dataInserter.c b/source/libs/executor/src/dataInserter.c index e9c46843c02cbe9c736dda90a0cf66fa22a780b4..90d740bebd8658ced2346292f527e9ecf1ade2cf 100644 --- a/source/libs/executor/src/dataInserter.c +++ b/source/libs/executor/src/dataInserter.c @@ -408,6 +408,7 @@ int32_t createDataInserter(SDataSinkManager* pManager, const SDataSinkNode* pDat void* pParam) { SDataInserterHandle* inserter = taosMemoryCalloc(1, sizeof(SDataInserterHandle)); if (NULL == inserter) { + taosMemoryFree(pParam); terrno = TSDB_CODE_OUT_OF_MEMORY; return TSDB_CODE_OUT_OF_MEMORY; } diff --git a/source/libs/executor/src/dataSinkMgt.c b/source/libs/executor/src/dataSinkMgt.c index 7fbde07f596989c3a27ab5551c2b0215c13d4a59..b3cb57325ba5dee1d2d2f3b0d07304a1d1ea5ae2 100644 --- a/source/libs/executor/src/dataSinkMgt.c +++ b/source/libs/executor/src/dataSinkMgt.c @@ -23,7 +23,6 @@ SDataSinkStat gDataSinkStat = {0}; int32_t dsDataSinkMgtInit(SDataSinkMgtCfg* cfg) { gDataSinkManager.cfg = *cfg; - taosThreadMutexInit(&gDataSinkManager.mutex, NULL); return 0; // to avoid compiler eror } diff --git a/source/libs/executor/src/executil.c b/source/libs/executor/src/executil.c index 7d318786ba109927035c3b212aafbb85270d14bd..f61fd1ae0159c8219a7070d56f9874c10190f329 100644 --- a/source/libs/executor/src/executil.c +++ b/source/libs/executor/src/executil.c @@ -27,15 +27,21 @@ #include "executorimpl.h" #include "tcompression.h" +typedef struct STableListIdInfo { + uint64_t suid; + uint64_t uid; + int32_t tableType; +} STableListIdInfo; + // If the numOfOutputGroups is 1, the data blocks that belongs to different groups will be provided randomly // The numOfOutputGroups is specified by physical plan. and will not be affect by numOfGroups struct STableListInfo { - bool oneTableForEachGroup; - int32_t numOfOuputGroups; // the data block will be generated one by one - int32_t* groupOffset; // keep the offset value for each group in the tableList - SArray* pTableList; - SHashObj* map; // speedup acquire the tableQueryInfo by table uid - uint64_t suid; + bool oneTableForEachGroup; + int32_t numOfOuputGroups; // the data block will be generated one by one + int32_t* groupOffset; // keep the offset value for each group in the tableList + SArray* pTableList; + SHashObj* map; // speedup acquire the tableQueryInfo by table uid + STableListIdInfo idInfo; // this maybe the super table or ordinary table }; typedef struct tagFilterAssist { @@ -470,7 +476,7 @@ int32_t getColInfoResultForGroupby(void* metaHandle, SNodeList* group, STableLis } // int64_t stt = taosGetTimestampUs(); - code = metaGetTableTags(metaHandle, pTableListInfo->suid, pUidTagList); + code = metaGetTableTags(metaHandle, pTableListInfo->idInfo.suid, pUidTagList); if (code != TSDB_CODE_SUCCESS) { goto end; } @@ -953,7 +959,7 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN FilterCondType condType = checkTagCond(pTagCond); - int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->suid, pUidTagList, pTagCond); + int32_t filter = optimizeTbnameInCond(metaHandle, pListInfo->idInfo.suid, pUidTagList, pTagCond); if (filter == 0) { // tbname in filter is activated, do nothing and return taosArrayClear(pUidList); @@ -966,12 +972,12 @@ static int32_t doFilterByTagCond(STableListInfo* pListInfo, SArray* pUidList, SN terrno = 0; } else { if ((condType == FILTER_NO_LOGIC || condType == FILTER_AND) && status != SFLT_NOT_INDEX) { - code = metaGetTableTagsByUids(metaHandle, pListInfo->suid, pUidTagList); + code = metaGetTableTagsByUids(metaHandle, pListInfo->idInfo.suid, pUidTagList); } else { - code = metaGetTableTags(metaHandle, pListInfo->suid, pUidTagList); + code = metaGetTableTags(metaHandle, pListInfo->idInfo.suid, pUidTagList); } if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->suid); + qError("failed to get table tags from meta, reason:%s, suid:%" PRIu64, tstrerror(code), pListInfo->idInfo.suid); terrno = code; goto end; } @@ -1025,15 +1031,17 @@ int32_t getTableList(void* metaHandle, void* pVnode, SScanPhysiNode* pScanNode, int32_t code = TSDB_CODE_SUCCESS; size_t numOfTables = 0; - pListInfo->suid = pScanNode->suid; + pListInfo->idInfo.suid = pScanNode->suid; + pListInfo->idInfo.tableType = pScanNode->tableType; + SArray* pUidList = taosArrayInit(8, sizeof(uint64_t)); SIdxFltStatus status = SFLT_NOT_INDEX; if (pScanNode->tableType != TSDB_SUPER_TABLE) { + pListInfo->idInfo.uid = pScanNode->uid; if (metaIsTableExist(metaHandle, pScanNode->uid)) { taosArrayPush(pUidList, &pScanNode->uid); } - code = doFilterByTagCond(pListInfo, pUidList, pTagCond, metaHandle, status); if (code != TSDB_CODE_SUCCESS) { goto _end; @@ -1794,7 +1802,9 @@ uint64_t tableListGetSize(const STableListInfo* pTableList) { return taosArrayGetSize(pTableList->pTableList); } -uint64_t tableListGetSuid(const STableListInfo* pTableList) { return pTableList->suid; } +uint64_t tableListGetSuid(const STableListInfo* pTableList) { + return pTableList->idInfo.suid; +} STableKeyInfo* tableListGetInfo(const STableListInfo* pTableList, int32_t index) { if (taosArrayGetSize(pTableList->pTableList) == 0) { @@ -1819,6 +1829,12 @@ int32_t tableListFind(const STableListInfo* pTableList, uint64_t uid, int32_t st return -1; } +void tableListGetSourceTableInfo(const STableListInfo* pTableList, uint64_t* psuid, uint64_t* uid, int32_t* type) { + *psuid = pTableList->idInfo.suid; + *uid = pTableList->idInfo.uid; + *type = pTableList->idInfo.tableType; +} + uint64_t getTableGroupId(const STableListInfo* pTableList, uint64_t tableUid) { int32_t* slot = taosHashGet(pTableList->map, &tableUid, sizeof(tableUid)); ASSERT(pTableList->map != NULL && slot != NULL); diff --git a/source/libs/executor/src/executor.c b/source/libs/executor/src/executor.c index 6e05ea0d4f4bea99eb7bc17e93f70877e83c4dbb..1732ec04a77f54d10ab57a31aae663ea5e47e9d2 100644 --- a/source/libs/executor/src/executor.c +++ b/source/libs/executor/src/executor.c @@ -14,6 +14,7 @@ */ #include "executor.h" +#include #include "executorimpl.h" #include "planner.h" #include "tdatablock.h" @@ -127,12 +128,10 @@ static int32_t doSetStreamBlock(SOperatorInfo* pOperator, void* input, size_t nu pOperator->status = OP_NOT_OPENED; SStreamScanInfo* pInfo = pOperator->info; - qDebug("stream set total blocks:%d, task id:%s" PRIx64, (int32_t)numOfBlocks, id); - ASSERT(pInfo->validBlockIndex == 0); - ASSERT(taosArrayGetSize(pInfo->pBlockLists) == 0); + qDebug("s-task set source blocks:%d %s", (int32_t)numOfBlocks, id); + ASSERT(pInfo->validBlockIndex == 0 && taosArrayGetSize(pInfo->pBlockLists) == 0); if (type == STREAM_INPUT__MERGED_SUBMIT) { - // ASSERT(numOfBlocks > 1); for (int32_t i = 0; i < numOfBlocks; i++) { SPackedData* pReq = POINTER_SHIFT(input, i * sizeof(SPackedData)); taosArrayPush(pInfo->pBlockLists, pReq); @@ -175,9 +174,7 @@ void doSetTaskId(SOperatorInfo* pOperator) { void qSetTaskId(qTaskInfo_t tinfo, uint64_t taskId, uint64_t queryId) { SExecTaskInfo* pTaskInfo = tinfo; pTaskInfo->id.queryId = queryId; - - taosMemoryFreeClear(pTaskInfo->id.str); - pTaskInfo->id.str = buildTaskId(taskId, queryId); + buildTaskId(taskId, queryId, pTaskInfo->id.str); // set the idstr for tsdbReader doSetTaskId(pTaskInfo->pRoot); @@ -200,6 +197,13 @@ int32_t qSetStreamOpOpen(qTaskInfo_t tinfo) { return code; } +void qGetCheckpointVersion(qTaskInfo_t tinfo, int64_t* dataVer, int64_t* ckId) { + SExecTaskInfo* pTaskInfo = tinfo; + *dataVer = pTaskInfo->streamInfo.dataVersion; + *ckId = pTaskInfo->streamInfo.checkPointId; +} + + int32_t qSetMultiStreamInput(qTaskInfo_t tinfo, const void* pBlocks, size_t numOfBlocks, int32_t type) { if (tinfo == NULL) { return TSDB_CODE_APP_ERROR; @@ -324,6 +328,13 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S return qa; } + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + + uint64_t suid = 0; + uint64_t uid = 0; + int32_t type = 0; + tableListGetSourceTableInfo(pTableScanInfo->base.pTableListInfo, &suid, &uid, &type); + // let's discard the tables those are not created according to the queried super table. SMetaReader mr = {0}; metaReaderInit(&mr, pScanInfo->readHandle.meta, 0); @@ -338,9 +349,21 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S tDecoderClear(&mr.coder); - // TODO handle ntb case - if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != pScanInfo->tableUid) { + if (mr.me.type == TSDB_SUPER_TABLE) { continue; + } else { + if (type == TSDB_SUPER_TABLE) { + // this new created child table does not belong to the scanned super table. + if (mr.me.type != TSDB_CHILD_TABLE || mr.me.ctbEntry.suid != suid) { + continue; + } + } else { // ordinary table + // In case that the scanned target table is an ordinary table. When replay the WAL during restore the vnode, we + // should check all newly created ordinary table to make sure that this table isn't the destination table. + if (mr.me.uid != uid) { + continue; + } + } } if (pScanInfo->pTagCond != NULL) { @@ -365,27 +388,23 @@ static SArray* filterUnqualifiedTables(const SStreamScanInfo* pScanInfo, const S return qa; } -int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { +int32_t qUpdateTableListForStreamScanner(qTaskInfo_t tinfo, const SArray* tableIdList, bool isAdd) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + const char* id = GET_TASKID(pTaskInfo); + int32_t code = 0; if (isAdd) { - qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), pTaskInfo->id.str); + qDebug("add %d tables id into query list, %s", (int32_t)taosArrayGetSize(tableIdList), id); } // traverse to the stream scanner node to add this table id - SOperatorInfo* pInfo = pTaskInfo->pRoot; - while (pInfo->operatorType != QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { - pInfo = pInfo->pDownstream[0]; - } - - int32_t code = 0; + SOperatorInfo* pInfo = extractOperatorInTree(pTaskInfo->pRoot, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); SStreamScanInfo* pScanInfo = pInfo->info; + if (isAdd) { // add new table id - SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, GET_TASKID(pTaskInfo)); + SArray* qa = filterUnqualifiedTables(pScanInfo, tableIdList, id); int32_t numOfQualifiedTables = taosArrayGetSize(qa); - - qDebug(" %d qualified child tables added into stream scanner", numOfQualifiedTables); - + qDebug("%d qualified child tables added into stream scanner, %s", numOfQualifiedTables, id); code = tqReaderAddTbUidList(pScanInfo->tqReader, qa); if (code != TSDB_CODE_SUCCESS) { taosArrayDestroy(qa); @@ -426,19 +445,6 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo } } -#if 0 - bool exists = false; - for (int32_t k = 0; k < taosArrayGetSize(pListInfo->pTableList); ++k) { - STableKeyInfo* pKeyInfo = taosArrayGet(pListInfo->pTableList, k); - if (pKeyInfo->uid == keyInfo.uid) { - qWarn("ignore duplicated query table uid:%" PRIu64 " added, %s", pKeyInfo->uid, pTaskInfo->id.str); - exists = true; - } - } - - if (!exists) { -#endif - tableListAddTableInfo(pTableListInfo, keyInfo.uid, keyInfo.groupId); } @@ -449,7 +455,7 @@ int32_t qUpdateQualifiedTableId(qTaskInfo_t tinfo, const SArray* tableIdList, bo taosArrayDestroy(qa); } else { // remove the table id in current list - qDebug(" %d remove child tables from the stream scanner", (int32_t)taosArrayGetSize(tableIdList)); + qDebug("%d remove child tables from the stream scanner, %s", (int32_t)taosArrayGetSize(tableIdList), id); taosWLockLatch(&pTaskInfo->lock); code = tqReaderRemoveTbUidList(pScanInfo->tqReader, tableIdList); taosWUnLockLatch(&pTaskInfo->lock); @@ -505,21 +511,14 @@ int32_t qCreateExecTask(SReadHandle* readHandle, int32_t vgId, uint64_t taskId, if (handle) { void* pSinkParam = NULL; - - SArray* pInfoList = getTableListInfo(*pTask); - STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); - taosArrayDestroy(pInfoList); - - code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, pTableListInfo, readHandle); + code = createDataSinkParam(pSubplan->pDataSink, &pSinkParam, (*pTask), readHandle); if (code != TSDB_CODE_SUCCESS) { qError("failed to createDataSinkParam, vgId:%d, code:%s, %s", vgId, tstrerror(code), (*pTask)->id.str); goto _error; } + // pSinkParam has been freed during create sinker. code = dsCreateDataSinker(pSubplan->pDataSink, handle, pSinkParam, (*pTask)->id.str); - if (code != TSDB_CODE_SUCCESS) { - taosMemoryFreeClear(pSinkParam); - } } qDebug("subplan task create completed, TID:0x%" PRIx64 " QID:0x%" PRIx64, taskId, pSubplan->id.queryId); @@ -1029,15 +1028,9 @@ SMqMetaRsp* qStreamExtractMetaMsg(qTaskInfo_t tinfo) { return &pTaskInfo->streamInfo.metaRsp; } -int64_t qStreamExtractPrepareUid(qTaskInfo_t tinfo) { - SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - return pTaskInfo->streamInfo.prepareStatus.uid; -} - -int32_t qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { +void qStreamExtractOffset(qTaskInfo_t tinfo, STqOffsetVal* pOffset) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; - memcpy(pOffset, &pTaskInfo->streamInfo.lastStatus, sizeof(STqOffsetVal)); - return 0; + memcpy(pOffset, &pTaskInfo->streamInfo.currentOffset, sizeof(STqOffsetVal)); } int32_t initQueryTableDataCondForTmq(SQueryTableDataCond* pCond, SSnapContext* sContext, SMetaTableInfo* pMtInfo) { @@ -1075,6 +1068,7 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; if ((pTaskInfo->execModel != OPTR_EXEC_MODEL_QUEUE) || (pTaskInfo->streamInfo.submit.msgStr != NULL)) { qError("qStreamSetScanMemData err:%d,%p", pTaskInfo->execModel, pTaskInfo->streamInfo.submit.msgStr); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } qDebug("set the submit block for future scan"); @@ -1083,22 +1077,27 @@ int32_t qStreamSetScanMemData(qTaskInfo_t tinfo, SPackedData submit) { return 0; } +void qStreamSetOpen(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; + SOperatorInfo* pOperator = pTaskInfo->pRoot; + pOperator->status = OP_NOT_OPENED; +} + int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subType) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)tinfo; SOperatorInfo* pOperator = pTaskInfo->pRoot; const char* id = GET_TASKID(pTaskInfo); - pTaskInfo->streamInfo.prepareStatus = *pOffset; - pTaskInfo->streamInfo.returned = 0; - - if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.lastStatus)) { + // if pOffset equal to current offset, means continue consume + if (tOffsetEqual(pOffset, &pTaskInfo->streamInfo.currentOffset)) { return 0; } if (subType == TOPIC_SUB_TYPE__COLUMN) { - pOperator->status = OP_OPENED; pOperator = extractOperatorInTree(pOperator, QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN, id); - + if (pOperator == NULL) { + return -1; + } SStreamScanInfo* pInfo = pOperator->info; STableScanInfo* pScanInfo = pInfo->pTableScanOp->info; STableScanBase* pScanBaseInfo = &pScanInfo->base; @@ -1109,7 +1108,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanBaseInfo->dataReader = NULL; // let's seek to the next version in wal file - if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, pTaskInfo->id.str) < 0) { + if (tqSeekVer(pInfo->tqReader, pOffset->version + 1, id) < 0) { qError("tqSeekVer failed ver:%" PRId64 ", %s", pOffset->version + 1, id); return -1; } @@ -1133,10 +1132,13 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else { taosRUnLockLatch(&pTaskInfo->lock); qError("no table in table list, %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } } + qDebug("switch to table uid:%" PRId64 " ts:%" PRId64 "% " PRId64 " rows returned", uid, ts, + pInfo->pTableScanOp->resultInfo.totalRows); pInfo->pTableScanOp->resultInfo.totalRows = 0; // start from current accessed position @@ -1150,6 +1152,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } else { qError("vgId:%d uid:%" PRIu64 " not found in table list, total:%d, index:%d %s", pTaskInfo->id.vgId, uid, numOfTables, pScanInfo->currentTable, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1182,6 +1185,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT pScanBaseInfo->cond.twindows.skey = oldSkey; } else { qError("invalid pOffset->type:%d, %s", pOffset->type, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1196,6 +1200,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT if (setForSnapShot(sContext, pOffset->uid) != 0) { qError("setDataForSnapShot error. uid:%" PRId64 " , %s", pOffset->uid, id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } @@ -1207,7 +1212,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT tableListClear(pTableListInfo); if (mtInfo.uid == 0) { - return 0; // no data + goto end; // no data } initQueryTableDataCondForTmq(&pTaskInfo->streamInfo.tableCond, sContext, &mtInfo); @@ -1232,6 +1237,7 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT SSnapContext* sContext = pInfo->sContext; if (setForSnapShot(sContext, pOffset->uid) != 0) { qError("setForSnapShot error. uid:%" PRIu64 " ,version:%" PRId64, pOffset->uid, pOffset->version); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return -1; } qDebug("tmqsnap qStreamPrepareScan snapshot meta uid:%" PRId64 " ts %" PRId64 " %s", pOffset->uid, pOffset->ts, @@ -1244,6 +1250,9 @@ int32_t qStreamPrepareScan(qTaskInfo_t tinfo, STqOffsetVal* pOffset, int8_t subT } } +end: + pTaskInfo->streamInfo.currentOffset = *pOffset; + return 0; } @@ -1270,3 +1279,22 @@ void qProcessRspMsg(void* parent, SRpcMsg* pMsg, SEpSet* pEpSet) { rpcFreeCont(pMsg->pCont); destroySendMsgInfo(pSendInfo); } + +SArray* qGetQueriedTableListInfo(qTaskInfo_t tinfo) { + SExecTaskInfo* pTaskInfo = tinfo; + SArray* plist = getTableListInfo(pTaskInfo); + + // only extract table in the first elements + STableListInfo* pTableListInfo = taosArrayGetP(plist, 0); + + SArray* pUidList = taosArrayInit(10, sizeof(uint64_t)); + + int32_t numOfTables = tableListGetSize(pTableListInfo); + for(int32_t i = 0; i < numOfTables; ++i) { + STableKeyInfo* pKeyInfo = tableListGetInfo(pTableListInfo, i); + taosArrayPush(pUidList, &pKeyInfo->uid); + } + + taosArrayDestroy(plist); + return pUidList; +} diff --git a/source/libs/executor/src/executorimpl.c b/source/libs/executor/src/executorimpl.c index 5105fbc181d7a4351492a9433bfd8fe2e4669708..7594079cfb03fbe7b8a3a06b21aa6b8b625deb23 100644 --- a/source/libs/executor/src/executorimpl.c +++ b/source/libs/executor/src/executorimpl.c @@ -998,6 +998,7 @@ int32_t getTableScanInfo(SOperatorInfo* pOperator, int32_t* order, int32_t* scan SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, const char* id) { if (pOperator == NULL) { qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return NULL; } @@ -1006,6 +1007,7 @@ SOperatorInfo* extractOperatorInTree(SOperatorInfo* pOperator, int32_t type, con } else { if (pOperator->pDownstream == NULL || pOperator->pDownstream[0] == NULL) { qError("invalid operator, failed to find tableScanOperator %s", id); + terrno = TSDB_CODE_PAR_INTERNAL_ERROR; return NULL; } @@ -1149,8 +1151,8 @@ void cleanupExprSupp(SExprSupp* pSupp) { void cleanupBasicInfo(SOptrBasicInfo* pInfo) { pInfo->pRes = blockDataDestroy(pInfo->pRes); } -char* buildTaskId(uint64_t taskId, uint64_t queryId) { - char* p = taosMemoryMalloc(64); +void buildTaskId(uint64_t taskId, uint64_t queryId, char* dst) { + char* p = dst; int32_t offset = 6; memcpy(p, "TID:0x", offset); @@ -1161,7 +1163,6 @@ char* buildTaskId(uint64_t taskId, uint64_t queryId) { offset += tintToHex(queryId, &p[offset]); p[offset] = 0; - return p; } SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t vgId, EOPTR_EXEC_MODEL model, @@ -1183,7 +1184,9 @@ SExecTaskInfo* doCreateExecTaskInfo(uint64_t queryId, uint64_t taskId, int32_t v taosInitRWLatch(&pTaskInfo->lock); pTaskInfo->id.vgId = vgId; pTaskInfo->id.queryId = queryId; - pTaskInfo->id.str = buildTaskId(taskId, queryId); + + pTaskInfo->id.str = taosMemoryMalloc(64); + buildTaskId(taskId, queryId, pTaskInfo->id.str); return pTaskInfo; } @@ -1591,8 +1594,7 @@ int32_t extractTableScanNode(SPhysiNode* pNode, STableScanPhysiNode** ppNode) { return -1; } -int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* pTableListInfo, - SReadHandle* readHandle) { +int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, SExecTaskInfo* pTask, SReadHandle* readHandle) { switch (pNode->type) { case QUERY_NODE_PHYSICAL_PLAN_QUERY_INSERT: { SInserterParam* pInserterParam = taosMemoryCalloc(1, sizeof(SInserterParam)); @@ -1610,23 +1612,26 @@ int32_t createDataSinkParam(SDataSinkNode* pNode, void** pParam, STableListInfo* return TSDB_CODE_OUT_OF_MEMORY; } - int32_t tbNum = tableListGetSize(pTableListInfo); + SArray* pInfoList = getTableListInfo(pTask); + STableListInfo* pTableListInfo = taosArrayGetP(pInfoList, 0); + taosArrayDestroy(pInfoList); + pDeleterParam->suid = tableListGetSuid(pTableListInfo); // TODO extract uid list - pDeleterParam->pUidList = taosArrayInit(tbNum, sizeof(uint64_t)); + int32_t numOfTables = tableListGetSize(pTableListInfo); + pDeleterParam->pUidList = taosArrayInit(numOfTables, sizeof(uint64_t)); if (NULL == pDeleterParam->pUidList) { taosMemoryFree(pDeleterParam); return TSDB_CODE_OUT_OF_MEMORY; } - for (int32_t i = 0; i < tbNum; ++i) { + for (int32_t i = 0; i < numOfTables; ++i) { STableKeyInfo* pTable = tableListGetInfo(pTableListInfo, i); taosArrayPush(pDeleterParam->pUidList, &pTable->uid); } *pParam = pDeleterParam; - break; } default: @@ -1973,11 +1978,11 @@ void qStreamCloseTsdbReader(void* task) { SExecTaskInfo* pTaskInfo = (SExecTaskInfo*)task; SOperatorInfo* pOp = pTaskInfo->pRoot; - qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.lastStatus.uid, - pTaskInfo->streamInfo.lastStatus.ts); + qDebug("stream close tsdb reader, reset status uid:%" PRId64 " ts:%" PRId64, pTaskInfo->streamInfo.currentOffset.uid, + pTaskInfo->streamInfo.currentOffset.ts); // todo refactor, other thread may already use this read to extract data. - pTaskInfo->streamInfo.lastStatus = (STqOffsetVal){0}; + pTaskInfo->streamInfo.currentOffset = (STqOffsetVal){0}; while (pOp->numOfDownstream == 1 && pOp->pDownstream[0]) { SOperatorInfo* pDownstreamOp = pOp->pDownstream[0]; if (pDownstreamOp->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { @@ -2004,7 +2009,11 @@ void qStreamCloseTsdbReader(void* task) { } static void extractTableList(SArray* pList, const SOperatorInfo* pOperator) { - if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { + if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_STREAM_SCAN) { + SStreamScanInfo* pScanInfo = pOperator->info; + STableScanInfo* pTableScanInfo = pScanInfo->pTableScanOp->info; + taosArrayPush(pList, &pTableScanInfo->base.pTableListInfo); + } else if (pOperator->operatorType == QUERY_NODE_PHYSICAL_PLAN_TABLE_SCAN) { STableScanInfo* pScanInfo = pOperator->info; taosArrayPush(pList, &pScanInfo->base.pTableListInfo); } else { diff --git a/source/libs/executor/src/groupoperator.c b/source/libs/executor/src/groupoperator.c index 3d9bacf39f3ea14e5e753d24dd618d6b53de0cbe..612ecb468424323bc9d359ff784ec14761d44b3e 100644 --- a/source/libs/executor/src/groupoperator.c +++ b/source/libs/executor/src/groupoperator.c @@ -871,9 +871,9 @@ SOperatorInfo* createPartitionOperatorInfo(SOperatorInfo* downstream, SPartition getBufferPgSize(pInfo->binfo.pRes->info.rowSize, &defaultPgsz, &defaultBufsz); if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NO_DISKSPACE; pTaskInfo->code = terrno; - qError("Create partition operator info failed since %s", terrstr(terrno)); + qError("Create partition operator info failed since %s, tempDir:%s", terrstr(), tsTempDir); goto _error; } diff --git a/source/libs/executor/src/projectoperator.c b/source/libs/executor/src/projectoperator.c index c943270df9dff636b90c9e955b0030c49eb0a5a2..86c49e0fc82b76a595480e61b53167aab4b81f18 100644 --- a/source/libs/executor/src/projectoperator.c +++ b/source/libs/executor/src/projectoperator.c @@ -227,17 +227,8 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { blockDataCleanup(pFinalRes); SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; - if (pTaskInfo->streamInfo.submit.msgStr) { - pOperator->status = OP_OPENED; - } if (pOperator->status == OP_EXEC_DONE) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - pOperator->status = OP_OPENED; - qDebug("projection in queue model, set status open and return null"); - return NULL; - } - return NULL; } @@ -263,23 +254,14 @@ SSDataBlock* doProjectOperation(SOperatorInfo* pOperator) { // The downstream exec may change the value of the newgroup, so use a local variable instead. SSDataBlock* pBlock = downstream->fpSet.getNextFn(downstream); if (pBlock == NULL) { - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE && pFinalRes->info.rows == 0) { - pOperator->status = OP_OPENED; - if (pOperator->status == OP_EXEC_RECV) { - continue; - } else { - return NULL; - } - } - qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, - pFinalRes->info.rows); + qDebug("set op close, exec %d, status %d rows %" PRId64 , pTaskInfo->execModel, pOperator->status, pFinalRes->info.rows); setOperatorCompleted(pOperator); break; } - if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { - qDebug("set status recv"); - pOperator->status = OP_EXEC_RECV; - } +// if (pTaskInfo->execModel == OPTR_EXEC_MODEL_QUEUE) { +// qDebug("set status recv"); +// pOperator->status = OP_EXEC_RECV; +// } // for stream interval if (pBlock->info.type == STREAM_RETRIEVE || pBlock->info.type == STREAM_DELETE_RESULT || diff --git a/source/libs/executor/src/scanoperator.c b/source/libs/executor/src/scanoperator.c index b9fd8512d0b0e854f4b264c79cebe4ebec261a5e..6d62d55024d090bb42d19baa9dd5fc7e580a441d 100644 --- a/source/libs/executor/src/scanoperator.c +++ b/source/libs/executor/src/scanoperator.c @@ -51,6 +51,16 @@ typedef struct STableMergeScanSortSourceParam { STsdbReader* dataReader; } STableMergeScanSortSourceParam; +typedef struct STableCountScanOperatorInfo { + SReadHandle readHandle; + SSDataBlock* pRes; + + STableCountScanSupp supp; + + int32_t currGrpIdx; + SArray* stbUidList; // when group by db_name and/or stable_name +} STableCountScanOperatorInfo; + static bool processBlockWithProbability(const SSampleExecInfo* pInfo); bool processBlockWithProbability(const SSampleExecInfo* pInfo) { @@ -320,7 +330,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca qDebug("%s data block skipped, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64 ", uid:%" PRIu64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows, pBlockInfo->id.uid); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); pCost->skipBlocks += 1; tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; @@ -331,7 +341,7 @@ static int32_t loadDataBlock(SOperatorInfo* pOperator, STableScanBase* pTableSca if (success) { // failed to load the block sma data, data block statistics does not exist, load data block instead qDebug("%s data block SMA loaded, brange:%" PRId64 "-%" PRId64 ", rows:%" PRId64, GET_TASKID(pTaskInfo), pBlockInfo->window.skey, pBlockInfo->window.ekey, pBlockInfo->rows); - doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, 1); + doSetTagColumnData(pTableScanInfo, pBlock, pTaskInfo, pBlock->info.rows); tsdbReleaseDataBlock(pTableScanInfo->dataReader); return TSDB_CODE_SUCCESS; } else { @@ -680,7 +690,7 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { } uint32_t status = 0; - int32_t code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); + code = loadDataBlock(pOperator, &pTableScanInfo->base, pBlock, &status); if (code != TSDB_CODE_SUCCESS) { T_LONG_JMP(pTaskInfo->env, code); } @@ -698,9 +708,9 @@ static SSDataBlock* doTableScanImpl(SOperatorInfo* pOperator) { // todo refactor /*pTableScanInfo->lastStatus.uid = pBlock->info.id.uid;*/ /*pTableScanInfo->lastStatus.ts = pBlock->info.window.ekey;*/ - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; + // pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; + // pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; + // pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; return pBlock; } @@ -890,9 +900,11 @@ static void destroyTableScanOperatorInfo(void* param) { SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SReadHandle* readHandle, STableListInfo* pTableListInfo, SExecTaskInfo* pTaskInfo) { + int32_t code = 0; STableScanInfo* pInfo = taosMemoryCalloc(1, sizeof(STableScanInfo)); SOperatorInfo* pOperator = taosMemoryCalloc(1, sizeof(SOperatorInfo)); if (pInfo == NULL || pOperator == NULL) { + code = TSDB_CODE_OUT_OF_MEMORY; goto _error; } @@ -900,7 +912,7 @@ SOperatorInfo* createTableScanOperatorInfo(STableScanPhysiNode* pTableScanNode, SDataBlockDescNode* pDescNode = pScanNode->node.pOutputDataBlockDesc; int32_t numOfCols = 0; - int32_t code = + code = extractColMatchInfo(pScanNode->pScanCols, pDescNode, &numOfCols, COL_MATCH_FROM_COL_ID, &pInfo->base.matchInfo); if (code != TSDB_CODE_SUCCESS) { goto _error; @@ -1625,18 +1637,16 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { if (pTaskInfo->streamInfo.submit.msgStr != NULL) { if (pInfo->tqReader->msg2.msgStr == NULL) { SPackedData submit = pTaskInfo->streamInfo.submit; - if (tqReaderSetSubmitReq2(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { - qError("submit msg messed up when initing stream submit block %p, %s", submit.msgStr, id); - pInfo->tqReader->msg2 = (SPackedData){0}; - pInfo->tqReader->setMsg = 0; - ASSERT(0); + if (tqReaderSetSubmitMsg(pInfo->tqReader, submit.msgStr, submit.msgLen, submit.ver) < 0) { + qError("submit msg messed up when initing stream submit block %p", submit.msgStr); + return NULL; } } blockDataCleanup(pInfo->pRes); SDataBlockInfo* pBlockInfo = &pInfo->pRes->info; - while (tqNextDataBlock2(pInfo->tqReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); @@ -1652,73 +1662,54 @@ static SSDataBlock* doQueueScan(SOperatorInfo* pOperator) { } pInfo->tqReader->msg2 = (SPackedData){0}; - pInfo->tqReader->setMsg = 0; pTaskInfo->streamInfo.submit = (SPackedData){0}; return NULL; } - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { SSDataBlock* pResult = doTableScan(pInfo->pTableScanOp); if (pResult && pResult->info.rows > 0) { - qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64 " %s", + qDebug("queue scan tsdb return %" PRId64 " rows min:%" PRId64 " max:%" PRId64 " wal curVersion:%" PRId64, pResult->info.rows, pResult->info.window.skey, pResult->info.window.ekey, - pInfo->tqReader->pWalReader->curVersion, id); - pTaskInfo->streamInfo.returned = 1; + pInfo->tqReader->pWalReader->curVersion); + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pResult->info.id.uid, pResult->info.window.ekey); return pResult; - } else { - // no data has return already, try to extract data in the WAL - if (!pTaskInfo->streamInfo.returned) { - STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; - tsdbReaderClose(pTSInfo->base.dataReader); - pTSInfo->base.dataReader = NULL; - tqOffsetResetToLog(&pTaskInfo->streamInfo.prepareStatus, pTaskInfo->streamInfo.snapshotVer); - - qDebug("queue scan tsdb over, switch to wal ver:%" PRId64 " %s", pTaskInfo->streamInfo.snapshotVer + 1, id); - if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { - tqOffsetResetToLog(&pTaskInfo->streamInfo.lastStatus, pTaskInfo->streamInfo.snapshotVer); - return NULL; - } - } else { - return NULL; - } } + STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; + tsdbReaderClose(pTSInfo->base.dataReader); + pTSInfo->base.dataReader = NULL; + qDebug("queue scan tsdb over, switch to wal ver %" PRId64 "", pTaskInfo->streamInfo.snapshotVer + 1); + if (tqSeekVer(pInfo->tqReader, pTaskInfo->streamInfo.snapshotVer + 1, pTaskInfo->id.str) < 0) { + return NULL; + } + tqOffsetResetToLog(&pTaskInfo->streamInfo.currentOffset, pTaskInfo->streamInfo.snapshotVer); } - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__LOG) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__LOG) { while (1) { SFetchRet ret = {0}; - terrno = 0; - - if (tqNextBlock(pInfo->tqReader, &ret) < 0) { - // if the end is reached, terrno is 0 - if (terrno != 0) { - qError("failed to get next log block since %s, %s", terrstr(), id); - } - } + tqNextBlock(pInfo->tqReader, &ret); + tqOffsetResetToLog( + &pTaskInfo->streamInfo.currentOffset, + pInfo->tqReader->pWalReader->curVersion - 1); // curVersion move to next, so currentOffset = curVersion - 1 if (ret.fetchType == FETCH_TYPE__DATA) { + qDebug("doQueueScan get data from log %" PRId64 " rows, version:%" PRId64, ret.data.info.rows, + pTaskInfo->streamInfo.currentOffset.version); blockDataCleanup(pInfo->pRes); setBlockIntoRes(pInfo, &ret.data, true); if (pInfo->pRes->info.rows > 0) { - pOperator->status = OP_EXEC_RECV; - qDebug("queue scan log return %" PRId64 " rows", pInfo->pRes->info.rows); + qDebug("doQueueScan get data from log %" PRId64 " rows, return, version:%" PRId64, pInfo->pRes->info.rows, + pTaskInfo->streamInfo.currentOffset.version); return pInfo->pRes; } - } else if (ret.fetchType == FETCH_TYPE__META) { - qError("unexpected ret.fetchType:%d", ret.fetchType); - continue; - } else if (ret.fetchType == FETCH_TYPE__NONE || - (ret.fetchType == FETCH_TYPE__SEP && pOperator->status == OP_EXEC_RECV)) { - pTaskInfo->streamInfo.lastStatus = ret.offset; - char formatBuf[80]; - tFormatOffset(formatBuf, 80, &ret.offset); - qDebug("queue scan log return null, offset %s", formatBuf); - pOperator->status = OP_OPENED; + } else if (ret.fetchType == FETCH_TYPE__NONE) { + qDebug("doQueueScan get none from log, return, version:%" PRId64, pTaskInfo->streamInfo.currentOffset.version); return NULL; } } } else { - qError("unexpected streamInfo prepare type: %d %s", pTaskInfo->streamInfo.prepareStatus.type, id); + qError("unexpected streamInfo prepare type: %d", pTaskInfo->streamInfo.currentOffset.type); return NULL; } } @@ -1821,7 +1812,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { /*resetTableScanInfo(pTSInfo, pWin);*/ tsdbReaderClose(pTSInfo->base.dataReader); - qDebug("4"); pTSInfo->base.dataReader = NULL; pInfo->pTableScanOp->status = OP_OPENED; @@ -1904,7 +1894,6 @@ static SSDataBlock* doStreamScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.recoverStep = STREAM_RECOVER_STEP__NONE; STableScanInfo* pTSInfo = pInfo->pTableScanOp->info; tsdbReaderClose(pTSInfo->base.dataReader); - qDebug("5"); pTSInfo->base.dataReader = NULL; @@ -1931,6 +1920,7 @@ FETCH_NEXT_BLOCK: if (pBlock->info.parTbName[0]) { streamStatePutParName(pTaskInfo->streamInfo.pState, pBlock->info.id.groupId, pBlock->info.parTbName); } + // TODO move into scan pBlock->info.calWin.skey = INT64_MIN; pBlock->info.calWin.ekey = INT64_MAX; @@ -2073,8 +2063,7 @@ FETCH_NEXT_BLOCK: int32_t current = pInfo->validBlockIndex++; SPackedData* pSubmit = taosArrayGet(pInfo->pBlockLists, current); - /*if (tqReaderSetDataMsg(pInfo->tqReader, pSubmit, 0) < 0) {*/ - if (tqReaderSetSubmitReq2(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { + if (tqReaderSetSubmitMsg(pInfo->tqReader, pSubmit->msgStr, pSubmit->msgLen, pSubmit->ver) < 0) { qError("submit msg messed up when initing stream submit block %p, current %d, total %d", pSubmit, current, totBlockNum); continue; @@ -2083,7 +2072,7 @@ FETCH_NEXT_BLOCK: blockDataCleanup(pInfo->pRes); - while (tqNextDataBlock2(pInfo->tqReader)) { + while (tqNextDataBlock(pInfo->tqReader)) { SSDataBlock block = {0}; int32_t code = tqRetrieveDataBlock2(&block, pInfo->tqReader, NULL); @@ -2165,7 +2154,7 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { pTaskInfo->streamInfo.metaRsp.metaRsp = NULL; qDebug("tmqsnap doRawScan called"); - if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_DATA) { + if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_DATA) { bool hasNext = false; if (pInfo->dataReader) { code = tsdbNextDataBlock(pInfo->dataReader, &hasNext); @@ -2187,28 +2176,23 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { } qDebug("tmqsnap doRawScan get data uid:%" PRId64 "", pBlock->info.id.uid); - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.lastStatus.uid = pBlock->info.id.uid; - pTaskInfo->streamInfo.lastStatus.ts = pBlock->info.window.ekey; + tqOffsetResetToData(&pTaskInfo->streamInfo.currentOffset, pBlock->info.id.uid, pBlock->info.window.ekey); return pBlock; } SMetaTableInfo mtInfo = getUidfromSnapShot(pInfo->sContext); + STqOffsetVal offset = {0}; if (mtInfo.uid == 0) { // read snapshot done, change to get data from wal qDebug("tmqsnap read snapshot done, change to get data from wal"); - pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__LOG; - pTaskInfo->streamInfo.lastStatus.version = pInfo->sContext->snapVersion; + tqOffsetResetToLog(&offset, pInfo->sContext->snapVersion); } else { - pTaskInfo->streamInfo.prepareStatus.uid = mtInfo.uid; - pTaskInfo->streamInfo.prepareStatus.ts = INT64_MIN; + tqOffsetResetToData(&offset, mtInfo.uid, INT64_MIN); qDebug("tmqsnap change get data uid:%" PRId64 "", mtInfo.uid); - qStreamPrepareScan(pTaskInfo, &pTaskInfo->streamInfo.prepareStatus, pInfo->sContext->subType); } + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); tDeleteSSchemaWrapper(mtInfo.schema); - qDebug("tmqsnap stream scan tsdb return null"); return NULL; - } else if (pTaskInfo->streamInfo.prepareStatus.type == TMQ_OFFSET__SNAPSHOT_META) { + } else if (pTaskInfo->streamInfo.currentOffset.type == TMQ_OFFSET__SNAPSHOT_META) { SSnapContext* sContext = pInfo->sContext; void* data = NULL; int32_t dataLen = 0; @@ -2220,16 +2204,12 @@ static SSDataBlock* doRawScan(SOperatorInfo* pOperator) { return NULL; } - if (!sContext->queryMetaOrData) { // change to get data next poll request - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; - pTaskInfo->streamInfo.lastStatus.uid = uid; - pTaskInfo->streamInfo.metaRsp.rspOffset.type = TMQ_OFFSET__SNAPSHOT_DATA; - pTaskInfo->streamInfo.metaRsp.rspOffset.uid = 0; - pTaskInfo->streamInfo.metaRsp.rspOffset.ts = INT64_MIN; + if (!sContext->queryMeta) { // change to get data next poll request + STqOffsetVal offset = {0}; + tqOffsetResetToData(&offset, 0, INT64_MIN); + qStreamPrepareScan(pTaskInfo, &offset, pInfo->sContext->subType); } else { - pTaskInfo->streamInfo.lastStatus.type = TMQ_OFFSET__SNAPSHOT_META; - pTaskInfo->streamInfo.lastStatus.uid = uid; - pTaskInfo->streamInfo.metaRsp.rspOffset = pTaskInfo->streamInfo.lastStatus; + tqOffsetResetToMeta(&pTaskInfo->streamInfo.currentOffset, uid); pTaskInfo->streamInfo.metaRsp.resMsgType = type; pTaskInfo->streamInfo.metaRsp.metaRspLen = dataLen; pTaskInfo->streamInfo.metaRsp.metaRsp = data; @@ -2443,7 +2423,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys if (pHandle->initTableReader) { pTSInfo->scanMode = TABLE_SCAN__TABLE_ORDER; pTSInfo->base.dataReader = NULL; - pTaskInfo->streamInfo.lastStatus.uid = -1; } if (pHandle->initTqReader) { @@ -2462,7 +2441,6 @@ SOperatorInfo* createStreamScanOperatorInfo(SReadHandle* pHandle, STableScanPhys } pInfo->readHandle = *pHandle; - pInfo->tableUid = pScanPhyNode->uid; pTaskInfo->streamInfo.snapshotVer = pHandle->version; pInfo->pCreateTbRes = buildCreateTableBlock(&pInfo->tbnameCalSup, &pInfo->tagCalSup); blockDataEnsureCapacity(pInfo->pCreateTbRes, 8); @@ -2534,6 +2512,51 @@ _error: return NULL; } +static void doTagScanOneTable(SOperatorInfo* pOperator, const SSDataBlock* pRes, int32_t count, SMetaReader* mr) { + SExecTaskInfo* pTaskInfo = pOperator->pTaskInfo; + STagScanInfo* pInfo = pOperator->info; + SExprInfo* pExprInfo = &pOperator->exprSupp.pExprInfo[0]; + + STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); + int32_t code = metaGetTableEntryByUid(mr, item->uid); + tDecoderClear(&(*mr).coder); + if (code != TSDB_CODE_SUCCESS) { + qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), + GET_TASKID(pTaskInfo)); + metaReaderClear(mr); + T_LONG_JMP(pTaskInfo->env, terrno); + } + + char str[512]; + for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { + SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); + + // refactor later + if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { + STR_TO_VARSTR(str, (*mr).me.name); + colDataSetVal(pDst, (count), str, false); + } else { // it is a tag value + STagVal val = {0}; + val.cid = pExprInfo[j].base.pParam[0].pCol->colId; + const char* p = metaGetTableTagVal((*mr).me.ctbEntry.pTags, pDst->info.type, &val); + + char* data = NULL; + if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { + data = tTagValToData((const STagVal*)p, false); + } else { + data = (char*)p; + } + colDataSetVal(pDst, (count), data, + (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); + + if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && + data != NULL) { + taosMemoryFree(data); + } + } + } +} + static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { if (pOperator->status == OP_EXEC_DONE) { return NULL; @@ -2558,48 +2581,22 @@ static SSDataBlock* doTagScan(SOperatorInfo* pOperator) { metaReaderInit(&mr, pInfo->readHandle.meta, 0); while (pInfo->curPos < size && count < pOperator->resultInfo.capacity) { - STableKeyInfo* item = tableListGetInfo(pInfo->pTableListInfo, pInfo->curPos); - int32_t code = metaGetTableEntryByUid(&mr, item->uid); - tDecoderClear(&mr.coder); - if (code != TSDB_CODE_SUCCESS) { - qError("failed to get table meta, uid:0x%" PRIx64 ", code:%s, %s", item->uid, tstrerror(terrno), - GET_TASKID(pTaskInfo)); - metaReaderClear(&mr); - T_LONG_JMP(pTaskInfo->env, terrno); - } - - for (int32_t j = 0; j < pOperator->exprSupp.numOfExprs; ++j) { - SColumnInfoData* pDst = taosArrayGet(pRes->pDataBlock, pExprInfo[j].base.resSchema.slotId); - - // refactor later - if (fmIsScanPseudoColumnFunc(pExprInfo[j].pExpr->_function.functionId)) { - STR_TO_VARSTR(str, mr.me.name); - colDataSetVal(pDst, count, str, false); - } else { // it is a tag value - STagVal val = {0}; - val.cid = pExprInfo[j].base.pParam[0].pCol->colId; - const char* p = metaGetTableTagVal(mr.me.ctbEntry.pTags, pDst->info.type, &val); - - char* data = NULL; - if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL) { - data = tTagValToData((const STagVal*)p, false); - } else { - data = (char*)p; - } - colDataSetVal(pDst, count, data, - (data == NULL) || (pDst->info.type == TSDB_DATA_TYPE_JSON && tTagIsJsonNull(data))); - - if (pDst->info.type != TSDB_DATA_TYPE_JSON && p != NULL && IS_VAR_DATA_TYPE(((const STagVal*)p)->type) && - data != NULL) { - taosMemoryFree(data); - } - } - } - - count += 1; + doTagScanOneTable(pOperator, pRes, count, &mr); + ++count; if (++pInfo->curPos >= size) { setOperatorCompleted(pOperator); } + // each table with tbname is a group, hence its own block, but only group when slimit exists for performance reason. + if (pInfo->pSlimit != NULL) { + if (pInfo->curPos < pInfo->pSlimit->offset) { + continue; + } + pInfo->pRes->info.id.groupId = calcGroupId(mr.me.name, strlen(mr.me.name)); + if (pInfo->curPos >= (pInfo->pSlimit->offset + pInfo->pSlimit->limit) - 1) { + setOperatorCompleted(pOperator); + } + break; + } } metaReaderClear(&mr); @@ -2650,6 +2647,7 @@ SOperatorInfo* createTagScanOperatorInfo(SReadHandle* pReadHandle, STagScanPhysi pInfo->pRes = createDataBlockFromDescNode(pDescNode); pInfo->readHandle = *pReadHandle; pInfo->curPos = 0; + pInfo->pSlimit = (SLimitNode*)pPhyNode->node.pSlimit; //TODO: slimit now only indicate group setOperatorInfo(pOperator, "TagScanOperator", QUERY_NODE_PHYSICAL_PLAN_TAG_SCAN, false, OP_NOT_OPENED, pInfo, pTaskInfo); diff --git a/source/libs/executor/src/tfill.c b/source/libs/executor/src/tfill.c index f4c8521b941b4e82e22195e2c42098705dacc251..e59ea253ccee8e3badda35445d4f681e76c1542e 100644 --- a/source/libs/executor/src/tfill.c +++ b/source/libs/executor/src/tfill.c @@ -44,7 +44,7 @@ static void setNotFillColumn(SFillInfo* pFillInfo, SColumnInfoData* pDstColInfo, } else { p = FILL_IS_ASC_FILL(pFillInfo) ? &pFillInfo->prev : &pFillInfo->next; } - + SGroupKeys* pKey = taosArrayGet(p->pRowVal, colIdx); doSetVal(pDstColInfo, rowIndex, pKey); } @@ -578,7 +578,12 @@ int32_t taosGetLinearInterpolationVal(SPoint* point, int32_t outputType, SPoint* GET_TYPED_DATA(v1, double, inputType, point1->val); GET_TYPED_DATA(v2, double, inputType, point2->val); - double r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); + double r = 0; + if (!IS_BOOLEAN_TYPE(inputType)) { + r = DO_INTERPOLATION(v1, v2, point1->key, point2->key, point->key); + } else { + r = (v1 < 1 || v2 < 1) ? 0 : 1; + } SET_TYPED_DATA(point->val, outputType, r); return TSDB_CODE_SUCCESS; @@ -630,6 +635,7 @@ SFillColInfo* createFillColInfo(SExprInfo* pExpr, int32_t numOfFillExpr, SExprIn nodesValueNodeToVariant(pv, &pFillCol[i].fillVal); } } + pFillCol->numOfFillExpr = numOfFillExpr; for (int32_t i = 0; i < numOfNoFillExpr; ++i) { SExprInfo* pExprInfo = &pNotFillExpr[i]; diff --git a/source/libs/executor/src/timesliceoperator.c b/source/libs/executor/src/timesliceoperator.c index 911700be8534d7b6b7762854b82bd7ceaad12489..f0e25d8cc5a3786a66720de1e314c406361af11a 100644 --- a/source/libs/executor/src/timesliceoperator.c +++ b/source/libs/executor/src/timesliceoperator.c @@ -156,6 +156,16 @@ static FORCE_INLINE int32_t timeSliceEnsureBlockCapacity(STimeSliceOperatorInfo* return TSDB_CODE_SUCCESS; } +static bool isIrowtsPseudoColumn(SExprInfo* pExprInfo) { + char *name = pExprInfo->pExpr->_function.functionName; + return (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_irowts") == 0); +} + +static bool isIsfilledPseudoColumn(SExprInfo* pExprInfo) { + char *name = pExprInfo->pExpr->_function.functionName; + return (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type) && strcasecmp(name, "_isfilled") == 0); +} + static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* pExprSup, SSDataBlock* pResBlock, bool beforeTs) { int32_t rows = pResBlock->info.rows; @@ -170,10 +180,10 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + if (isIrowtsPseudoColumn(pExprInfo)) { colDataSetVal(pDst, rows, (char*)&pSliceInfo->current, false); continue; - } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + } else if (isIsfilledPseudoColumn(pExprInfo)) { bool isFilled = true; colDataAppend(pDst, pResBlock->info.rows, (char*)&isFilled, false); continue; @@ -203,6 +213,14 @@ static bool genInterpolationResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp int64_t v = 0; GET_TYPED_DATA(v, int64_t, pVar->nType, &pVar->i); colDataSetVal(pDst, rows, (char*)&v, false); + } else if (IS_BOOLEAN_TYPE(pDst->info.type)) { + bool v = false; + if (!IS_VAR_DATA_TYPE(pVar->nType)) { + GET_TYPED_DATA(v, bool, pVar->nType, &pVar->i); + } else { + v = taosStr2Int8(varDataVal(pVar->pz), NULL, 10); + } + colDataSetVal(pDst, rows, (char*)&v, false); } break; } @@ -288,9 +306,9 @@ static void addCurrentRowToResult(STimeSliceOperatorInfo* pSliceInfo, SExprSupp* int32_t dstSlot = pExprInfo->base.resSchema.slotId; SColumnInfoData* pDst = taosArrayGet(pResBlock->pDataBlock, dstSlot); - if (IS_TIMESTAMP_TYPE(pExprInfo->base.resSchema.type)) { + if (isIrowtsPseudoColumn(pExprInfo)) { colDataSetVal(pDst, pResBlock->info.rows, (char*)&pSliceInfo->current, false); - } else if (IS_BOOLEAN_TYPE(pExprInfo->base.resSchema.type)) { + } else if (isIsfilledPseudoColumn(pExprInfo)) { bool isFilled = false; colDataSetVal(pDst, pResBlock->info.rows, (char*)&isFilled, false); } else { @@ -643,6 +661,9 @@ void destroyTimeSliceOperatorInfo(void* param) { taosArrayDestroy(pInfo->pLinearInfo); cleanupExprSupp(&pInfo->scalarSup); + for (int32_t i = 0; i < pInfo->pFillColInfo->numOfFillExpr; ++i) { + taosVariantDestroy(&pInfo->pFillColInfo[i].fillVal); + } taosMemoryFree(pInfo->pFillColInfo); taosMemoryFreeClear(param); } diff --git a/source/libs/executor/src/timewindowoperator.c b/source/libs/executor/src/timewindowoperator.c index 987a82436f92f942cc9b65001834112bea342270..7afda3f597350ef69be8f21a6d5ec932bba5cbcb 100644 --- a/source/libs/executor/src/timewindowoperator.c +++ b/source/libs/executor/src/timewindowoperator.c @@ -2333,9 +2333,15 @@ static int32_t getNextQualifiedFinalWindow(SInterval* pInterval, STimeWindow* pN return startPos; } +static void setStreamDataVersion(SExecTaskInfo* pTaskInfo, int64_t version, int64_t ckId) { + pTaskInfo->streamInfo.dataVersion = version; + pTaskInfo->streamInfo.checkPointId = ckId; +} + static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* pSDataBlock, uint64_t groupId, SSHashObj* pUpdatedMap) { SStreamIntervalOperatorInfo* pInfo = (SStreamIntervalOperatorInfo*)pOperatorInfo->info; + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); SResultRowInfo* pResultRowInfo = &(pInfo->binfo.resultRowInfo); SExecTaskInfo* pTaskInfo = pOperatorInfo->pTaskInfo; @@ -2432,10 +2438,8 @@ static void doStreamIntervalAggImpl(SOperatorInfo* pOperatorInfo, SSDataBlock* p updateTimeWindowInfo(&pInfo->twAggSup.timeWindowData, &nextWin, true); applyAggFunctionOnPartialTuples(pTaskInfo, pSup->pCtx, &pInfo->twAggSup.timeWindowData, startPos, forwardRows, pSDataBlock->info.rows, numOfOutput); - SWinKey key = { - .ts = nextWin.skey, - .groupId = groupId, - }; + + SWinKey key = { .ts = nextWin.skey, .groupId = groupId }; saveOutputBuf(pInfo->pState, &key, pResult, pInfo->aggSup.resultRowSize); releaseOutputBuf(pInfo->pState, &key, pResult); if (pInfo->delKey.ts > key.ts) { @@ -2503,6 +2507,7 @@ static SSDataBlock* doStreamFinalIntervalAgg(SOperatorInfo* pOperator) { clearFunctionContext(&pOperator->exprSupp); // semi interval operator clear disk buffer clearStreamIntervalOperator(pInfo); + setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); qDebug("===stream===clear semi operator"); } else { deleteIntervalDiscBuf(pInfo->pState, pInfo->pPullDataMap, pInfo->twAggSup.maxTs - pInfo->twAggSup.deleteMark, @@ -2776,6 +2781,7 @@ SOperatorInfo* createStreamFinalIntervalOperatorInfo(SOperatorInfo* downstream, pInfo->numOfDatapack = 0; pInfo->pUpdated = NULL; pInfo->pUpdatedMap = NULL; + pInfo->dataVersion = 0; pOperator->operatorType = pPhyNode->type; pOperator->blocking = true; @@ -2905,8 +2911,8 @@ int32_t initStreamAggSupporter(SStreamAggSupporter* pSup, SqlFunctionCtx* pCtx, bufSize = pageSize * 4; } if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - qError("Init stream agg supporter failed since %s", terrstr(terrno)); + terrno = TSDB_CODE_NO_DISKSPACE; + qError("Init stream agg supporter failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } int32_t code = createDiskbasedBuf(&pSup->pResultBuf, pageSize, bufSize, "function", tsTempDir); @@ -3126,6 +3132,8 @@ static void doStreamSessionAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSData int32_t rows = pSDataBlock->info.rows; int32_t winRows = 0; + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + SColumnInfoData* pStartTsCol = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); TSKEY* startTsCols = (int64_t*)pStartTsCol->pData; SColumnInfoData* pEndTsCol = NULL; @@ -3589,6 +3597,7 @@ SOperatorInfo* createStreamSessionAggOperatorInfo(SOperatorInfo* downstream, SPh pInfo->ignoreExpiredDataSaved = false; pInfo->pUpdated = NULL; pInfo->pStUpdated = NULL; + pInfo->dataVersion = 0; setOperatorInfo(pOperator, "StreamSessionWindowAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_SESSION, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -3899,6 +3908,9 @@ static void doStreamStateAggImpl(SOperatorInfo* pOperator, SSDataBlock* pSDataBl TSKEY* tsCols = NULL; SResultRow* pResult = NULL; int32_t winRows = 0; + + pInfo->dataVersion = TMAX(pInfo->dataVersion, pSDataBlock->info.version); + if (pSDataBlock->pDataBlock != NULL) { SColumnInfoData* pColDataInfo = taosArrayGet(pSDataBlock->pDataBlock, pInfo->primaryTsIndex); tsCols = (int64_t*)pColDataInfo->pData; @@ -4115,6 +4127,7 @@ SOperatorInfo* createStreamStateAggOperatorInfo(SOperatorInfo* downstream, SPhys pInfo->ignoreExpiredDataSaved = false; pInfo->pUpdated = NULL; pInfo->pSeUpdated = NULL; + pInfo->dataVersion = 0; setOperatorInfo(pOperator, "StreamStateAggOperator", QUERY_NODE_PHYSICAL_PLAN_STREAM_STATE, true, OP_NOT_OPENED, pInfo, pTaskInfo); @@ -4750,6 +4763,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { &pInfo->delKey); setOperatorCompleted(pOperator); streamStateCommit(pTaskInfo->streamInfo.pState); + setStreamDataVersion(pTaskInfo, pInfo->dataVersion, pInfo->pState->checkPointId); return NULL; } @@ -4758,6 +4772,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { if (!pInfo->pUpdated) { pInfo->pUpdated = taosArrayInit(4, sizeof(SWinKey)); } + if (!pInfo->pUpdatedMap) { _hash_fn_t hashFn = taosGetDefaultHashFunction(TSDB_DATA_TYPE_BINARY); pInfo->pUpdatedMap = tSimpleHashInit(1024, hashFn); @@ -4770,6 +4785,7 @@ static SSDataBlock* doStreamIntervalAgg(SOperatorInfo* pOperator) { pInfo->numOfDatapack = 0; break; } + pInfo->numOfDatapack++; printDataBlock(pBlock, "single interval recv"); diff --git a/source/libs/executor/src/tlinearhash.c b/source/libs/executor/src/tlinearhash.c index 2cba3855c71a39dae10625393fcfd52662860b1f..023583fcde62195727db3bdeca26b564a10e0cce 100644 --- a/source/libs/executor/src/tlinearhash.c +++ b/source/libs/executor/src/tlinearhash.c @@ -248,8 +248,8 @@ SLHashObj* tHashInit(int32_t inMemPages, int32_t pageSize, _hash_fn_t fn, int32_ } if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - printf("tHash Init failed since %s", terrstr(terrno)); + terrno = TSDB_CODE_NO_DISKSPACE; + printf("tHash Init failed since %s, tempDir:%s", terrstr(), tsTempDir); taosMemoryFree(pHashObj); return NULL; } diff --git a/source/libs/executor/src/tsort.c b/source/libs/executor/src/tsort.c index 6c8e581b3fe837aa1bf101bf8cf6748a2cac5f4d..da5f65fdf274aa4338359c000c7bb2e4807e8c8d 100644 --- a/source/libs/executor/src/tsort.c +++ b/source/libs/executor/src/tsort.c @@ -195,8 +195,8 @@ static int32_t doAddToBuf(SSDataBlock* pDataBlock, SSortHandle* pHandle) { if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - qError("Add to buf failed since %s", terrstr(terrno)); + terrno = TSDB_CODE_NO_DISKSPACE; + qError("Add to buf failed since %s, tempDir:%s", terrstr(), tsTempDir); return terrno; } @@ -261,9 +261,8 @@ static int32_t sortComparInit(SMsortComparParam* pParam, SArray* pSources, int32 // multi-pass internal merge sort is required if (pHandle->pBuf == NULL) { if (!osTempSpaceAvailable()) { - code = TSDB_CODE_NO_AVAIL_DISK; - terrno = code; - qError("Sort compare init failed since %s, %s", tstrerror(code), pHandle->idStr); + code = terrno = TSDB_CODE_NO_DISKSPACE; + qError("Sort compare init failed since %s, tempDir:%s, idStr:%s", terrstr(), tsTempDir, pHandle->idStr); return code; } diff --git a/source/libs/function/src/builtins.c b/source/libs/function/src/builtins.c index a36204bab7c10a67af4ddec40f61b21cde2d2842..fe98a1dd537af5fc48b66c431dcfe524fdefe972 100644 --- a/source/libs/function/src/builtins.c +++ b/source/libs/function/src/builtins.c @@ -1575,7 +1575,7 @@ static int32_t translateInterp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) uint8_t nodeType = nodeType(nodesListGetNode(pFunc->pParameterList, 0)); uint8_t paraType = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; - if (!IS_NUMERIC_TYPE(paraType) || QUERY_NODE_VALUE == nodeType) { + if ((!IS_NUMERIC_TYPE(paraType) && !IS_BOOLEAN_TYPE(paraType))|| QUERY_NODE_VALUE == nodeType) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } @@ -1933,14 +1933,35 @@ static int32_t translateToIso8601(SFunctionNode* pFunc, char* pErrBuf, int32_t l } static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int32_t len) { - if (1 != LIST_LENGTH(pFunc->pParameterList)) { + int32_t numOfParams = LIST_LENGTH(pFunc->pParameterList); + int16_t resType = TSDB_DATA_TYPE_BIGINT; + + if (1 != numOfParams && 2 != numOfParams) { return invaildFuncParaNumErrMsg(pErrBuf, len, pFunc->functionName); } - if (!IS_STR_DATA_TYPE(((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type)) { + uint8_t para1Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 0))->resType.type; + if (!IS_STR_DATA_TYPE(para1Type)) { return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); } + if (2 == numOfParams) { + uint8_t para2Type = ((SExprNode*)nodesListGetNode(pFunc->pParameterList, 1))->resType.type; + if (!IS_INTEGER_TYPE(para2Type)) { + return invaildFuncParaTypeErrMsg(pErrBuf, len, pFunc->functionName); + } + + SValueNode* pValue = (SValueNode*)nodesListGetNode(pFunc->pParameterList, 1); + if (pValue->datum.i == 1) { + resType = TSDB_DATA_TYPE_TIMESTAMP; + } else if (pValue->datum.i == 0) { + resType = TSDB_DATA_TYPE_BIGINT; + } else { + return buildFuncErrMsg(pErrBuf, len, TSDB_CODE_FUNC_FUNTION_ERROR, + "TO_UNIXTIMESTAMP function second parameter should be 0/1"); + } + } + // add database precision as param uint8_t dbPrec = pFunc->node.resType.precision; int32_t code = addDbPrecisonParam(&pFunc->pParameterList, dbPrec); @@ -1948,7 +1969,7 @@ static int32_t translateToUnixtimestamp(SFunctionNode* pFunc, char* pErrBuf, int return code; } - pFunc->node.resType = (SDataType){.bytes = tDataTypes[TSDB_DATA_TYPE_BIGINT].bytes, .type = TSDB_DATA_TYPE_BIGINT}; + pFunc->node.resType = (SDataType){.bytes = tDataTypes[resType].bytes, .type = resType}; return TSDB_CODE_SUCCESS; } @@ -3279,7 +3300,7 @@ const SBuiltinFuncDefinition funcMgtBuiltins[] = { { .name = "_irowts", .type = FUNCTION_TYPE_IROWTS, - .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC|FUNC_MGT_KEEP_ORDER_FUNC, + .classification = FUNC_MGT_PSEUDO_COLUMN_FUNC | FUNC_MGT_INTERP_PC_FUNC | FUNC_MGT_KEEP_ORDER_FUNC, .translateFunc = translateTimePseudoColumn, .getEnvFunc = getTimePseudoFuncEnv, .initFunc = NULL, diff --git a/source/libs/function/src/builtinsimpl.c b/source/libs/function/src/builtinsimpl.c index 01f81c9a99e9d00528691abec5a345881d9a1991..ead53aade6393e5c31012fd483f1a8d1764be5d5 100644 --- a/source/libs/function/src/builtinsimpl.c +++ b/source/libs/function/src/builtinsimpl.c @@ -855,7 +855,9 @@ int32_t setSelectivityValue(SqlFunctionCtx* pCtx, SSDataBlock* pBlock, const STu int32_t numOfCols = pCtx->subsidiaries.num; const char* p = loadTupleData(pCtx, pTuplePos); if (p == NULL) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NOT_FOUND; + qError("Load tuple data failed since %s, groupId:%" PRIu64 ", ts:%" PRId64, terrstr(), + pTuplePos->streamTupleKey.groupId, pTuplePos->streamTupleKey.ts); return terrno; } @@ -3353,7 +3355,7 @@ int32_t spreadFunction(SqlFunctionCtx* pCtx) { goto _spread_over; } double tmin = 0.0, tmax = 0.0; - if (IS_SIGNED_NUMERIC_TYPE(type)) { + if (IS_SIGNED_NUMERIC_TYPE(type) || IS_TIMESTAMP_TYPE(type)) { tmin = (double)GET_INT64_VAL(&pAgg->min); tmax = (double)GET_INT64_VAL(&pAgg->max); } else if (IS_FLOAT_TYPE(type)) { @@ -5098,7 +5100,9 @@ int32_t modeFinalize(SqlFunctionCtx* pCtx, SSDataBlock* pBlock) { if (maxCount != 0) { const char* pData = loadTupleData(pCtx, &resDataPos); if (pData == NULL) { - code = TSDB_CODE_NO_AVAIL_DISK; + code = terrno = TSDB_CODE_NOT_FOUND; + qError("Load tuple data failed since %s, groupId:%" PRIu64 ", ts:%" PRId64, terrstr(), + resDataPos.streamTupleKey.groupId, resDataPos.streamTupleKey.ts); modeFunctionCleanup(pInfo); return code; } diff --git a/source/libs/function/src/functionMgt.c b/source/libs/function/src/functionMgt.c index 94ab616dda984f8ebb87c4a8d709851292cd2881..18f6e8050b56e1c73a28cf64bb24c1da3844651f 100644 --- a/source/libs/function/src/functionMgt.c +++ b/source/libs/function/src/functionMgt.c @@ -174,6 +174,8 @@ bool fmIsSelectFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC bool fmIsTimelineFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_TIMELINE_FUNC); } +bool fmIsDateTimeFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_DATETIME_FUNC); } + bool fmIsPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_PSEUDO_COLUMN_FUNC); } bool fmIsScanPseudoColumnFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SCAN_PC_FUNC); } @@ -184,6 +186,7 @@ bool fmIsWindowClauseFunc(int32_t funcId) { return fmIsAggFunc(funcId) || fmIsWi bool fmIsIndefiniteRowsFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_INDEFINITE_ROWS_FUNC); } + bool fmIsSpecialDataRequiredFunc(int32_t funcId) { return isSpecificClassifyFunc(funcId, FUNC_MGT_SPECIAL_DATA_REQUIRED); } diff --git a/source/libs/function/src/tpercentile.c b/source/libs/function/src/tpercentile.c index de381fadbd6efa25494559c5a9f63ab6a0422cf9..2edfb33e81f22b50f7a9c1c653f13e1baf6af1c1 100644 --- a/source/libs/function/src/tpercentile.c +++ b/source/libs/function/src/tpercentile.c @@ -277,7 +277,7 @@ tMemBucket *tMemBucketCreate(int16_t nElemSize, int16_t dataType, double minval, resetSlotInfo(pBucket); if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; + terrno = TSDB_CODE_NO_DISKSPACE; // qError("MemBucket create disk based Buf failed since %s", terrstr(terrno)); tMemBucketDestroy(pBucket); return NULL; diff --git a/source/libs/function/src/tudf.c b/source/libs/function/src/tudf.c index fad118297e6412c054f5c22ecc82b8cb291aa1f5..6b70422ac8fe20bf8089eb8efba11ecc3c5e6ae5 100644 --- a/source/libs/function/src/tudf.c +++ b/source/libs/function/src/tudf.c @@ -343,7 +343,7 @@ typedef struct SUdfcFuncStub { char udfName[TSDB_FUNC_NAME_LEN + 1]; UdfcFuncHandle handle; int32_t refCount; - int64_t lastRefTime; + int64_t createTime; } SUdfcFuncStub; typedef struct SUdfcProxy { @@ -363,6 +363,7 @@ typedef struct SUdfcProxy { uv_mutex_t udfStubsMutex; SArray *udfStubs; // SUdfcFuncStub + SArray *expiredUdfStubs; //SUdfcFuncStub uv_mutex_t udfcUvMutex; int8_t initialized; @@ -959,7 +960,7 @@ int32_t udfcOpen(); int32_t udfcClose(); int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle); -void releaseUdfFuncHandle(char *udfName); +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle); int32_t cleanUpUdfs(); bool udfAggGetEnv(struct SFunctionNode *pFunc, SFuncExecEnv *pEnv); @@ -967,6 +968,8 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pRes int32_t udfAggProcess(struct SqlFunctionCtx *pCtx); int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock); +void cleanupNotExpiredUdfs(); +void cleanupExpiredUdfs(); int compareUdfcFuncSub(const void *elem1, const void *elem2) { SUdfcFuncStub *stub1 = (SUdfcFuncStub *)elem1; SUdfcFuncStub *stub2 = (SUdfcFuncStub *)elem2; @@ -982,16 +985,24 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { if (stubIndex != -1) { SUdfcFuncStub *foundStub = taosArrayGet(gUdfcProxy.udfStubs, stubIndex); UdfcFuncHandle handle = foundStub->handle; - if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { - *pHandle = foundStub->handle; - ++foundStub->refCount; - foundStub->lastRefTime = taosGetTimestampUs(); - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return 0; + int64_t currUs = taosGetTimestampUs(); + bool expired = (currUs - foundStub->createTime) >= 10 * 1000 * 1000; + if (!expired) { + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + *pHandle = foundStub->handle; + ++foundStub->refCount; + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return 0; + } else { + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", udfName, + foundStub->refCount, foundStub->createTime); + taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + } } else { - fnInfo("invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", udfName, - foundStub->refCount, foundStub->lastRefTime); + fnInfo("udf handle expired for %s, will setup udf. move it to expired list", udfName); taosArrayRemove(gUdfcProxy.udfStubs, stubIndex); + taosArrayPush(gUdfcProxy.expiredUdfStubs, foundStub); + taosArraySort(gUdfcProxy.expiredUdfStubs, compareUdfcFuncSub); } } *pHandle = NULL; @@ -1001,7 +1012,7 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { strncpy(stub.udfName, udfName, TSDB_FUNC_NAME_LEN); stub.handle = *pHandle; ++stub.refCount; - stub.lastRefTime = taosGetTimestampUs(); + stub.createTime = taosGetTimestampUs(); taosArrayPush(gUdfcProxy.udfStubs, &stub); taosArraySort(gUdfcProxy.udfStubs, compareUdfcFuncSub); } else { @@ -1012,32 +1023,51 @@ int32_t acquireUdfFuncHandle(char *udfName, UdfcFuncHandle *pHandle) { return code; } -void releaseUdfFuncHandle(char *udfName) { +void releaseUdfFuncHandle(char *udfName, UdfcFuncHandle handle) { uv_mutex_lock(&gUdfcProxy.udfStubsMutex); SUdfcFuncStub key = {0}; strncpy(key.udfName, udfName, TSDB_FUNC_NAME_LEN); SUdfcFuncStub *foundStub = taosArraySearch(gUdfcProxy.udfStubs, &key, compareUdfcFuncSub, TD_EQ); - if (!foundStub) { + SUdfcFuncStub *expiredStub = taosArraySearch(gUdfcProxy.expiredUdfStubs, &key, compareUdfcFuncSub, TD_EQ); + if (!foundStub && !expiredStub) { uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return; } - if (foundStub->refCount > 0) { + if (foundStub != NULL && foundStub->handle == handle && foundStub->refCount > 0) { --foundStub->refCount; } + if (expiredStub != NULL && expiredStub->handle == handle && expiredStub->refCount > 0) { + --expiredStub->refCount; + } uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); } -int32_t cleanUpUdfs() { - int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); - if (!initialized) { - return TSDB_CODE_SUCCESS; +void cleanupExpiredUdfs() { + int32_t i = 0; + SArray *expiredUdfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); + while (i < taosArrayGetSize(gUdfcProxy.expiredUdfStubs)) { + SUdfcFuncStub *stub = taosArrayGet(gUdfcProxy.expiredUdfStubs, i); + if (stub->refCount == 0) { + fnInfo("tear down udf. expired. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); + doTeardownUdf(stub->handle); + } else { + fnInfo("udf still in use. expired. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); + UdfcFuncHandle handle = stub->handle; + if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { + taosArrayPush(expiredUdfStubs, stub); + } else { + fnInfo("udf invalid handle for %s, expired. refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); + } + } + ++i; } + taosArrayDestroy(gUdfcProxy.expiredUdfStubs); + gUdfcProxy.expiredUdfStubs = expiredUdfStubs; +} - uv_mutex_lock(&gUdfcProxy.udfStubsMutex); - if (gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) { - uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); - return TSDB_CODE_SUCCESS; - } +void cleanupNotExpiredUdfs() { SArray *udfStubs = taosArrayInit(16, sizeof(SUdfcFuncStub)); int32_t i = 0; while (i < taosArrayGetSize(gUdfcProxy.udfStubs)) { @@ -1046,20 +1076,38 @@ int32_t cleanUpUdfs() { fnInfo("tear down udf. udf name: %s, handle: %p, ref count: %d", stub->udfName, stub->handle, stub->refCount); doTeardownUdf(stub->handle); } else { - fnInfo("udf still in use. udf name: %s, ref count: %d, last ref time: %" PRId64 ", handle: %p", stub->udfName, - stub->refCount, stub->lastRefTime, stub->handle); + fnInfo("udf still in use. udf name: %s, ref count: %d, create time: %" PRId64 ", handle: %p", stub->udfName, + stub->refCount, stub->createTime, stub->handle); UdfcFuncHandle handle = stub->handle; if (handle != NULL && ((SUdfcUvSession *)handle)->udfUvPipe != NULL) { taosArrayPush(udfStubs, stub); } else { - fnInfo("udf invalid handle for %s, refCount: %d, last ref time: %" PRId64 ". remove it from cache", - stub->udfName, stub->refCount, stub->lastRefTime); + fnInfo("udf invalid handle for %s, refCount: %d, create time: %" PRId64 ". remove it from cache", + stub->udfName, stub->refCount, stub->createTime); } } ++i; } taosArrayDestroy(gUdfcProxy.udfStubs); gUdfcProxy.udfStubs = udfStubs; +} + +int32_t cleanUpUdfs() { + int8_t initialized = atomic_load_8(&gUdfcProxy.initialized); + if (!initialized) { + return TSDB_CODE_SUCCESS; + } + + uv_mutex_lock(&gUdfcProxy.udfStubsMutex); + if ((gUdfcProxy.udfStubs == NULL || taosArrayGetSize(gUdfcProxy.udfStubs) == 0) && + (gUdfcProxy.expiredUdfStubs == NULL || taosArrayGetSize(gUdfcProxy.expiredUdfStubs) == 0)) { + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); + return TSDB_CODE_SUCCESS; + } + + cleanupNotExpiredUdfs(); + cleanupExpiredUdfs(); + uv_mutex_unlock(&gUdfcProxy.udfStubsMutex); return 0; } @@ -1075,7 +1123,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, code = doCallUdfScalarFunc(handle, input, numOfCols, output); if (code != TSDB_CODE_SUCCESS) { fnError("udfc scalar function execution failure"); - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1089,7 +1137,7 @@ int32_t callUdfScalarFunc(char *udfName, SScalarParam *input, int32_t numOfCols, code = TSDB_CODE_UDF_INVALID_OUTPUT_TYPE; } } - releaseUdfFuncHandle(udfName); + releaseUdfFuncHandle(udfName, handle); return code; } @@ -1122,7 +1170,7 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult SUdfInterBuf buf = {0}; if ((udfCode = doCallUdfAggInit(handle, &buf)) != 0) { fnError("udfAggInit error. step doCallUdfAggInit. udf code: %d", udfCode); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } if (buf.bufLen <= session->bufSize) { @@ -1131,10 +1179,10 @@ bool udfAggInit(struct SqlFunctionCtx *pCtx, struct SResultRowEntryInfo *pResult udfRes->interResNum = buf.numOfResult; } else { fnError("udfc inter buf size %d is greater than function bufSize %d", buf.bufLen, session->bufSize); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return false; } - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&buf); return true; } @@ -1191,7 +1239,7 @@ int32_t udfAggProcess(struct SqlFunctionCtx *pCtx) { taosArrayDestroy(pTempBlock->pDataBlock); taosMemoryFree(pTempBlock); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); freeUdfInterBuf(&newState); return udfCode; } @@ -1236,7 +1284,7 @@ int32_t udfAggFinalize(struct SqlFunctionCtx *pCtx, SSDataBlock *pBlock) { freeUdfInterBuf(&resultBuf); int32_t numOfResults = functionFinalizeWithResultBuf(pCtx, pBlock, udfRes->finalResBuf); - releaseUdfFuncHandle(pCtx->udfName); + releaseUdfFuncHandle(pCtx->udfName, handle); return udfCallCode == 0 ? numOfResults : udfCallCode; } @@ -1400,8 +1448,9 @@ void udfcUvHandleError(SClientUvConn *conn) { QUEUE_REMOVE(&task->procTaskQueue); uv_sem_post(&task->taskSem); } - - uv_close((uv_handle_t *)conn->pipe, onUdfcPipeClose); + if (!uv_is_closing((uv_handle_t *)conn->pipe)) { + uv_close((uv_handle_t *)conn->pipe, onUdfcPipeClose); + } } void onUdfcPipeRead(uv_stream_t *client, ssize_t nread, const uv_buf_t *buf) { @@ -1531,7 +1580,7 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { case UV_TASK_REQ_RSP: { uv_pipe_t *pipe = uvTask->pipe; if (pipe == NULL) { - code = TSDB_CODE_UDF_PIPE_NO_PIPE; + code = TSDB_CODE_UDF_PIPE_NOT_EXIST; } else { uv_write_t *write = taosMemoryMalloc(sizeof(uv_write_t)); write->data = pipe->data; @@ -1549,11 +1598,13 @@ int32_t udfcStartUvTask(SClientUvTaskNode *uvTask) { case UV_TASK_DISCONNECT: { uv_pipe_t *pipe = uvTask->pipe; if (pipe == NULL) { - code = TSDB_CODE_UDF_PIPE_NO_PIPE; + code = TSDB_CODE_UDF_PIPE_NOT_EXIST; } else { SClientUvConn *conn = pipe->data; QUEUE_INSERT_TAIL(&conn->taskQueue, &uvTask->connTaskQueue); - uv_close((uv_handle_t *)uvTask->pipe, onUdfcPipeClose); + if (!uv_is_closing((uv_handle_t *)uvTask->pipe)) { + uv_close((uv_handle_t *)uvTask->pipe, onUdfcPipeClose); + } code = 0; } break; @@ -1660,6 +1711,7 @@ int32_t udfcOpen() { uv_barrier_wait(&proxy->initBarrier); uv_mutex_init(&proxy->udfStubsMutex); proxy->udfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); + proxy->expiredUdfStubs = taosArrayInit(8, sizeof(SUdfcFuncStub)); uv_mutex_init(&proxy->udfcUvMutex); fnInfo("udfc initialized") return 0; } @@ -1676,6 +1728,7 @@ int32_t udfcClose() { uv_thread_join(&udfc->loopThread); uv_mutex_destroy(&udfc->taskQueueMutex); uv_barrier_destroy(&udfc->initBarrier); + taosArrayDestroy(udfc->expiredUdfStubs); taosArrayDestroy(udfc->udfStubs); uv_mutex_destroy(&udfc->udfStubsMutex); uv_mutex_destroy(&udfc->udfcUvMutex); @@ -1706,9 +1759,6 @@ int32_t udfcRunUdfUvTask(SClientUdfTask *task, int8_t uvTaskType) { } int32_t doSetupUdf(char udfName[], UdfcFuncHandle *funcHandle) { - if (gUdfcProxy.udfcState != UDFC_STATE_READY) { - return TSDB_CODE_UDF_INVALID_STATE; - } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; task->session = taosMemoryCalloc(1, sizeof(SUdfcUvSession)); @@ -1751,7 +1801,7 @@ int32_t callUdf(UdfcFuncHandle handle, int8_t callType, SSDataBlock *input, SUdf SUdfcUvSession *session = (SUdfcUvSession *)handle; if (session->udfUvPipe == NULL) { fnError("No pipe to udfd"); - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NOT_EXIST; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); task->errCode = 0; @@ -1875,7 +1925,7 @@ int32_t doTeardownUdf(UdfcFuncHandle handle) { if (session->udfUvPipe == NULL) { fnError("tear down udf. pipe to udfd does not exist. udf name: %s", session->udfName); taosMemoryFree(session); - return TSDB_CODE_UDF_PIPE_NO_PIPE; + return TSDB_CODE_UDF_PIPE_NOT_EXIST; } SClientUdfTask *task = taosMemoryCalloc(1, sizeof(SClientUdfTask)); diff --git a/source/libs/function/src/udfd.c b/source/libs/function/src/udfd.c index a8b993290e6367bf49a833f36a6455f3bae67abe..d2553ba96dbc242025fa9489da25fbbbee70e7f8 100644 --- a/source/libs/function/src/udfd.c +++ b/source/libs/function/src/udfd.c @@ -53,6 +53,45 @@ int32_t udfdCPluginOpen(SScriptUdfEnvItem *items, int numItems) { return 0; } int32_t udfdCPluginClose() { return 0; } +const char *udfdCPluginUdfInitLoadInitDestoryFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { + char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char *initSuffix = "_init"; + strcpy(initFuncName, udfName); + strncat(initFuncName, initSuffix, strlen(initSuffix)); + uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc)); + + char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; + char *destroySuffix = "_destroy"; + strcpy(destroyFuncName, udfName); + strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); + uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)); + return udfName; +} + +void udfdCPluginUdfInitLoadAggFuncs(SUdfCPluginCtx *udfCtx, const char *udfName) { + char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; + strcpy(processFuncName, udfName); + uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); + + char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *startSuffix = "_start"; + strncpy(startFuncName, processFuncName, sizeof(startFuncName)); + strncat(startFuncName, startSuffix, strlen(startSuffix)); + uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc)); + + char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; + char *finishSuffix = "_finish"; + strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); + strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); + uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc)); + + char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; + char *mergeSuffix = "_merge"; + strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); + strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); + uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); +} + int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { int32_t err = 0; SUdfCPluginCtx *udfCtx = taosMemoryCalloc(1, sizeof(SUdfCPluginCtx)); @@ -62,45 +101,19 @@ int32_t udfdCPluginUdfInit(SScriptUdfInfo *udf, void **pUdfCtx) { return TSDB_CODE_UDF_LOAD_UDF_FAILURE; } const char *udfName = udf->name; - char initFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; - char *initSuffix = "_init"; - strcpy(initFuncName, udfName); - strncat(initFuncName, initSuffix, strlen(initSuffix)); - uv_dlsym(&udfCtx->lib, initFuncName, (void **)(&udfCtx->initFunc)); - char destroyFuncName[TSDB_FUNC_NAME_LEN + 5] = {0}; - char *destroySuffix = "_destroy"; - strcpy(destroyFuncName, udfName); - strncat(destroyFuncName, destroySuffix, strlen(destroySuffix)); - uv_dlsym(&udfCtx->lib, destroyFuncName, (void **)(&udfCtx->destroyFunc)); + udfdCPluginUdfInitLoadInitDestoryFuncs(udfCtx, udfName); if (udf->funcType == UDF_FUNC_TYPE_SCALAR) { char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; strcpy(processFuncName, udfName); uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->scalarProcFunc)); } else if (udf->funcType == UDF_FUNC_TYPE_AGG) { - char processFuncName[TSDB_FUNC_NAME_LEN] = {0}; - strcpy(processFuncName, udfName); - uv_dlsym(&udfCtx->lib, processFuncName, (void **)(&udfCtx->aggProcFunc)); - char startFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; - char *startSuffix = "_start"; - strncpy(startFuncName, processFuncName, sizeof(startFuncName)); - strncat(startFuncName, startSuffix, strlen(startSuffix)); - uv_dlsym(&udfCtx->lib, startFuncName, (void **)(&udfCtx->aggStartFunc)); - char finishFuncName[TSDB_FUNC_NAME_LEN + 7] = {0}; - char *finishSuffix = "_finish"; - strncpy(finishFuncName, processFuncName, sizeof(finishFuncName)); - strncat(finishFuncName, finishSuffix, strlen(finishSuffix)); - uv_dlsym(&udfCtx->lib, finishFuncName, (void **)(&udfCtx->aggFinishFunc)); - char mergeFuncName[TSDB_FUNC_NAME_LEN + 6] = {0}; - char *mergeSuffix = "_merge"; - strncpy(mergeFuncName, processFuncName, sizeof(mergeFuncName)); - strncat(mergeFuncName, mergeSuffix, strlen(mergeSuffix)); - uv_dlsym(&udfCtx->lib, mergeFuncName, (void **)(&udfCtx->aggMergeFunc)); + udfdCPluginUdfInitLoadAggFuncs(udfCtx, udfName); } + int32_t code = 0; if (udfCtx->initFunc) { - // TODO: handle init call return error code = (udfCtx->initFunc)(); if (code != 0) { uv_dlclose(&udfCtx->lib); @@ -216,6 +229,7 @@ typedef struct SUdfdContext { SArray *residentFuncs; + char udfDataDir[PATH_MAX]; bool printVersion; } SUdfdContext; @@ -245,7 +259,9 @@ typedef struct SUvUdfWork { typedef enum { UDF_STATE_INIT = 0, UDF_STATE_LOADING, UDF_STATE_READY } EUdfState; typedef struct SUdf { - char name[TSDB_FUNC_NAME_LEN + 1]; + char name[TSDB_FUNC_NAME_LEN + 1]; + int32_t version; + int64_t createdTime; int8_t funcType; int8_t scriptType; @@ -263,9 +279,11 @@ typedef struct SUdf { SUdfScriptPlugin *scriptPlugin; void *scriptUdfCtx; + + int64_t lastFetchTime; // last fetch time in milliseconds + bool expired; } SUdf; -// TODO: add private udf structure. typedef struct SUdfcFuncHandle { SUdf *udf; } SUdfcFuncHandle; @@ -318,6 +336,9 @@ static void udfdCloseWalkCb(uv_handle_t *handle, void *arg); static int32_t udfdRun(); static void udfdConnectMnodeThreadFunc(void *args); +SUdf *udfdNewUdf(const char *udfName); +void udfdGetFuncBodyPath(const SUdf *udf, char *path); + void udfdInitializeCPlugin(SUdfScriptPlugin *plugin) { plugin->scriptType = TSDB_FUNC_SCRIPT_BIN_LIB; plugin->openFunc = udfdCPluginOpen; @@ -370,12 +391,13 @@ int32_t udfdInitializePythonPlugin(SUdfScriptPlugin *plugin) { } if (plugin->openFunc) { - int16_t lenPythonPath = strlen(tsUdfdLdLibPath) + strlen(tsTempDir) + 1 + 1; // tsTempDir:tsUdfdLdLibPath - char *pythonPath = taosMemoryMalloc(lenPythonPath); + int16_t lenPythonPath = + strlen(tsUdfdLdLibPath) + strlen(global.udfDataDir) + 1 + 1; // global.udfDataDir:tsUdfdLdLibPath + char *pythonPath = taosMemoryMalloc(lenPythonPath); #ifdef WINDOWS - snprintf(pythonPath, lenPythonPath, "%s;%s", tsTempDir, tsUdfdLdLibPath); + snprintf(pythonPath, lenPythonPath, "%s;%s", global.udfDataDir, tsUdfdLdLibPath); #else - snprintf(pythonPath, lenPythonPath, "%s:%s", tsTempDir, tsUdfdLdLibPath); + snprintf(pythonPath, lenPythonPath, "%s:%s", global.udfDataDir, tsUdfdLdLibPath); #endif SScriptUdfEnvItem items[] = {{"PYTHONPATH", pythonPath}, {"LOGDIR", tsLogDir}}; err = plugin->openFunc(items, 2); @@ -500,28 +522,14 @@ void convertUdf2UdfInfo(SUdf *udf, SScriptUdfInfo *udfInfo) { udfInfo->funcType = UDF_FUNC_TYPE_SCALAR; } udfInfo->name = udf->name; + udfInfo->version = udf->version; + udfInfo->createdTime = udf->createdTime; udfInfo->outputLen = udf->outputLen; udfInfo->outputType = udf->outputType; udfInfo->path = udf->path; udfInfo->scriptType = udf->scriptType; } -int32_t udfdRenameUdfFile(SUdf *udf) { - char newPath[PATH_MAX]; - if (udf->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) { - snprintf(newPath, PATH_MAX, "%s/lib%s.so", tsTempDir, udf->name); - } else if (udf->scriptType == TSDB_FUNC_SCRIPT_PYTHON) { - snprintf(newPath, PATH_MAX, "%s/%s.py", tsTempDir, udf->name); - } else { - return TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED; - } - int32_t code = taosRenameFile(udf->path, newPath); - if (code == 0) { - sprintf(udf->path, "%s", newPath); - } - return 0; -} - int32_t udfdInitUdf(char *udfName, SUdf *udf) { int32_t err = 0; err = udfdFillUdfInfoFromMNode(global.clientRpc, udfName, udf); @@ -546,8 +554,6 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) { uv_mutex_unlock(&global.scriptPluginsMutex); udf->scriptPlugin = global.scriptPlugins[udf->scriptType]; - udfdRenameUdfFile(udf); - SScriptUdfInfo info = {0}; convertUdf2UdfInfo(udf, &info); err = udf->scriptPlugin->udfInitFunc(&info, &udf->scriptUdfCtx); @@ -556,40 +562,60 @@ int32_t udfdInitUdf(char *udfName, SUdf *udf) { return err; } - fnInfo("udf init succeeded. name %s type %d context %p", udf->name, udf->scriptType, (void*)udf->scriptUdfCtx); + fnInfo("udf init succeeded. name %s type %d context %p", udf->name, udf->scriptType, (void *)udf->scriptUdfCtx); return 0; } +SUdf *udfdNewUdf(const char *udfName) { + SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); + udfNew->refCount = 1; + udfNew->lastFetchTime = taosGetTimestampMs(); + strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); + + udfNew->state = UDF_STATE_INIT; + uv_mutex_init(&udfNew->lock); + uv_cond_init(&udfNew->condReady); + + udfNew->resident = false; + udfNew->expired = false; + for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { + char *funcName = taosArrayGet(global.residentFuncs, i); + if (strcmp(udfName, funcName) == 0) { + udfNew->resident = true; + break; + } + } + return udfNew; +} + SUdf *udfdGetOrCreateUdf(const char *udfName) { - SUdf *udf = NULL; uv_mutex_lock(&global.udfsMutex); - SUdf **udfInHash = taosHashGet(global.udfsHash, udfName, strlen(udfName)); - if (udfInHash) { - ++(*udfInHash)->refCount; - udf = *udfInHash; - uv_mutex_unlock(&global.udfsMutex); - } else { - SUdf *udfNew = taosMemoryCalloc(1, sizeof(SUdf)); - udfNew->refCount = 1; - strncpy(udfNew->name, udfName, TSDB_FUNC_NAME_LEN); - - udfNew->state = UDF_STATE_INIT; - uv_mutex_init(&udfNew->lock); - uv_cond_init(&udfNew->condReady); - - udf = udfNew; - udf->resident = false; - for (int32_t i = 0; i < taosArrayGetSize(global.residentFuncs); ++i) { - char *funcName = taosArrayGet(global.residentFuncs, i); - if (strcmp(udfName, funcName) == 0) { - udf->resident = true; - break; - } + SUdf **pUdfHash = taosHashGet(global.udfsHash, udfName, strlen(udfName)); + int64_t currTime = taosGetTimestampMs(); + bool expired = false; + if (pUdfHash) { + expired = currTime - (*pUdfHash)->lastFetchTime > 10 * 1000; // 10s + if (!expired) { + ++(*pUdfHash)->refCount; + SUdf *udf = *pUdfHash; + uv_mutex_unlock(&global.udfsMutex); + fnInfo("udfd reuse existing udf. udf %s udf version %d, udf created time %" PRIx64, udf->name, udf->version, + udf->createdTime); + return udf; + } else { + (*pUdfHash)->expired = true; + taosHashRemove(global.udfsHash, udfName, strlen(udfName)); + fnInfo("udfd expired, check for new version. existing udf %s udf version %d, udf created time %" PRIx64, + (*pUdfHash)->name, (*pUdfHash)->version, (*pUdfHash)->createdTime); } - SUdf **pUdf = &udf; - taosHashPut(global.udfsHash, udfName, strlen(udfName), pUdf, POINTER_BYTES); - uv_mutex_unlock(&global.udfsMutex); } + + SUdf *udf = udfdNewUdf(udfName); + + SUdf **pUdf = &udf; + taosHashPut(global.udfsHash, udfName, strlen(udfName), pUdf, POINTER_BYTES); + uv_mutex_unlock(&global.udfsMutex); + return udf; } @@ -662,6 +688,8 @@ void udfdProcessCallRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { output.colMeta.type = udf->outputType; output.colMeta.precision = 0; output.colMeta.scale = 0; + udfColEnsureCapacity(&output, call->block.info.rows); + SUdfDataBlock input = {0}; convertDataBlockToUdfDataBlock(&call->block, &input); code = udf->scriptPlugin->udfScalarProcFunc(&input, &output, udf->scriptUdfCtx); @@ -760,13 +788,13 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { uv_mutex_lock(&global.udfsMutex); udf->refCount--; - if (udf->refCount == 0 && !udf->resident) { + if (udf->refCount == 0 && (!udf->resident || udf->expired)) { unloadUdf = true; taosHashRemove(global.udfsHash, udf->name, strlen(udf->name)); } uv_mutex_unlock(&global.udfsMutex); if (unloadUdf) { - fnInfo("udf teardown. udf name: %s type %d: context %p", udf->name, udf->scriptType, (void*)(udf->scriptUdfCtx)); + fnInfo("udf teardown. udf name: %s type %d: context %p", udf->name, udf->scriptType, (void *)(udf->scriptUdfCtx)); uv_cond_destroy(&udf->condReady); uv_mutex_destroy(&udf->lock); code = udf->scriptPlugin->udfDestroyFunc(udf->scriptUdfCtx); @@ -791,6 +819,61 @@ void udfdProcessTeardownRequest(SUvUdfWork *uvUdf, SUdfRequest *request) { return; } +void udfdGetFuncBodyPath(const SUdf *udf, char *path) { + if (udf->scriptType == TSDB_FUNC_SCRIPT_BIN_LIB) { +#ifdef WINDOWS + snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64 ".dll", global.udfDataDir, udf->name, udf->version, udf->createdTime); +#else + snprintf(path, PATH_MAX, "%s/lib%s_%d_%" PRIx64 ".so", global.udfDataDir, udf->name, udf->version, + udf->createdTime); +#endif + } else if (udf->scriptType == TSDB_FUNC_SCRIPT_PYTHON) { +#ifdef WINDOWS + snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64 ".py", global.udfDataDir, udf->name, udf->version, udf->createdTime); +#else + snprintf(path, PATH_MAX, "%s/%s_%d_%" PRIx64 ".py", global.udfDataDir, udf->name, udf->version, udf->createdTime); +#endif + } else { +#ifdef WINDOWS + snprintf(path, PATH_MAX, "%s%s_%d_%" PRIx64, global.udfDataDir, udf->name, udf->version, udf->createdTime); +#else + snprintf(path, PATH_MAX, "%s/lib%s_%d_%" PRIx64, global.udfDataDir, udf->name, udf->version, udf->createdTime); +#endif + } +} + +int32_t udfdSaveFuncBodyToFile(SFuncInfo *pFuncInfo, SUdf *udf) { + if (!osDataSpaceAvailable()) { + terrno = TSDB_CODE_NO_DISKSPACE; + fnError("udfd create shared library failed since %s", terrstr(terrno)); + return terrno; + } + + char path[PATH_MAX] = {0}; + udfdGetFuncBodyPath(udf, path); + bool fileExist = !(taosStatFile(path, NULL, NULL) < 0); + if (fileExist) { + strncpy(udf->path, path, PATH_MAX); + fnInfo("udfd func body file. reuse existing file %s", path); + return TSDB_CODE_SUCCESS; + } + + TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); + if (file == NULL) { + fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); + return TSDB_CODE_FILE_CORRUPTED; + } + int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); + if (count != pFuncInfo->codeSize) { + fnError("udfd write udf shared library failed"); + return TSDB_CODE_FILE_CORRUPTED; + } + taosCloseFile(&file); + + strncpy(udf->path, path, PATH_MAX); + return TSDB_CODE_SUCCESS; +} + void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { SUdfdRpcSendRecvInfo *msgInfo = (SUdfdRpcSendRecvInfo *)pMsg->info.ahandle; @@ -829,49 +912,25 @@ void udfdProcessRpcRsp(void *parent, SRpcMsg *pMsg, SEpSet *pEpSet) { } else if (msgInfo->rpcType == UDFD_RPC_RETRIVE_FUNC) { SRetrieveFuncRsp retrieveRsp = {0}; tDeserializeSRetrieveFuncRsp(pMsg->pCont, pMsg->contLen, &retrieveRsp); - if (retrieveRsp.pFuncInfos == NULL) { - goto _return; - } + SFuncInfo *pFuncInfo = (SFuncInfo *)taosArrayGet(retrieveRsp.pFuncInfos, 0); - // SUdf *udf = msgInfo->param; - SUdf *udf = msgInfo->param; + SUdf *udf = msgInfo->param; udf->funcType = pFuncInfo->funcType; udf->scriptType = pFuncInfo->scriptType; udf->outputType = pFuncInfo->outputType; udf->outputLen = pFuncInfo->outputLen; udf->bufSize = pFuncInfo->bufSize; - if (!osTempSpaceAvailable()) { - terrno = TSDB_CODE_NO_AVAIL_DISK; - msgInfo->code = terrno; - fnError("udfd create shared library failed since %s", terrstr(terrno)); - goto _return; - } - - char path[PATH_MAX] = {0}; -#ifdef WINDOWS - snprintf(path, sizeof(path), "%s%s", tsTempDir, pFuncInfo->name); -#else - snprintf(path, sizeof(path), "%s/%s", tsTempDir, pFuncInfo->name); -#endif - TdFilePtr file = taosOpenFile(path, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_READ | TD_FILE_TRUNC); - if (file == NULL) { - fnError("udfd write udf shared library: %s failed, error: %d %s", path, errno, strerror(errno)); - msgInfo->code = TSDB_CODE_FILE_CORRUPTED; - goto _return; - } - - int64_t count = taosWriteFile(file, pFuncInfo->pCode, pFuncInfo->codeSize); - if (count != pFuncInfo->codeSize) { - fnError("udfd write udf shared library failed"); - msgInfo->code = TSDB_CODE_FILE_CORRUPTED; - goto _return; + SFuncExtraInfo *pFuncExtraInfo = (SFuncExtraInfo *)taosArrayGet(retrieveRsp.pFuncExtraInfos, 0); + udf->version = pFuncExtraInfo->funcVersion; + udf->createdTime = pFuncExtraInfo->funcCreatedTime; + msgInfo->code = udfdSaveFuncBodyToFile(pFuncInfo, udf); + if (msgInfo->code != 0) { + udf->lastFetchTime = 0; } - taosCloseFile(&file); - strncpy(udf->path, path, PATH_MAX); tFreeSFuncInfo(pFuncInfo); taosArrayDestroy(retrieveRsp.pFuncInfos); - msgInfo->code = 0; + taosArrayDestroy(retrieveRsp.pFuncExtraInfos); } _return: @@ -1380,6 +1439,24 @@ int32_t udfdCleanup() { return 0; } +int32_t udfdCreateUdfSourceDir() { + snprintf(global.udfDataDir, PATH_MAX, "%s/.udf", tsDataDir); + int32_t code = taosMkDir(global.udfDataDir); + if (code != TSDB_CODE_SUCCESS) { + snprintf(global.udfDataDir, PATH_MAX, "%s/.udf", tsTempDir); + code = taosMkDir(global.udfDataDir); + } + fnInfo("udfd create udf source directory %s. result: %s", global.udfDataDir, tstrerror(code)); + + return code; +} + +int32_t udfdDestroyUdfSourceDir() { + fnInfo("destory udf source directory %s", global.udfDataDir); + taosRemoveDir(global.udfDataDir); + return 0; +} + int main(int argc, char *argv[]) { if (!taosCheckSystemIsLittleEnd()) { printf("failed to start since on non-little-end machines\n"); @@ -1408,10 +1485,15 @@ int main(int argc, char *argv[]) { initEpSetFromCfg(tsFirst, tsSecond, &global.mgmtEp); if (udfdOpenClientRpc() != 0) { - fnError("open rpc connection to mnode failure"); + fnError("open rpc connection to mnode failed"); return -3; } + if (udfdCreateUdfSourceDir() != 0) { + fnError("create udf source directory failed"); + return -4; + } + if (udfdUvInit() != 0) { fnError("uv init failure"); return -5; @@ -1425,6 +1507,7 @@ int main(int argc, char *argv[]) { udfdRun(); removeListeningPipe(); + udfdDestroyUdfSourceDir(); udfdCloseClientRpc(); udfdDeinitResidentFuncs(); diff --git a/source/libs/function/test/udf1.c b/source/libs/function/test/udf1.c index 7798a0bf3d957c03240a1f22914a54d0fdcf5b30..5b950879962b56dc38a1c26f8665c13cd6c29c9c 100644 --- a/source/libs/function/test/udf1.c +++ b/source/libs/function/test/udf1.c @@ -24,7 +24,7 @@ DLL_EXPORT int32_t udf1(SUdfDataBlock *block, SUdfColumn *resultCol) { } } if (j == block->numOfCols) { - int32_t luckyNum = 88; + int32_t luckyNum = 1; udfColDataSet(resultCol, i, (char *)&luckyNum, false); } } diff --git a/source/libs/nodes/src/nodesEqualFuncs.c b/source/libs/nodes/src/nodesEqualFuncs.c index 4e23999ec2c32e2c82a6b0c569b3ff5c60770ef7..156744ef1dceb9e47860de780904c54997023f5c 100644 --- a/source/libs/nodes/src/nodesEqualFuncs.c +++ b/source/libs/nodes/src/nodesEqualFuncs.c @@ -136,6 +136,7 @@ static bool logicConditionNodeEqual(const SLogicConditionNode* a, const SLogicCo static bool functionNodeEqual(const SFunctionNode* a, const SFunctionNode* b) { COMPARE_SCALAR_FIELD(funcId); + COMPARE_STRING_FIELD(functionName); COMPARE_NODE_LIST_FIELD(pParameterList); return true; } diff --git a/source/libs/nodes/src/nodesUtilFuncs.c b/source/libs/nodes/src/nodesUtilFuncs.c index 4180ba1d6c6879a5589f532784f711a28e740d3d..3f571e22ae720d3dd9b18d33da7a8b42c60d2b48 100644 --- a/source/libs/nodes/src/nodesUtilFuncs.c +++ b/source/libs/nodes/src/nodesUtilFuncs.c @@ -925,6 +925,7 @@ void nodesDestroyNode(SNode* pNode) { taosMemoryFree(((SDescribeStmt*)pNode)->pMeta); break; case QUERY_NODE_RESET_QUERY_CACHE_STMT: // no pointer field + break; case QUERY_NODE_COMPACT_DATABASE_STMT: { SCompactDatabaseStmt* pStmt = (SCompactDatabaseStmt*)pNode; nodesDestroyNode(pStmt->pStart); @@ -942,10 +943,10 @@ void nodesDestroyNode(SNode* pNode) { nodesDestroyNode(pStmt->pSubtable); break; } - case QUERY_NODE_DROP_STREAM_STMT: // no pointer field - case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field + case QUERY_NODE_DROP_STREAM_STMT: // no pointer field + case QUERY_NODE_BALANCE_VGROUP_STMT: // no pointer field case QUERY_NODE_BALANCE_VGROUP_LEADER_STMT: // no pointer field - case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field + case QUERY_NODE_MERGE_VGROUP_STMT: // no pointer field break; case QUERY_NODE_REDISTRIBUTE_VGROUP_STMT: nodesDestroyList(((SRedistributeVgroupStmt*)pNode)->pDnodes); diff --git a/source/libs/parser/inc/parAst.h b/source/libs/parser/inc/parAst.h index 69043e8915b3526548cf7976b4faea0ee3e8b986..e08b77e681918ee1b00c46120f9c239c62a2fbcd 100644 --- a/source/libs/parser/inc/parAst.h +++ b/source/libs/parser/inc/parAst.h @@ -82,6 +82,11 @@ typedef struct SAlterOption { SNodeList* pList; } SAlterOption; +typedef struct STokenPair { + SToken first; + SToken second; +} STokenPair; + extern SToken nil_token; void initAstCreateContext(SParseContext* pParseCxt, SAstCreateContext* pCxt); @@ -229,8 +234,10 @@ SNode* createMergeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId1, cons SNode* createRedistributeVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId, SNodeList* pDnodes); SNode* createSplitVgroupStmt(SAstCreateContext* pCxt, const SToken* pVgId); SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName); -SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName); -SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName); +SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond); +SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond); SNode* createDeleteStmt(SAstCreateContext* pCxt, SNode* pTable, SNode* pWhere); SNode* createInsertStmt(SAstCreateContext* pCxt, SNode* pTable, SNodeList* pCols, SNode* pQuery); diff --git a/source/libs/parser/inc/parUtil.h b/source/libs/parser/inc/parUtil.h index ce5a63f5d07d35fb7a432e878d60372b9bab6cf9..850571eea1533541318c4dc57c60b14311fcc5a8 100644 --- a/source/libs/parser/inc/parUtil.h +++ b/source/libs/parser/inc/parUtil.h @@ -97,9 +97,8 @@ int32_t reserveTableVgroupInCache(int32_t acctId, const char* pDb, const char* p int32_t reserveTableVgroupInCacheExt(const SName* pName, SParseMetaCache* pMetaCache); int32_t reserveDbVgVersionInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); int32_t reserveDbCfgInCache(int32_t acctId, const char* pDb, SParseMetaCache* pMetaCache); -int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, +int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, SParseMetaCache* pMetaCache); -int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache); int32_t reserveUdfInCache(const char* pFunc, SParseMetaCache* pMetaCache); int32_t reserveTableIndexInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); int32_t reserveTableCfgInCache(int32_t acctId, const char* pDb, const char* pTable, SParseMetaCache* pMetaCache); @@ -110,8 +109,7 @@ int32_t getTableVgroupFromCache(SParseMetaCache* pMetaCache, const SName* pName, int32_t getDbVgVersionFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, int32_t* pVersion, int64_t* pDbId, int32_t* pTableNum, int64_t* pStateTs); int32_t getDbCfgFromCache(SParseMetaCache* pMetaCache, const char* pDbFName, SDbCfgInfo* pInfo); -int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, - bool* pPass); +int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes); int32_t getUdfInfoFromCache(SParseMetaCache* pMetaCache, const char* pFunc, SFuncInfo* pInfo); int32_t getTableIndexFromCache(SParseMetaCache* pMetaCache, const SName* pName, SArray** pIndexes); int32_t getTableCfgFromCache(SParseMetaCache* pMetaCache, const SName* pName, STableCfg** pOutput); diff --git a/source/libs/parser/inc/sql.y b/source/libs/parser/inc/sql.y index ebd8f51f900ad6b244cd9c66894f91ff20c7d266..d7a6baaffeacd0f9c009467b3f41e4a49d335263 100644 --- a/source/libs/parser/inc/sql.y +++ b/source/libs/parser/inc/sql.y @@ -94,8 +94,8 @@ sysinfo_opt(A) ::= . sysinfo_opt(A) ::= SYSINFO NK_INTEGER(B). { A = taosStr2Int8(B.z, NULL, 10); } /************************************************ grant/revoke ********************************************************/ -cmd ::= GRANT privileges(A) ON priv_level(B) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C); } -cmd ::= REVOKE privileges(A) ON priv_level(B) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C); } +cmd ::= GRANT privileges(A) ON priv_level(B) with_opt(D) TO user_name(C). { pCxt->pRootNode = createGrantStmt(pCxt, A, &B, &C, D); } +cmd ::= REVOKE privileges(A) ON priv_level(B) with_opt(D) FROM user_name(C). { pCxt->pRootNode = createRevokeStmt(pCxt, A, &B, &C, D); } %type privileges { int64_t } %destructor privileges { } @@ -113,11 +113,15 @@ priv_type_list(A) ::= priv_type_list(B) NK_COMMA priv_type(C). priv_type(A) ::= READ. { A = PRIVILEGE_TYPE_READ; } priv_type(A) ::= WRITE. { A = PRIVILEGE_TYPE_WRITE; } -%type priv_level { SToken } +%type priv_level { STokenPair } %destructor priv_level { } -priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR. { A = B; } -priv_level(A) ::= db_name(B) NK_DOT NK_STAR. { A = B; } -priv_level(A) ::= topic_name(B). { A = B; } +priv_level(A) ::= NK_STAR(B) NK_DOT NK_STAR(C). { A.first = B; A.second = C; } +priv_level(A) ::= db_name(B) NK_DOT NK_STAR(C). { A.first = B; A.second = C; } +priv_level(A) ::= db_name(B) NK_DOT table_name(C). { A.first = B; A.second = C; } +priv_level(A) ::= topic_name(B). { A.first = B; A.second = nil_token; } + +with_opt(A) ::= . { A = NULL; } +with_opt(A) ::= WITH search_condition(B). { A = B; } /************************************************ create/drop/alter dnode *********************************************/ cmd ::= CREATE DNODE dnode_endpoint(A). { pCxt->pRootNode = createCreateDnodeStmt(pCxt, &A, NULL); } @@ -342,7 +346,7 @@ column_def_list(A) ::= column_def(B). column_def_list(A) ::= column_def_list(B) NK_COMMA column_def(C). { A = addNodeToList(pCxt, B, C); } column_def(A) ::= column_name(B) type_name(C). { A = createColumnDefNode(pCxt, &B, C, NULL); } -column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } +//column_def(A) ::= column_name(B) type_name(C) COMMENT NK_STRING(D). { A = createColumnDefNode(pCxt, &B, C, &D); } %type type_name { SDataType } %destructor type_name { } @@ -1043,8 +1047,8 @@ sliding_opt(A) ::= SLIDING NK_LP duration_literal(B) NK_RP. fill_opt(A) ::= . { A = NULL; } fill_opt(A) ::= FILL NK_LP fill_mode(B) NK_RP. { A = createFillNode(pCxt, B, NULL); } -fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } -fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA literal_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, B)); } +fill_opt(A) ::= FILL NK_LP VALUE_F NK_COMMA expression_list(B) NK_RP. { A = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, B)); } %type fill_mode { EFillMode } %destructor fill_mode { } diff --git a/source/libs/parser/src/parAstCreater.c b/source/libs/parser/src/parAstCreater.c index 2ec942d890f5c865f8a0cda0f8077fdb954c78f8..2afe34c1f7fcc2a1ca91df04659e496d20731b9e 100644 --- a/source/libs/parser/src/parAstCreater.c +++ b/source/libs/parser/src/parAstCreater.c @@ -1994,29 +1994,39 @@ SNode* createSyncdbStmt(SAstCreateContext* pCxt, const SToken* pDbName) { return pStmt; } -SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName) { +SNode* createGrantStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false) || !checkUserName(pCxt, pUserName)) { + if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) { return NULL; } SGrantStmt* pStmt = (SGrantStmt*)nodesMakeNode(QUERY_NODE_GRANT_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->privileges = privileges; - COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName); + COPY_STRING_FORM_ID_TOKEN(pStmt->objName, &pPrivLevel->first); + if (TK_NK_NIL != pPrivLevel->second.type && TK_NK_STAR != pPrivLevel->second.type) { + COPY_STRING_FORM_ID_TOKEN(pStmt->tabName, &pPrivLevel->second); + } COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); + pStmt->pTagCond = pTagCond; return (SNode*)pStmt; } -SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, SToken* pDbName, SToken* pUserName) { +SNode* createRevokeStmt(SAstCreateContext* pCxt, int64_t privileges, STokenPair* pPrivLevel, SToken* pUserName, + SNode* pTagCond) { CHECK_PARSER_STATUS(pCxt); - if (!checkDbName(pCxt, pDbName, false) || !checkUserName(pCxt, pUserName)) { + if (!checkDbName(pCxt, &pPrivLevel->first, false) || !checkUserName(pCxt, pUserName)) { return NULL; } SRevokeStmt* pStmt = (SRevokeStmt*)nodesMakeNode(QUERY_NODE_REVOKE_STMT); CHECK_OUT_OF_MEM(pStmt); pStmt->privileges = privileges; - COPY_STRING_FORM_ID_TOKEN(pStmt->objName, pDbName); + COPY_STRING_FORM_ID_TOKEN(pStmt->objName, &pPrivLevel->first); + if (TK_NK_NIL != pPrivLevel->second.type && TK_NK_STAR != pPrivLevel->second.type) { + COPY_STRING_FORM_ID_TOKEN(pStmt->tabName, &pPrivLevel->second); + } COPY_STRING_FORM_ID_TOKEN(pStmt->userName, pUserName); + pStmt->pTagCond = pTagCond; return (SNode*)pStmt; } diff --git a/source/libs/parser/src/parAstParser.c b/source/libs/parser/src/parAstParser.c index cd4b455e025c3ff67bd22fddbc6e44936a35fbc3..5db1f5dbdbca70bed529373c3bb2caa76d74efd2 100644 --- a/source/libs/parser/src/parAstParser.c +++ b/source/libs/parser/src/parAstParser.c @@ -154,7 +154,8 @@ static int32_t collectMetaKeyFromRealTableImpl(SCollectMetaKeyCxt* pCxt, const c code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pDb, pTable, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, authType, pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pDb, pTable, authType, + pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, pDb, pCxt->pMetaCache); @@ -247,7 +248,7 @@ static int32_t collectMetaKeyFromCreateTable(SCollectMetaKeyCxt* pCxt, SCreateTa code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, AUTH_TYPE_WRITE, + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_WRITE, pCxt->pMetaCache); } return code; @@ -267,8 +268,8 @@ static int32_t collectMetaKeyFromCreateMultiTable(SCollectMetaKeyCxt* pCxt, SCre code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, AUTH_TYPE_WRITE, - pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, NULL, + AUTH_TYPE_WRITE, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS != code) { break; @@ -286,6 +287,10 @@ static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableS if (TSDB_CODE_SUCCESS == code) { code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pClause->dbName, pClause->tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pClause->dbName, + pClause->tableName, AUTH_TYPE_WRITE, pCxt->pMetaCache); + } if (TSDB_CODE_SUCCESS != code) { break; } @@ -293,6 +298,11 @@ static int32_t collectMetaKeyFromDropTable(SCollectMetaKeyCxt* pCxt, SDropTableS return code; } +static int32_t collectMetaKeyFromDropStable(SCollectMetaKeyCxt* pCxt, SDropSuperTableStmt* pStmt) { + return reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, + AUTH_TYPE_WRITE, pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTableStmt* pStmt) { int32_t code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { @@ -301,6 +311,10 @@ static int32_t collectMetaKeyFromAlterTable(SCollectMetaKeyCxt* pCxt, SAlterTabl if (TSDB_CODE_SUCCESS == code) { code = reserveTableVgroupInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, + AUTH_TYPE_WRITE, pCxt->pMetaCache); + } return code; } @@ -309,6 +323,10 @@ static int32_t collectMetaKeyFromAlterStable(SCollectMetaKeyCxt* pCxt, SAlterTab if (TSDB_CODE_SUCCESS == code) { code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pStmt->tableName, pCxt->pMetaCache); } + if (TSDB_CODE_SUCCESS == code) { + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, pStmt->tableName, + AUTH_TYPE_WRITE, pCxt->pMetaCache); + } return code; } @@ -439,8 +457,9 @@ static int32_t collectMetaKeyFromShowStables(SCollectMetaKeyCxt* pCxt, SShowStmt int32_t code = reserveTableMetaInCache(pCxt->pParseCxt->acctId, TSDB_INFORMATION_SCHEMA_DB, TSDB_INS_TABLE_STABLES, pCxt->pMetaCache); if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, - ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); + code = + reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal, + NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); } return code; } @@ -457,8 +476,9 @@ static int32_t collectMetaKeyFromShowTables(SCollectMetaKeyCxt* pCxt, SShowStmt* code = reserveDbVgInfoInCache(pCxt->pParseCxt->acctId, ((SValueNode*)pStmt->pDbName)->literal, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, - ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); + code = + reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, ((SValueNode*)pStmt->pDbName)->literal, + NULL, AUTH_TYPE_READ_OR_WRITE, pCxt->pMetaCache); } return code; } @@ -561,7 +581,8 @@ static int32_t collectMetaKeyFromShowCreateTable(SCollectMetaKeyCxt* pCxt, SShow code = reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } if (TSDB_CODE_SUCCESS == code) { - code = reserveUserAuthInCacheExt(pCxt->pParseCxt->pUser, &name, AUTH_TYPE_READ, pCxt->pMetaCache); + code = reserveUserAuthInCache(pCxt->pParseCxt->acctId, pCxt->pParseCxt->pUser, pStmt->dbName, NULL, AUTH_TYPE_READ, + pCxt->pMetaCache); } return code; } @@ -610,6 +631,13 @@ static int32_t collectMetaKeyFromCompactDatabase(SCollectMetaKeyCxt* pCxt, SComp return reserveDbCfgInCache(pCxt->pParseCxt->acctId, pStmt->dbName, pCxt->pMetaCache); } +static int32_t collectMetaKeyFromGrant(SCollectMetaKeyCxt* pCxt, SGrantStmt* pStmt) { + if ('\0' == pStmt->tabName[0]) { + return TSDB_CODE_SUCCESS; + } + return reserveTableMetaInCache(pCxt->pParseCxt->acctId, pStmt->objName, pStmt->tabName, pCxt->pMetaCache); +} + static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { pCxt->pStmt = pStmt; switch (nodeType(pStmt)) { @@ -627,6 +655,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); case QUERY_NODE_DROP_TABLE_STMT: return collectMetaKeyFromDropTable(pCxt, (SDropTableStmt*)pStmt); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return collectMetaKeyFromDropStable(pCxt, (SDropSuperTableStmt*)pStmt); case QUERY_NODE_ALTER_TABLE_STMT: return collectMetaKeyFromAlterTable(pCxt, (SAlterTableStmt*)pStmt); case QUERY_NODE_ALTER_SUPER_TABLE_STMT: @@ -645,6 +675,8 @@ static int32_t collectMetaKeyFromQuery(SCollectMetaKeyCxt* pCxt, SNode* pStmt) { return collectMetaKeyFromCompactDatabase(pCxt, (SCompactDatabaseStmt*)pStmt); case QUERY_NODE_CREATE_STREAM_STMT: return collectMetaKeyFromCreateStream(pCxt, (SCreateStreamStmt*)pStmt); + case QUERY_NODE_GRANT_STMT: + return collectMetaKeyFromGrant(pCxt, (SGrantStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: return collectMetaKeyFromShowDnodes(pCxt, (SShowStmt*)pStmt); case QUERY_NODE_SHOW_MNODES_STMT: diff --git a/source/libs/parser/src/parAuthenticator.c b/source/libs/parser/src/parAuthenticator.c index e4de60fd051d6f7f4e781160305ffb95a3ec7d3e..b06d48a690c47889bf6a7cc3e74ff56f5b338487 100644 --- a/source/libs/parser/src/parAuthenticator.c +++ b/source/libs/parser/src/parAuthenticator.c @@ -23,49 +23,112 @@ typedef struct SAuthCxt { int32_t errCode; } SAuthCxt; +typedef struct SSelectAuthCxt { + SAuthCxt* pAuthCxt; + SSelectStmt* pSelect; +} SSelectAuthCxt; + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt); -static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, AUTH_TYPE type) { +static void setUserAuthInfo(SParseContext* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, + SUserAuthInfo* pAuth) { + snprintf(pAuth->user, sizeof(pAuth->user), "%s", pCxt->pUser); + if (NULL == pTabName) { + tNameSetDbName(&pAuth->tbName, pCxt->acctId, pDbName, strlen(pDbName)); + } else { + toName(pCxt->acctId, pDbName, pTabName, &pAuth->tbName); + } + pAuth->type = type; +} + +static int32_t checkAuth(SAuthCxt* pCxt, const char* pDbName, const char* pTabName, AUTH_TYPE type, SNode** pCond) { SParseContext* pParseCxt = pCxt->pParseCxt; if (pParseCxt->isSuperUser) { return TSDB_CODE_SUCCESS; } - SName name; - tNameSetDbName(&name, pParseCxt->acctId, pDbName, strlen(pDbName)); - char dbFname[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(&name, dbFname); - int32_t code = TSDB_CODE_SUCCESS; - bool pass = false; + + SUserAuthInfo authInfo = {0}; + setUserAuthInfo(pCxt->pParseCxt, pDbName, pTabName, type, &authInfo); + int32_t code = TSDB_CODE_SUCCESS; + SUserAuthRes authRes = {0}; if (NULL != pCxt->pMetaCache) { - code = getUserAuthFromCache(pCxt->pMetaCache, pParseCxt->pUser, dbFname, type, &pass); + code = getUserAuthFromCache(pCxt->pMetaCache, &authInfo, &authRes); } else { SRequestConnInfo conn = {.pTrans = pParseCxt->pTransporter, .requestId = pParseCxt->requestId, .requestObjRefId = pParseCxt->requestRid, .mgmtEps = pParseCxt->mgmtEpSet}; - - code = catalogChkAuth(pParseCxt->pCatalog, &conn, pParseCxt->pUser, dbFname, type, &pass); + code = catalogChkAuth(pParseCxt->pCatalog, &conn, &authInfo, &authRes); + } + if (TSDB_CODE_SUCCESS == code && NULL != pCond) { + *pCond = authRes.pCond; } - return TSDB_CODE_SUCCESS == code ? (pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; + return TSDB_CODE_SUCCESS == code ? (authRes.pass ? TSDB_CODE_SUCCESS : TSDB_CODE_PAR_PERMISSION_DENIED) : code; } static EDealRes authSubquery(SAuthCxt* pCxt, SNode* pStmt) { return TSDB_CODE_SUCCESS == authQuery(pCxt, pStmt) ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } +static int32_t mergeStableTagCond(SNode** pWhere, SNode** pTagCond) { + SLogicConditionNode* pLogicCond = (SLogicConditionNode*)nodesMakeNode(QUERY_NODE_LOGIC_CONDITION); + if (NULL == pLogicCond) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pLogicCond->node.resType.type = TSDB_DATA_TYPE_BOOL; + pLogicCond->node.resType.bytes = tDataTypes[TSDB_DATA_TYPE_BOOL].bytes; + pLogicCond->condType = LOGIC_COND_TYPE_AND; + int32_t code = nodesListMakeStrictAppend(&pLogicCond->pParameterList, *pTagCond); + if (TSDB_CODE_SUCCESS == code) { + code = nodesListMakeAppend(&pLogicCond->pParameterList, *pWhere); + } + if (TSDB_CODE_SUCCESS == code) { + *pWhere = (SNode*)pLogicCond; + } else { + nodesDestroyNode((SNode*)pLogicCond); + } + return code; +} + +static int32_t appendStableTagCond(SNode** pWhere, SNode* pTagCond) { + SNode* pTagCondCopy = nodesCloneNode(pTagCond); + if (NULL == pTagCondCopy) { + return TSDB_CODE_OUT_OF_MEMORY; + } + + if (NULL == *pWhere) { + *pWhere = pTagCondCopy; + return TSDB_CODE_SUCCESS; + } + + if (QUERY_NODE_LOGIC_CONDITION == nodeType(*pWhere) && + LOGIC_COND_TYPE_AND == ((SLogicConditionNode*)*pWhere)->condType) { + return nodesListStrictAppend(((SLogicConditionNode*)*pWhere)->pParameterList, pTagCondCopy); + } + + return mergeStableTagCond(pWhere, &pTagCondCopy); +} + static EDealRes authSelectImpl(SNode* pNode, void* pContext) { - SAuthCxt* pCxt = pContext; + SSelectAuthCxt* pCxt = pContext; + SAuthCxt* pAuthCxt = pCxt->pAuthCxt; if (QUERY_NODE_REAL_TABLE == nodeType(pNode)) { - pCxt->errCode = checkAuth(pCxt, ((SRealTableNode*)pNode)->table.dbName, AUTH_TYPE_READ); - return TSDB_CODE_SUCCESS == pCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; + SNode* pTagCond = NULL; + STableNode* pTable = (STableNode*)pNode; + pAuthCxt->errCode = checkAuth(pAuthCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_READ, &pTagCond); + if (TSDB_CODE_SUCCESS == pAuthCxt->errCode && NULL != pTagCond) { + pAuthCxt->errCode = appendStableTagCond(&pCxt->pSelect->pWhere, pTagCond); + } + return TSDB_CODE_SUCCESS == pAuthCxt->errCode ? DEAL_RES_CONTINUE : DEAL_RES_ERROR; } else if (QUERY_NODE_TEMP_TABLE == nodeType(pNode)) { - return authSubquery(pCxt, ((STempTableNode*)pNode)->pSubquery); + return authSubquery(pAuthCxt, ((STempTableNode*)pNode)->pSubquery); } return DEAL_RES_CONTINUE; } static int32_t authSelect(SAuthCxt* pCxt, SSelectStmt* pSelect) { - nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, authSelectImpl, pCxt); + SSelectAuthCxt cxt = {.pAuthCxt = pCxt, .pSelect = pSelect}; + nodesWalkSelectStmt(pSelect, SQL_CLAUSE_FROM, authSelectImpl, &cxt); return pCxt->errCode; } @@ -85,11 +148,20 @@ static int32_t authDropUser(SAuthCxt* pCxt, SDropUserStmt* pStmt) { } static int32_t authDelete(SAuthCxt* pCxt, SDeleteStmt* pDelete) { - return checkAuth(pCxt, ((SRealTableNode*)pDelete->pFromTable)->table.dbName, AUTH_TYPE_WRITE); + SNode* pTagCond = NULL; + STableNode* pTable = (STableNode*)pDelete->pFromTable; + int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond); + if (TSDB_CODE_SUCCESS == code && NULL != pTagCond) { + code = appendStableTagCond(&pDelete->pWhere, pTagCond); + } + return code; } static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) { - int32_t code = checkAuth(pCxt, ((SRealTableNode*)pInsert->pTable)->table.dbName, AUTH_TYPE_WRITE); + SNode* pTagCond = NULL; + STableNode* pTable = (STableNode*)pInsert->pTable; + // todo check tag condition for subtable + int32_t code = checkAuth(pCxt, pTable->dbName, pTable->tableName, AUTH_TYPE_WRITE, &pTagCond); if (TSDB_CODE_SUCCESS == code) { code = authQuery(pCxt, pInsert->pQuery); } @@ -97,22 +169,27 @@ static int32_t authInsert(SAuthCxt* pCxt, SInsertStmt* pInsert) { } static int32_t authShowTables(SAuthCxt* pCxt, SShowStmt* pStmt) { - return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, AUTH_TYPE_READ_OR_WRITE); + return checkAuth(pCxt, ((SValueNode*)pStmt->pDbName)->literal, NULL, AUTH_TYPE_READ_OR_WRITE, NULL); } static int32_t authShowCreateTable(SAuthCxt* pCxt, SShowCreateTableStmt* pStmt) { - return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_READ); + SNode* pTagCond = NULL; + // todo check tag condition for subtable + return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_READ, &pTagCond); } static int32_t authCreateTable(SAuthCxt* pCxt, SCreateTableStmt* pStmt) { - return checkAuth(pCxt, pStmt->dbName, AUTH_TYPE_WRITE); + SNode* pTagCond = NULL; + // todo check tag condition for subtable + return checkAuth(pCxt, pStmt->dbName, NULL, AUTH_TYPE_WRITE, &pTagCond); } static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; SNode* pNode = NULL; FOREACH(pNode, pStmt->pSubTables) { - code = checkAuth(pCxt, ((SCreateSubTableClause*)pNode)->dbName, AUTH_TYPE_WRITE); + SCreateSubTableClause* pClause = (SCreateSubTableClause*)pNode; + code = checkAuth(pCxt, pClause->dbName, NULL, AUTH_TYPE_WRITE, NULL); if (TSDB_CODE_SUCCESS != code) { break; } @@ -120,6 +197,29 @@ static int32_t authCreateMultiTable(SAuthCxt* pCxt, SCreateMultiTablesStmt* pStm return code; } +static int32_t authDropTable(SAuthCxt* pCxt, SDropTableStmt* pStmt) { + int32_t code = TSDB_CODE_SUCCESS; + SNode* pNode = NULL; + FOREACH(pNode, pStmt->pTables) { + SDropTableClause* pClause = (SDropTableClause*)pNode; + code = checkAuth(pCxt, pClause->dbName, pClause->tableName, AUTH_TYPE_WRITE, NULL); + if (TSDB_CODE_SUCCESS != code) { + break; + } + } + return code; +} + +static int32_t authDropStable(SAuthCxt* pCxt, SDropSuperTableStmt* pStmt) { + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL); +} + +static int32_t authAlterTable(SAuthCxt* pCxt, SAlterTableStmt* pStmt) { + SNode* pTagCond = NULL; + // todo check tag condition for subtable + return checkAuth(pCxt, pStmt->dbName, pStmt->tableName, AUTH_TYPE_WRITE, NULL); +} + static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { switch (nodeType(pStmt)) { case QUERY_NODE_SET_OPERATOR: @@ -136,6 +236,13 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { return authCreateTable(pCxt, (SCreateTableStmt*)pStmt); case QUERY_NODE_CREATE_MULTI_TABLES_STMT: return authCreateMultiTable(pCxt, (SCreateMultiTablesStmt*)pStmt); + case QUERY_NODE_DROP_TABLE_STMT: + return authDropTable(pCxt, (SDropTableStmt*)pStmt); + case QUERY_NODE_DROP_SUPER_TABLE_STMT: + return authDropStable(pCxt, (SDropSuperTableStmt*)pStmt); + case QUERY_NODE_ALTER_TABLE_STMT: + case QUERY_NODE_ALTER_SUPER_TABLE_STMT: + return authAlterTable(pCxt, (SAlterTableStmt*)pStmt); case QUERY_NODE_SHOW_DNODES_STMT: case QUERY_NODE_SHOW_MNODES_STMT: case QUERY_NODE_SHOW_MODULES_STMT: @@ -146,7 +253,7 @@ static int32_t authQuery(SAuthCxt* pCxt, SNode* pStmt) { case QUERY_NODE_SHOW_LICENCES_STMT: case QUERY_NODE_SHOW_VGROUPS_STMT: case QUERY_NODE_SHOW_DB_ALIVE_STMT: - case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: + case QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT: case QUERY_NODE_SHOW_CREATE_DATABASE_STMT: case QUERY_NODE_SHOW_TABLE_DISTRIBUTED_STMT: case QUERY_NODE_SHOW_VNODES_STMT: diff --git a/source/libs/parser/src/parInsertSml.c b/source/libs/parser/src/parInsertSml.c index 0bcd777d2a4bd332a146e6cceb1bb464cdbbe573..1c921b2d7c2dcfee70200498b3ee10511331251d 100644 --- a/source/libs/parser/src/parInsertSml.c +++ b/source/libs/parser/src/parInsertSml.c @@ -125,6 +125,12 @@ static int32_t smlBuildTagRow(SArray* cols, SBoundColInfo* tags, SSchema* pSchem SSchema* pTagSchema = &pSchema[tags->pColIndex[i]]; SSmlKv* kv = taosArrayGet(cols, i); + if(kv->keyLen != strlen(pTagSchema->name) || memcmp(kv->key, pTagSchema->name, kv->keyLen) != 0 || kv->type != pTagSchema->type){ + code = TSDB_CODE_SML_INVALID_DATA; + uError("SML smlBuildCol error col not same %s", pTagSchema->name); + goto end; + } + taosArrayPush(*tagName, pTagSchema->name); STagVal val = {.cid = pTagSchema->colId, .type = pTagSchema->type}; // strcpy(val.colName, pTagSchema->name); diff --git a/source/libs/parser/src/parInsertSql.c b/source/libs/parser/src/parInsertSql.c index 18d65dd47711e445633a4b2969decfab5f5dd07e..eb2efd573d1767ccbe1dac7e20f5e712796a84d7 100644 --- a/source/libs/parser/src/parInsertSql.c +++ b/source/libs/parser/src/parInsertSql.c @@ -15,6 +15,7 @@ #include "parInsertUtil.h" #include "parToken.h" +#include "scalar.h" #include "tglobal.h" #include "ttime.h" @@ -565,6 +566,120 @@ static int32_t checkAndTrimValue(SToken* pToken, char* tmpTokenBuf, SMsgBuf* pMs return TSDB_CODE_SUCCESS; } +typedef struct SRewriteTagCondCxt { + SArray* pTagVals; + SArray* pTagName; + int32_t code; +} SRewriteTagCondCxt; + +static int32_t rewriteTagCondColumnImpl(STagVal* pVal, SNode** pNode) { + SValueNode* pValue = (SValueNode*)nodesMakeNode(QUERY_NODE_VALUE); + if (NULL == pValue) { + return TSDB_CODE_OUT_OF_MEMORY; + } + pValue->node.resType.type = pVal->type; + switch (pVal->type) { + case TSDB_DATA_TYPE_BOOL: + pValue->datum.b = *(int8_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_TINYINT: + pValue->datum.i = *(int8_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_SMALLINT: + pValue->datum.i = *(int16_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_INT: + pValue->datum.i = *(int32_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_BIGINT: + pValue->datum.i = pVal->i64; + break; + case TSDB_DATA_TYPE_FLOAT: + pValue->datum.d = *(float*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_DOUBLE: + pValue->datum.d = *(double*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_VARCHAR: + case TSDB_DATA_TYPE_NCHAR: + pValue->datum.p = taosMemoryCalloc(1, pVal->nData + VARSTR_HEADER_SIZE); + if (NULL == pValue->datum.p) { + return TSDB_CODE_OUT_OF_MEMORY; + } + varDataSetLen(pValue->datum.p, pVal->nData); + memcpy(varDataVal(pValue->datum.p), pVal->pData, pVal->nData); + break; + case TSDB_DATA_TYPE_TIMESTAMP: + pValue->datum.i = pVal->i64; + break; + case TSDB_DATA_TYPE_UTINYINT: + pValue->datum.i = *(uint8_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_USMALLINT: + pValue->datum.i = *(uint16_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_UINT: + pValue->datum.i = *(uint32_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_UBIGINT: + pValue->datum.i = *(uint64_t*)(&pVal->i64); + break; + case TSDB_DATA_TYPE_JSON: + case TSDB_DATA_TYPE_VARBINARY: + case TSDB_DATA_TYPE_DECIMAL: + case TSDB_DATA_TYPE_BLOB: + case TSDB_DATA_TYPE_MEDIUMBLOB: + default: + return TSDB_CODE_FAILED; + } + return TSDB_CODE_SUCCESS; +} + +static int32_t rewriteTagCondColumn(SArray* pTagVals, SArray* pTagName, SNode** pNode) { + SColumnNode* pCol = (SColumnNode*)*pNode; + int32_t ntags = taosArrayGetSize(pTagName); + for (int32_t i = 0; i < ntags; ++i) { + char* pTagColName = taosArrayGet(pTagName, i); + if (0 == strcmp(pTagColName, pCol->colName)) { + return rewriteTagCondColumnImpl(taosArrayGet(pTagVals, i), pNode); + } + } + return TSDB_CODE_PAR_PERMISSION_DENIED; +} + +static EDealRes rewriteTagCond(SNode** pNode, void* pContext) { + if (QUERY_NODE_COLUMN == nodeType(*pNode)) { + SRewriteTagCondCxt* pCxt = pContext; + pCxt->code = rewriteTagCondColumn(pCxt->pTagVals, pCxt->pTagName, pNode); + return (TSDB_CODE_SUCCESS == pCxt->code ? DEAL_RES_IGNORE_CHILD : DEAL_RES_ERROR); + } + return DEAL_RES_CONTINUE; +} + +static int32_t setTagVal(SArray* pTagVals, SArray* pTagName, SNode* pCond) { + SRewriteTagCondCxt cxt = {.code = TSDB_CODE_SUCCESS, .pTagVals = pTagVals, .pTagName = pTagName}; + nodesRewriteExpr(&pCond, rewriteTagCond, &cxt); + return cxt.code; +} + +static int32_t checkTagCondResult(SNode* pResult) { + return (QUERY_NODE_VALUE == nodeType(pResult) && ((SValueNode*)pResult)->datum.b) ? TSDB_CODE_SUCCESS + : TSDB_CODE_PAR_PERMISSION_DENIED; +} + +int32_t checkSubtablePrivilege(SArray* pTagVals, SArray* pTagName, SNode* pCond) { + int32_t code = setTagVal(pTagVals, pTagName, pCond); + SNode* pNew = NULL; + if (TSDB_CODE_SUCCESS == code) { + code = scalarCalculateConstants(pCond, &pNew); + } + if (TSDB_CODE_SUCCESS == code) { + code = checkTagCondResult(pNew); + } + nodesDestroyNode(pNew); + return code; +} + // pSql -> tag1_value, ...) static int32_t parseTagsClauseImpl(SInsertParseContext* pCxt, SVnodeModifyOpStmt* pStmt) { int32_t code = TSDB_CODE_SUCCESS; @@ -722,25 +837,31 @@ static int32_t parseUsingClauseBottom(SInsertParseContext* pCxt, SVnodeModifyOpS return code; } +static void setUserAuthInfo(SParseContext* pCxt, SName* pTbName, SUserAuthInfo* pInfo) { + snprintf(pInfo->user, sizeof(pInfo->user), "%s", pCxt->pUser); + memcpy(&pInfo->tbName, pTbName, sizeof(SName)); + pInfo->type = AUTH_TYPE_WRITE; +} + static int32_t checkAuth(SParseContext* pCxt, SName* pTbName, bool* pMissCache) { - char dbFName[TSDB_DB_FNAME_LEN]; - tNameGetFullDbName(pTbName, dbFName); - int32_t code = TSDB_CODE_SUCCESS; - bool pass = true; - bool exists = true; + int32_t code = TSDB_CODE_SUCCESS; + SUserAuthInfo authInfo = {0}; + setUserAuthInfo(pCxt, pTbName, &authInfo); + SUserAuthRes authRes = {0}; + bool exists = true; if (pCxt->async) { - code = catalogChkAuthFromCache(pCxt->pCatalog, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass, &exists); + code = catalogChkAuthFromCache(pCxt->pCatalog, &authInfo, &authRes, &exists); } else { SRequestConnInfo conn = {.pTrans = pCxt->pTransporter, .requestId = pCxt->requestId, .requestObjRefId = pCxt->requestRid, .mgmtEps = pCxt->mgmtEpSet}; - code = catalogChkAuth(pCxt->pCatalog, &conn, pCxt->pUser, dbFName, AUTH_TYPE_WRITE, &pass); + code = catalogChkAuth(pCxt->pCatalog, &conn, &authInfo, &authRes); } if (TSDB_CODE_SUCCESS == code) { if (!exists) { *pMissCache = true; - } else if (!pass) { + } else if (!authRes.pass) { code = TSDB_CODE_PAR_PERMISSION_DENIED; } } @@ -1906,7 +2027,7 @@ static int32_t buildInsertUserAuthReq(const char* pUser, SName* pName, SArray** SUserAuthInfo userAuth = {.type = AUTH_TYPE_WRITE}; snprintf(userAuth.user, sizeof(userAuth.user), "%s", pUser); - tNameGetFullDbName(pName, userAuth.dbFName); + memcpy(&userAuth.tbName, pName, sizeof(SName)); taosArrayPush(*pUserAuth, &userAuth); return TSDB_CODE_SUCCESS; diff --git a/source/libs/parser/src/parInsertStmt.c b/source/libs/parser/src/parInsertStmt.c index 01a635e4b2d34bbb7982d45400429ac323e9547e..922a0f45ff38fba30fc4b766a3b6427026b49b36 100644 --- a/source/libs/parser/src/parInsertStmt.c +++ b/source/libs/parser/src/parInsertStmt.c @@ -251,7 +251,7 @@ int32_t qBindStmtColsValue(void* pBlock, TAOS_MULTI_BIND* bind, char* msgBuf, in goto _return; } - if (bind[c].buffer_type != pColSchema->type) { + if ((!(rowNum == 1 && bind[c].is_null && *bind[c].is_null)) && bind[c].buffer_type != pColSchema->type) { // for rowNum ==1 , connector may not set buffer_type code = buildInvalidOperationMsg(&pBuf, "column type mis-match with buffer type"); goto _return; } diff --git a/source/libs/parser/src/parInsertUtil.c b/source/libs/parser/src/parInsertUtil.c index 132a3b26187a833b5eeb3bd23ab8642331478635..ac504b9809ad495175d54025d881853d0f443ca9 100644 --- a/source/libs/parser/src/parInsertUtil.c +++ b/source/libs/parser/src/parInsertUtil.c @@ -470,11 +470,11 @@ int32_t insMergeTableDataCxt(SHashObj* pTableHash, SArray** pVgDataBlocks) { if (TSDB_CODE_SUCCESS == code) { SVgroupDataCxt* pVgCxt = NULL; int32_t vgId = pTableCxt->pMeta->vgId; - void** p = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); - if (NULL == p) { + void** pp = taosHashGet(pVgroupHash, &vgId, sizeof(vgId)); + if (NULL == pp) { code = createVgroupDataCxt(pTableCxt, pVgroupHash, pVgroupList, &pVgCxt); } else { - pVgCxt = *(SVgroupDataCxt**)p; + pVgCxt = *(SVgroupDataCxt**)pp; } if (TSDB_CODE_SUCCESS == code) { code = fillVgroupDataCxt(pTableCxt, pVgCxt); diff --git a/source/libs/parser/src/parTranslater.c b/source/libs/parser/src/parTranslater.c index 78fc097f6f5024e8a1932217ba2440cb342ec3e7..25e92a55ec2adce02505fc4b3c78615764ece5b9 100644 --- a/source/libs/parser/src/parTranslater.c +++ b/source/libs/parser/src/parTranslater.c @@ -645,6 +645,10 @@ static bool isSelectStmt(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SELECT_STMT == nodeType(pCurrStmt); } +static bool isDeleteStmt(SNode* pCurrStmt) { + return NULL != pCurrStmt && QUERY_NODE_DELETE_STMT == nodeType(pCurrStmt); +} + static bool isSetOperator(SNode* pCurrStmt) { return NULL != pCurrStmt && QUERY_NODE_SET_OPERATOR == nodeType(pCurrStmt); } @@ -669,6 +673,9 @@ static uint8_t getPrecisionFromCurrStmt(SNode* pCurrStmt, uint8_t defaultVal) { if (NULL != pCurrStmt && QUERY_NODE_CREATE_STREAM_STMT == nodeType(pCurrStmt)) { return getPrecisionFromCurrStmt(((SCreateStreamStmt*)pCurrStmt)->pQuery, defaultVal); } + if (isDeleteStmt(pCurrStmt)) { + return ((SDeleteStmt*)pCurrStmt)->precision; + } return defaultVal; } @@ -688,6 +695,10 @@ static bool isWindowPseudoColumnFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsWindowPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); } +static bool isInterpPseudoColumnFunc(const SNode* pNode) { + return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsInterpPseudoColumnFunc(((SFunctionNode*)pNode)->funcId)); +} + static bool isTimelineFunc(const SNode* pNode) { return (QUERY_NODE_FUNCTION == nodeType(pNode) && fmIsTimelineFunc(((SFunctionNode*)pNode)->funcId)); } @@ -978,7 +989,8 @@ static EDealRes translateColumnUseAlias(STranslateContext* pCxt, SColumnNode** p } static EDealRes translateColumn(STranslateContext* pCxt, SColumnNode** pCol) { - if (NULL == pCxt->pCurrStmt || isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable) { + if (NULL == pCxt->pCurrStmt || + (isSelectStmt(pCxt->pCurrStmt) && NULL == ((SSelectStmt*)pCxt->pCurrStmt)->pFromTable)) { return generateDealNodeErrMsg(pCxt, TSDB_CODE_PAR_INVALID_COLUMN, (*pCol)->colName); } @@ -1234,6 +1246,10 @@ static int32_t calcTypeBytes(SDataType dt) { } static EDealRes translateValue(STranslateContext* pCxt, SValueNode* pVal) { + if (pVal->translate) { + return TSDB_CODE_SUCCESS; + } + SDataType dt = pVal->node.resType; dt.bytes = calcTypeBytes(dt); return translateValueImpl(pCxt, pVal, dt, false); @@ -1294,7 +1310,8 @@ static EDealRes translateOperator(STranslateContext* pCxt, SOperatorNode* pOp) { } static EDealRes haveVectorFunction(SNode* pNode, void* pContext) { - if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode)) { + if (isAggFunc(pNode) || isIndefiniteRowsFunc(pNode) || isWindowPseudoColumnFunc(pNode) || + isInterpPseudoColumnFunc(pNode)) { *((bool*)pContext) = true; return DEAL_RES_END; } @@ -1521,6 +1538,21 @@ static int32_t translateInterpFunc(STranslateContext* pCxt, SFunctionNode* pFunc return TSDB_CODE_SUCCESS; } +static int32_t translateInterpPseudoColumnFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsInterpPseudoColumnFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + if (!isSelectStmt(pCxt->pCurrStmt)) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "%s must be used in select statements", pFunc->functionName); + } + if (pCxt->currClause == SQL_CLAUSE_WHERE) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE, + "%s is not allowed in where clause", pFunc->functionName); + } + return TSDB_CODE_SUCCESS; +} + static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { if (!fmIsTimelineFunc(pFunc->funcId)) { return TSDB_CODE_SUCCESS; @@ -1538,6 +1570,21 @@ static int32_t translateTimelineFunc(STranslateContext* pCxt, SFunctionNode* pFu return TSDB_CODE_SUCCESS; } +static int32_t translateDateTimeFunc(STranslateContext* pCxt, SFunctionNode* pFunc) { + if (!fmIsDateTimeFunc(pFunc->funcId)) { + return TSDB_CODE_SUCCESS; + } + + if (!isSelectStmt(pCxt->pCurrStmt)) { + return TSDB_CODE_SUCCESS; + } + + SSelectStmt* pSelect = (SSelectStmt*)pCxt->pCurrStmt; + pFunc->node.resType.precision = pSelect->precision; + + return TSDB_CODE_SUCCESS; +} + static bool hasFillClause(SNode* pCurrStmt) { if (!isSelectStmt(pCurrStmt)) { return false; @@ -1677,6 +1724,7 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { SSelectStmt* pSelect = (SSelectStmt*)pCurrStmt; pSelect->hasAggFuncs = pSelect->hasAggFuncs ? true : fmIsAggFunc(pFunc->funcId); pSelect->hasRepeatScanFuncs = pSelect->hasRepeatScanFuncs ? true : fmIsRepeatScanFunc(pFunc->funcId); + if (fmIsIndefiniteRowsFunc(pFunc->funcId)) { pSelect->hasIndefiniteRowsFunc = true; pSelect->returnRows = fmGetFuncReturnRows(pFunc); @@ -1691,6 +1739,8 @@ static void setFuncClassification(SNode* pCurrStmt, SFunctionNode* pFunc) { pSelect->hasUniqueFunc = pSelect->hasUniqueFunc ? true : (FUNCTION_TYPE_UNIQUE == pFunc->funcType); pSelect->hasTailFunc = pSelect->hasTailFunc ? true : (FUNCTION_TYPE_TAIL == pFunc->funcType); pSelect->hasInterpFunc = pSelect->hasInterpFunc ? true : (FUNCTION_TYPE_INTERP == pFunc->funcType); + pSelect->hasInterpPseudoColFunc = + pSelect->hasInterpPseudoColFunc ? true : fmIsInterpPseudoColumnFunc(pFunc->funcId); pSelect->hasLastRowFunc = pSelect->hasLastRowFunc ? true : (FUNCTION_TYPE_LAST_ROW == pFunc->funcType); pSelect->hasLastFunc = pSelect->hasLastFunc ? true : (FUNCTION_TYPE_LAST == pFunc->funcType); pSelect->hasTimeLineFunc = pSelect->hasTimeLineFunc ? true : fmIsTimelineFunc(pFunc->funcId); @@ -1817,9 +1867,15 @@ static int32_t translateNormalFunction(STranslateContext* pCxt, SFunctionNode* p if (TSDB_CODE_SUCCESS == code) { code = translateInterpFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateInterpPseudoColumnFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateTimelineFunc(pCxt, pFunc); } + if (TSDB_CODE_SUCCESS == code) { + code = translateDateTimeFunc(pCxt, pFunc); + } if (TSDB_CODE_SUCCESS == code) { code = translateBlockDistFunc(pCxt, pFunc); } @@ -2875,6 +2931,11 @@ static int32_t convertFillValue(STranslateContext* pCxt, SDataType dt, SNodeList if (TSDB_CODE_SUCCESS == code) { code = scalarCalculateConstants(pCaseFunc, &pCell->pNode); } + if (TSDB_CODE_SUCCESS == code && QUERY_NODE_VALUE != nodeType(pCell->pNode)) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Fill value is just a constant"); + } else if (TSDB_CODE_SUCCESS != code) { + code = generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + } return code; } @@ -2891,9 +2952,9 @@ static int32_t checkFillValues(STranslateContext* pCxt, SFillNode* pFill, SNodeL if (fillNo >= LIST_LENGTH(pFillValues->pNodeList)) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled values number mismatch"); } - if (TSDB_CODE_SUCCESS != - convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo)) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_WRONG_VALUE_TYPE, "Filled data type mismatch"); + int32_t code = convertFillValue(pCxt, ((SExprNode*)pProject)->resType, pFillValues->pNodeList, fillNo); + if (TSDB_CODE_SUCCESS != code) { + return code; } ++fillNo; } @@ -2974,12 +3035,13 @@ static int32_t translateSelectList(STranslateContext* pCxt, SSelectStmt* pSelect } static int32_t translateHaving(STranslateContext* pCxt, SSelectStmt* pSelect) { - if (NULL == pSelect->pGroupByList && NULL != pSelect->pHaving) { + if (NULL == pSelect->pGroupByList && NULL == pSelect->pPartitionByList && NULL == pSelect->pWindow && + NULL != pSelect->pHaving) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_GROUPBY_LACK_EXPRESSION); } pCxt->currClause = SQL_CLAUSE_HAVING; int32_t code = translateExpr(pCxt, &pSelect->pHaving); - if (TSDB_CODE_SUCCESS == code) { + if (TSDB_CODE_SUCCESS == code && (NULL != pSelect->pGroupByList || NULL != pSelect->pWindow)) { code = checkExprForGroupBy(pCxt, &pSelect->pHaving); } return code; @@ -3369,6 +3431,10 @@ static int32_t translateInterp(STranslateContext* pCxt, SSelectStmt* pSelect) { if (NULL != pSelect->pRange || NULL != pSelect->pEvery || NULL != pSelect->pFill) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_INTERP_CLAUSE); } + if (pSelect->hasInterpPseudoColFunc) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_NOT_ALLOWED_FUNC, + "Has Interp pseudo column(s) but missing interp function"); + } return TSDB_CODE_SUCCESS; } @@ -3736,6 +3802,7 @@ static int32_t translateDelete(STranslateContext* pCxt, SDeleteStmt* pDelete) { pCxt->pCurrStmt = (SNode*)pDelete; int32_t code = translateFrom(pCxt, pDelete->pFromTable); if (TSDB_CODE_SUCCESS == code) { + pDelete->precision = ((STableNode*)pDelete->pFromTable)->precision; code = translateDeleteWhere(pCxt, pDelete); } pCxt->currClause = SQL_CLAUSE_SELECT; @@ -5138,26 +5205,32 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_ONLY_ONE_JSON_TAG); } - if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); - } - int32_t tagsLen = 0; for (int32_t i = 0; i < pTableMeta->tableInfo.numOfTags; ++i) { tagsLen += pTagsSchema[i].bytes; } if (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || - TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) { + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType || + TSDB_ALTER_TABLE_DROP_TAG == pStmt->alterType) { if (TSDB_SUPER_TABLE != pTableMeta->tableType) { return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "Table is not super table"); } const SSchema* pSchema = getColSchema(pTableMeta, pStmt->colName); if (NULL == pSchema) { - return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_COLUMN, pStmt->colName); - } else if (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || - pSchema->bytes >= calcTypeBytes(pStmt->dataType)) { + return generateSyntaxErrMsg( + &pCxt->msgBuf, + (TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || TSDB_ALTER_TABLE_DROP_COLUMN == pStmt->alterType) + ? TSDB_CODE_PAR_INVALID_COLUMN + : TSDB_CODE_PAR_INVALID_TAG_NAME, + pStmt->colName); + } + + if ((TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES == pStmt->alterType || + TSDB_ALTER_TABLE_UPDATE_TAG_BYTES == pStmt->alterType) && + (!IS_VAR_DATA_TYPE(pSchema->type) || pSchema->type != pStmt->dataType.type || + pSchema->bytes >= calcTypeBytes(pStmt->dataType))) { return generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_MODIFY_COL); } @@ -5202,6 +5275,10 @@ static int32_t checkAlterSuperTableBySchema(STranslateContext* pCxt, SAlterTable } } + if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); + } + return TSDB_CODE_SUCCESS; } @@ -5674,6 +5751,14 @@ static int32_t translateDropCGroup(STranslateContext* pCxt, SDropCGroupStmt* pSt static int32_t translateAlterLocal(STranslateContext* pCxt, SAlterLocalStmt* pStmt) { // The statement is executed directly on the client without constructing a message. + if ('\0' != pStmt->value[0]) { + return TSDB_CODE_SUCCESS; + } + char* p = strchr(pStmt->config, ' '); + if (NULL != p) { + *p = 0; + strcpy(pStmt->value, p + 1); + } return TSDB_CODE_SUCCESS; } @@ -6485,22 +6570,69 @@ static int32_t translateDropFunction(STranslateContext* pCxt, SDropFunctionStmt* return buildCmdMsg(pCxt, TDMT_MND_DROP_FUNC, (FSerializeFunc)tSerializeSDropFuncReq, &req); } +static int32_t createRealTableForGrantTable(SGrantStmt* pStmt, SRealTableNode** pTable) { + SRealTableNode* pRealTable = (SRealTableNode*)nodesMakeNode(QUERY_NODE_REAL_TABLE); + if (NULL == pRealTable) { + return TSDB_CODE_OUT_OF_MEMORY; + } + strcpy(pRealTable->table.dbName, pStmt->objName); + strcpy(pRealTable->table.tableName, pStmt->tabName); + strcpy(pRealTable->table.tableAlias, pStmt->tabName); + *pTable = pRealTable; + return TSDB_CODE_SUCCESS; +} + +static int32_t translateGrantTagCond(STranslateContext* pCxt, SGrantStmt* pStmt, SAlterUserReq* pReq) { + if (NULL == pStmt->pTagCond) { + return TSDB_CODE_SUCCESS; + } + if ('\0' == pStmt->tabName[0] || '*' == pStmt->tabName[0]) { + return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_SYNTAX_ERROR, + "The With clause can only be used for table level privilege"); + } + + pCxt->pCurrStmt = (SNode*)pStmt; + SRealTableNode* pTable = NULL; + int32_t code = createRealTableForGrantTable(pStmt, &pTable); + if (TSDB_CODE_SUCCESS == code) { + SName name; + code = getTableMetaImpl(pCxt, toName(pCxt->pParseCxt->acctId, pTable->table.dbName, pTable->table.tableName, &name), + &(pTable->pMeta)); + } + if (TSDB_CODE_SUCCESS == code) { + code = addNamespace(pCxt, pTable); + } + if (TSDB_CODE_SUCCESS == code) { + code = translateExpr(pCxt, &pStmt->pTagCond); + } + if (TSDB_CODE_SUCCESS == code) { + code = nodesNodeToString(pStmt->pTagCond, false, &pReq->tagCond, &pReq->tagCondLen); + } + nodesDestroyNode((SNode*)pTable); + return code; +} + static int32_t translateGrant(STranslateContext* pCxt, SGrantStmt* pStmt) { SAlterUserReq req = {0}; if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) || (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) && BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) { - req.alterType = TSDB_ALTER_USER_ADD_ALL_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_ALL_DB : TSDB_ALTER_USER_ADD_ALL_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) { - req.alterType = TSDB_ALTER_USER_ADD_READ_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_READ_DB : TSDB_ALTER_USER_ADD_READ_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { - req.alterType = TSDB_ALTER_USER_ADD_WRITE_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_ADD_WRITE_DB : TSDB_ALTER_USER_ADD_WRITE_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) { req.alterType = TSDB_ALTER_USER_ADD_SUBSCRIBE_TOPIC; } strcpy(req.user, pStmt->userName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); - return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); + sprintf(req.tabName, "%s", pStmt->tabName); + int32_t code = translateGrantTagCond(pCxt, pStmt, &req); + if (TSDB_CODE_SUCCESS == code) { + code = buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); + } + return code; } static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { @@ -6508,16 +6640,17 @@ static int32_t translateRevoke(STranslateContext* pCxt, SRevokeStmt* pStmt) { if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_ALL) || (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ) && BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE))) { - req.alterType = TSDB_ALTER_USER_REMOVE_ALL_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_ALL_DB : TSDB_ALTER_USER_REMOVE_ALL_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_READ)) { - req.alterType = TSDB_ALTER_USER_REMOVE_READ_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_READ_DB : TSDB_ALTER_USER_REMOVE_READ_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_WRITE)) { - req.alterType = TSDB_ALTER_USER_REMOVE_WRITE_DB; + req.alterType = ('\0' == pStmt->tabName[0] ? TSDB_ALTER_USER_REMOVE_WRITE_DB : TSDB_ALTER_USER_REMOVE_WRITE_TABLE); } else if (BIT_FLAG_TEST_MASK(pStmt->privileges, PRIVILEGE_TYPE_SUBSCRIBE)) { req.alterType = TSDB_ALTER_USER_REMOVE_SUBSCRIBE_TOPIC; } strcpy(req.user, pStmt->userName); sprintf(req.objname, "%d.%s", pCxt->pParseCxt->acctId, pStmt->objName); + sprintf(req.tabName, "%s", pStmt->tabName); return buildCmdMsg(pCxt, TDMT_MND_ALTER_USER, (FSerializeFunc)tSerializeSAlterUserReq, &req); } @@ -8204,10 +8337,6 @@ static void destoryAlterTbReq(SVAlterTbReq* pReq) { static int32_t rewriteAlterTableImpl(STranslateContext* pCxt, SAlterTableStmt* pStmt, STableMeta* pTableMeta, SQuery* pQuery) { - if (getNumOfTags(pTableMeta) == 1 && pStmt->alterType == TSDB_ALTER_TABLE_DROP_TAG) { - return generateSyntaxErrMsgExt(&pCxt->msgBuf, TSDB_CODE_PAR_INVALID_ALTER_TABLE, "the only tag cannot be dropped"); - } - if (TSDB_SUPER_TABLE == pTableMeta->tableType) { return TSDB_CODE_SUCCESS; } else if (TSDB_CHILD_TABLE != pTableMeta->tableType && TSDB_NORMAL_TABLE != pTableMeta->tableType) { diff --git a/source/libs/parser/src/parUtil.c b/source/libs/parser/src/parUtil.c index 563bc5e7800ddbb8752b4070bd203ad3410bd5c1..14da6f8aabcfba0cfdc3a71a731b1c34acdb32d5 100644 --- a/source/libs/parser/src/parUtil.c +++ b/source/libs/parser/src/parUtil.c @@ -17,7 +17,7 @@ #include "cJSON.h" #include "querynodes.h" -#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_DB_FNAME_LEN + 2 +#define USER_AUTH_KEY_MAX_LEN TSDB_USER_LEN + TSDB_TABLE_FNAME_LEN + 2 const void* nullPointer = NULL; @@ -496,24 +496,44 @@ int32_t getVnodeSysTableTargetName(int32_t acctId, SNode* pWhere, SName* pName) return TSDB_CODE_SUCCESS; } -static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%d.%s*%d", pUser, acctId, pDb, type); +static int32_t userAuthToString(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, + char* pStr) { + return sprintf(pStr, "%s*%d*%s*%s*%d", pUser, acctId, pDb, (NULL != pTable && '\0' == pTable[0]) ? NULL : pTable, + type); } -static int32_t userAuthToStringExt(const char* pUser, const char* pDbFName, AUTH_TYPE type, char* pStr) { - return sprintf(pStr, "%s*%s*%d", pUser, pDbFName, type); +static int32_t getIntegerFromAuthStr(const char* pStart, char** pNext) { + char* p = strchr(pStart, '*'); + char buf[10] = {0}; + if (NULL == p) { + strcpy(buf, pStart); + *pNext = NULL; + } else { + strncpy(buf, pStart, p - pStart); + *pNext = ++p; + } + return taosStr2Int32(buf, NULL, 10); +} + +static void getStringFromAuthStr(const char* pStart, char* pStr, char** pNext) { + char* p = strchr(pStart, '*'); + if (NULL == p) { + strcpy(pStr, pStart); + *pNext = NULL; + } else { + strncpy(pStr, pStart, p - pStart); + *pNext = ++p; + } } static void stringToUserAuth(const char* pStr, int32_t len, SUserAuthInfo* pUserAuth) { - char* p1 = strchr(pStr, '*'); - strncpy(pUserAuth->user, pStr, p1 - pStr); - ++p1; - char* p2 = strchr(p1, '*'); - strncpy(pUserAuth->dbFName, p1, p2 - p1); - ++p2; - char buf[10] = {0}; - strncpy(buf, p2, len - (p2 - pStr)); - pUserAuth->type = taosStr2Int32(buf, NULL, 10); + char* p = NULL; + getStringFromAuthStr(pStr, pUserAuth->user, &p); + pUserAuth->tbName.acctId = getIntegerFromAuthStr(p, &p); + getStringFromAuthStr(p, pUserAuth->tbName.dbname, &p); + getStringFromAuthStr(p, pUserAuth->tbName.tname, &p); + pUserAuth->tbName.type = TSDB_TABLE_NAME_T; + pUserAuth->type = getIntegerFromAuthStr(p, &p); } static int32_t buildTableReq(SHashObj* pTablesHash, SArray** pTables) { @@ -584,10 +604,12 @@ static int32_t buildUserAuthReq(SHashObj* pUserAuthHash, SArray** pUserAuth) { } void* p = taosHashIterate(pUserAuthHash, NULL); while (NULL != p) { - size_t len = 0; - char* pKey = taosHashGetKey(p, &len); + size_t len = 0; + char* pKey = taosHashGetKey(p, &len); + char key[USER_AUTH_KEY_MAX_LEN] = {0}; + strncpy(key, pKey, len); SUserAuthInfo userAuth = {0}; - stringToUserAuth(pKey, len, &userAuth); + stringToUserAuth(key, len, &userAuth); taosArrayPush(*pUserAuth, &userAuth); p = taosHashIterate(pUserAuthHash, p); } @@ -712,7 +734,8 @@ static int32_t putUserAuthToCache(const SArray* pUserAuthReq, const SArray* pUse for (int32_t i = 0; i < nvgs; ++i) { SUserAuthInfo* pUser = taosArrayGet(pUserAuthReq, i); char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToStringExt(pUser->user, pUser->dbFName, pUser->type, key); + int32_t len = userAuthToString(pUser->tbName.acctId, pUser->user, pUser->tbName.dbname, pUser->tbName.tname, + pUser->type, key); if (TSDB_CODE_SUCCESS != putMetaDataToHash(key, len, pUserAuthData, i, pUserAuth)) { return TSDB_CODE_OUT_OF_MEMORY; } @@ -910,33 +933,24 @@ static int32_t reserveUserAuthInCacheImpl(const char* pKey, int32_t len, SParseM return TSDB_CODE_OUT_OF_MEMORY; } } - bool pass = false; - return taosHashPut(pMetaCache->pUserAuth, pKey, len, &pass, sizeof(pass)); + return taosHashPut(pMetaCache->pUserAuth, pKey, len, &nullPointer, POINTER_BYTES); } -int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, AUTH_TYPE type, +int32_t reserveUserAuthInCache(int32_t acctId, const char* pUser, const char* pDb, const char* pTable, AUTH_TYPE type, SParseMetaCache* pMetaCache) { char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToString(acctId, pUser, pDb, type, key); - return reserveUserAuthInCacheImpl(key, len, pMetaCache); -} - -int32_t reserveUserAuthInCacheExt(const char* pUser, const SName* pName, AUTH_TYPE type, SParseMetaCache* pMetaCache) { - char dbFName[TSDB_DB_FNAME_LEN] = {0}; - tNameGetFullDbName(pName, dbFName); - char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToStringExt(pUser, dbFName, type, key); + int32_t len = userAuthToString(acctId, pUser, pDb, pTable, type, key); return reserveUserAuthInCacheImpl(key, len, pMetaCache); } -int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, const char* pUser, const char* pDbFName, AUTH_TYPE type, - bool* pPass) { - char key[USER_AUTH_KEY_MAX_LEN] = {0}; - int32_t len = userAuthToStringExt(pUser, pDbFName, type, key); - bool* pRes = NULL; - int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pRes); +int32_t getUserAuthFromCache(SParseMetaCache* pMetaCache, SUserAuthInfo* pAuthReq, SUserAuthRes* pAuthRes) { + char key[USER_AUTH_KEY_MAX_LEN] = {0}; + int32_t len = userAuthToString(pAuthReq->tbName.acctId, pAuthReq->user, pAuthReq->tbName.dbname, + pAuthReq->tbName.tname, pAuthReq->type, key); + SUserAuthRes* pAuth = NULL; + int32_t code = getMetaDataFromHash(key, len, pMetaCache->pUserAuth, (void**)&pAuth); if (TSDB_CODE_SUCCESS == code) { - *pPass = *pRes; + memcpy(pAuthRes, pAuth, sizeof(SUserAuthRes)); } return code; } diff --git a/source/libs/parser/src/sql.c b/source/libs/parser/src/sql.c index b1437bbb6bba7de0aadaa3b0a4c8950865fa731d..a38e2368ddd23702b0e2b5a993b2e11da1fbc950 100644 --- a/source/libs/parser/src/sql.c +++ b/source/libs/parser/src/sql.c @@ -104,26 +104,27 @@ #endif /************* Begin control #defines *****************************************/ #define YYCODETYPE unsigned short int -#define YYNOCODE 475 +#define YYNOCODE 476 #define YYACTIONTYPE unsigned short int #define ParseTOKENTYPE SToken typedef union { int yyinit; ParseTOKENTYPE yy0; - EOrder yy88; - EFillMode yy94; - SToken yy129; - SDataType yy184; - SNodeList* yy274; - int32_t yy310; - bool yy337; - int8_t yy353; - int64_t yy359; - EOperatorType yy440; - SAlterOption yy595; - SNode* yy712; - ENullOrder yy907; - EJoinType yy912; + EFillMode yy46; + SAlterOption yy53; + SToken yy113; + EOperatorType yy156; + bool yy369; + SNodeList* yy432; + SNode* yy448; + int8_t yy551; + ENullOrder yy585; + EJoinType yy596; + EOrder yy666; + SDataType yy728; + STokenPair yy777; + int32_t yy788; + int64_t yy837; } YYMINORTYPE; #ifndef YYSTACKDEPTH #define YYSTACKDEPTH 100 @@ -139,17 +140,17 @@ typedef union { #define ParseCTX_FETCH #define ParseCTX_STORE #define YYFALLBACK 1 -#define YYNSTATE 761 -#define YYNRULE 581 +#define YYNSTATE 762 +#define YYNRULE 583 #define YYNTOKEN 330 -#define YY_MAX_SHIFT 760 -#define YY_MIN_SHIFTREDUCE 1133 -#define YY_MAX_SHIFTREDUCE 1713 -#define YY_ERROR_ACTION 1714 -#define YY_ACCEPT_ACTION 1715 -#define YY_NO_ACTION 1716 -#define YY_MIN_REDUCE 1717 -#define YY_MAX_REDUCE 2297 +#define YY_MAX_SHIFT 761 +#define YY_MIN_SHIFTREDUCE 1136 +#define YY_MAX_SHIFTREDUCE 1718 +#define YY_ERROR_ACTION 1719 +#define YY_ACCEPT_ACTION 1720 +#define YY_NO_ACTION 1721 +#define YY_MIN_REDUCE 1722 +#define YY_MAX_REDUCE 2304 /************* End control #defines *******************************************/ #define YY_NLOOKAHEAD ((int)(sizeof(yy_lookahead)/sizeof(yy_lookahead[0]))) @@ -216,827 +217,814 @@ typedef union { ** yy_default[] Default action for each state. ** *********** Begin parsing tables **********************************************/ -#define YY_ACTTAB_COUNT (3096) +#define YY_ACTTAB_COUNT (3030) static const YYACTIONTYPE yy_action[] = { - /* 0 */ 1873, 2109, 2003, 428, 1875, 1869, 631, 427, 2096, 167, - /* 10 */ 650, 1729, 45, 43, 1641, 2091, 176, 2001, 637, 2091, - /* 20 */ 389, 613, 1490, 38, 37, 2268, 187, 44, 42, 41, - /* 30 */ 40, 39, 1937, 1571, 2127, 1488, 1783, 354, 1986, 367, - /* 40 */ 612, 182, 664, 1884, 106, 2269, 614, 2077, 1935, 666, - /* 50 */ 588, 2087, 2093, 588, 2268, 2087, 2093, 2268, 1566, 1517, - /* 60 */ 140, 2273, 660, 178, 18, 2268, 660, 2003, 1876, 2274, - /* 70 */ 182, 1496, 2274, 182, 2269, 614, 1924, 2269, 614, 380, - /* 80 */ 2108, 2272, 2000, 637, 2144, 2269, 2271, 328, 2110, 670, - /* 90 */ 2112, 2113, 665, 663, 660, 651, 2162, 757, 1258, 649, - /* 100 */ 14, 501, 734, 733, 732, 731, 399, 497, 730, 729, - /* 110 */ 143, 724, 723, 722, 721, 720, 719, 718, 156, 714, - /* 120 */ 713, 712, 398, 397, 709, 708, 707, 706, 705, 1515, - /* 130 */ 495, 1315, 496, 1753, 1260, 122, 1573, 1574, 121, 120, - /* 140 */ 119, 118, 117, 116, 115, 114, 113, 1306, 692, 691, - /* 150 */ 690, 1310, 689, 1312, 1313, 688, 685, 649, 1321, 682, - /* 160 */ 1323, 1324, 679, 676, 650, 61, 1546, 1556, 2273, 649, - /* 170 */ 38, 37, 1572, 1575, 44, 42, 41, 40, 39, 48, - /* 180 */ 132, 504, 1937, 496, 1753, 1517, 1491, 534, 1489, 352, - /* 190 */ 1515, 35, 292, 1710, 635, 38, 37, 1884, 1935, 44, - /* 200 */ 42, 41, 40, 39, 238, 38, 37, 273, 237, 44, - /* 210 */ 42, 41, 40, 39, 2109, 1494, 1495, 1518, 1545, 1548, - /* 220 */ 1549, 1550, 1551, 1552, 1553, 1554, 1555, 662, 658, 1564, - /* 230 */ 1565, 1567, 1568, 1569, 1570, 2, 45, 43, 166, 513, - /* 240 */ 636, 337, 625, 1513, 389, 1825, 1490, 2127, 1740, 48, - /* 250 */ 465, 346, 1739, 479, 625, 628, 478, 1571, 2044, 1488, - /* 260 */ 2077, 613, 666, 1680, 2070, 2268, 1703, 49, 89, 341, - /* 270 */ 61, 448, 366, 480, 568, 139, 450, 1516, 434, 2095, - /* 280 */ 612, 182, 1566, 13, 12, 2269, 614, 139, 18, 511, - /* 290 */ 2091, 1996, 1709, 2108, 2077, 1496, 1600, 2144, 2077, 608, - /* 300 */ 109, 2110, 670, 2112, 2113, 665, 240, 660, 1185, 274, - /* 310 */ 1184, 588, 179, 86, 2197, 2268, 2109, 185, 383, 2193, - /* 320 */ 334, 757, 355, 222, 14, 86, 2087, 2093, 370, 356, - /* 330 */ 2274, 182, 184, 273, 438, 2269, 614, 660, 1879, 1186, - /* 340 */ 2223, 627, 180, 2205, 2206, 1862, 137, 2210, 1516, 2127, - /* 350 */ 1880, 61, 1601, 92, 181, 2205, 2206, 667, 137, 2210, - /* 360 */ 1573, 1574, 2077, 476, 666, 251, 470, 469, 468, 467, - /* 370 */ 464, 463, 462, 461, 460, 456, 455, 454, 453, 336, - /* 380 */ 445, 444, 443, 1614, 440, 439, 353, 1931, 1932, 147, - /* 390 */ 1546, 1556, 1738, 1349, 1350, 2108, 1572, 1575, 566, 2144, - /* 400 */ 392, 185, 109, 2110, 670, 2112, 2113, 665, 161, 660, - /* 410 */ 1491, 564, 1489, 562, 2172, 368, 2197, 1886, 603, 185, - /* 420 */ 383, 2193, 185, 652, 1935, 2169, 34, 387, 1595, 1596, - /* 430 */ 1597, 1598, 1599, 1603, 1604, 1605, 1606, 625, 2077, 1494, - /* 440 */ 1495, 704, 1545, 1548, 1549, 1550, 1551, 1552, 1553, 1554, - /* 450 */ 1555, 662, 658, 1564, 1565, 1567, 1568, 1569, 1570, 2, - /* 460 */ 11, 45, 43, 394, 1421, 1422, 1930, 1932, 1514, 389, - /* 470 */ 139, 1490, 552, 551, 550, 1669, 548, 547, 1982, 542, - /* 480 */ 136, 546, 1571, 239, 1488, 545, 1737, 223, 2109, 190, - /* 490 */ 544, 549, 362, 361, 494, 1515, 543, 499, 1759, 1967, - /* 500 */ 1420, 1423, 171, 185, 609, 604, 597, 1566, 530, 526, - /* 510 */ 522, 518, 220, 18, 44, 42, 41, 40, 39, 650, - /* 520 */ 1496, 2127, 600, 599, 1667, 1668, 1670, 1671, 1672, 667, - /* 530 */ 100, 636, 2077, 65, 2077, 54, 666, 271, 2205, 624, - /* 540 */ 503, 133, 623, 499, 1759, 2268, 757, 61, 1937, 14, - /* 550 */ 1463, 1464, 1884, 87, 1877, 377, 218, 1645, 728, 726, - /* 560 */ 612, 182, 2127, 1515, 1935, 2269, 614, 2108, 165, 1718, - /* 570 */ 607, 2144, 1786, 312, 169, 2110, 670, 2112, 2113, 665, - /* 580 */ 634, 660, 1996, 2272, 702, 1573, 1574, 310, 72, 421, - /* 590 */ 122, 71, 1861, 121, 120, 119, 118, 117, 116, 115, - /* 600 */ 114, 113, 154, 153, 699, 698, 697, 151, 575, 205, - /* 610 */ 491, 489, 486, 423, 419, 1546, 1556, 606, 41, 40, - /* 620 */ 39, 1572, 1575, 217, 211, 11, 615, 2289, 216, 61, - /* 630 */ 509, 538, 426, 176, 425, 1491, 381, 1489, 552, 551, - /* 640 */ 550, 1404, 1405, 636, 164, 542, 136, 546, 209, 2095, - /* 650 */ 61, 545, 537, 1886, 452, 1987, 544, 549, 362, 361, - /* 660 */ 2091, 424, 543, 451, 1494, 1495, 1496, 1545, 1548, 1549, - /* 670 */ 1550, 1551, 1552, 1553, 1554, 1555, 662, 658, 1564, 1565, - /* 680 */ 1567, 1568, 1569, 1570, 2, 45, 43, 1576, 108, 1717, - /* 690 */ 1736, 1735, 645, 389, 1996, 1490, 2087, 2093, 371, 185, - /* 700 */ 11, 2109, 9, 1715, 702, 1871, 1571, 660, 1488, 1547, - /* 710 */ 1185, 1166, 1184, 131, 130, 129, 128, 127, 126, 125, - /* 720 */ 124, 123, 154, 153, 699, 698, 697, 151, 80, 79, - /* 730 */ 431, 1566, 650, 189, 2127, 650, 2077, 2077, 1171, 1172, - /* 740 */ 1734, 1186, 667, 650, 1496, 695, 472, 2077, 132, 666, - /* 750 */ 1168, 432, 1171, 1172, 335, 539, 1269, 417, 716, 433, - /* 760 */ 415, 411, 407, 404, 424, 1884, 2095, 1937, 1884, 1268, - /* 770 */ 757, 402, 1490, 46, 382, 401, 1884, 2091, 1860, 2109, - /* 780 */ 2108, 185, 2098, 1935, 2144, 1488, 2077, 109, 2110, 670, - /* 790 */ 2112, 2113, 665, 654, 660, 2169, 1867, 142, 1581, 149, - /* 800 */ 2168, 2197, 185, 650, 1515, 383, 2193, 198, 197, 1573, - /* 810 */ 1574, 625, 2127, 2087, 2093, 384, 1937, 164, 588, 442, - /* 820 */ 667, 1496, 2268, 393, 660, 2077, 1887, 666, 283, 284, - /* 830 */ 471, 481, 1935, 282, 194, 2100, 1884, 2274, 182, 1546, - /* 840 */ 1556, 704, 2269, 614, 139, 1572, 1575, 757, 650, 38, - /* 850 */ 37, 1518, 616, 44, 42, 41, 40, 39, 2108, 1491, - /* 860 */ 1273, 1489, 2144, 392, 457, 109, 2110, 670, 2112, 2113, - /* 870 */ 665, 164, 660, 1272, 1733, 83, 1515, 2170, 82, 2197, - /* 880 */ 1886, 1884, 1732, 383, 2193, 27, 2071, 1602, 1494, 1495, - /* 890 */ 1638, 1545, 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, - /* 900 */ 662, 658, 1564, 1565, 1567, 1568, 1569, 1570, 2, 45, - /* 910 */ 43, 183, 2205, 2206, 395, 137, 2210, 389, 650, 1490, - /* 920 */ 2077, 2109, 164, 248, 2217, 1634, 650, 1731, 2077, 73, - /* 930 */ 1571, 1886, 1488, 588, 458, 483, 1491, 2268, 1489, 38, - /* 940 */ 37, 557, 512, 44, 42, 41, 40, 39, 2212, 694, - /* 950 */ 1547, 1884, 2274, 182, 2127, 1566, 567, 2269, 614, 1884, - /* 960 */ 1728, 32, 628, 2212, 2060, 1494, 1495, 2077, 1496, 666, - /* 970 */ 236, 1607, 1727, 2077, 2209, 38, 37, 1982, 81, 44, - /* 980 */ 42, 41, 40, 39, 717, 560, 1846, 650, 192, 2208, - /* 990 */ 554, 1726, 8, 152, 757, 235, 1725, 46, 1724, 191, - /* 1000 */ 2108, 1518, 1723, 1881, 2144, 2109, 2077, 109, 2110, 670, - /* 1010 */ 2112, 2113, 665, 1722, 660, 1721, 1720, 31, 2077, 179, - /* 1020 */ 1884, 2197, 1547, 38, 37, 383, 2193, 44, 42, 41, - /* 1030 */ 40, 39, 1657, 1573, 1574, 1859, 69, 2077, 2127, 68, - /* 1040 */ 141, 650, 2077, 2168, 2077, 1982, 667, 2224, 2077, 650, - /* 1050 */ 53, 2077, 1937, 666, 435, 2212, 196, 241, 51, 2077, - /* 1060 */ 3, 2077, 2077, 1546, 1556, 584, 33, 436, 1936, 1572, - /* 1070 */ 1575, 650, 38, 37, 1884, 617, 44, 42, 41, 40, - /* 1080 */ 39, 2207, 1884, 1491, 2108, 1489, 696, 629, 2144, 1928, - /* 1090 */ 2063, 109, 2110, 670, 2112, 2113, 665, 700, 660, 701, - /* 1100 */ 1928, 620, 1928, 653, 1884, 2197, 152, 2109, 306, 383, - /* 1110 */ 2193, 1914, 1494, 1495, 540, 1545, 1548, 1549, 1550, 1551, - /* 1120 */ 1552, 1553, 1554, 1555, 662, 658, 1564, 1565, 1567, 1568, - /* 1130 */ 1569, 1570, 2, 45, 43, 163, 1256, 587, 409, 1637, - /* 1140 */ 2127, 389, 650, 1490, 570, 650, 569, 702, 667, 250, - /* 1150 */ 145, 152, 134, 2077, 1571, 666, 1488, 228, 633, 573, - /* 1160 */ 226, 287, 245, 1458, 661, 154, 153, 699, 698, 697, - /* 1170 */ 151, 230, 359, 416, 229, 1884, 38, 37, 1884, 1566, - /* 1180 */ 44, 42, 41, 40, 39, 232, 2108, 650, 231, 249, - /* 1190 */ 2144, 63, 1496, 109, 2110, 670, 2112, 2113, 665, 650, - /* 1200 */ 660, 650, 1826, 647, 1634, 2288, 588, 2197, 1461, 52, - /* 1210 */ 2268, 383, 2193, 2109, 1499, 648, 63, 293, 757, 650, - /* 1220 */ 1884, 14, 2231, 1773, 234, 2274, 182, 233, 1766, 255, - /* 1230 */ 2269, 614, 1884, 90, 1884, 396, 2109, 1498, 360, 105, - /* 1240 */ 358, 357, 1764, 536, 541, 553, 2127, 538, 1666, 102, - /* 1250 */ 555, 2273, 1884, 152, 667, 2268, 1730, 1573, 1574, 2077, - /* 1260 */ 2237, 666, 1712, 1713, 558, 268, 1254, 710, 537, 2127, - /* 1270 */ 601, 2272, 657, 1665, 47, 2269, 2270, 667, 280, 13, - /* 1280 */ 12, 70, 2077, 221, 666, 262, 257, 1546, 1556, 1234, - /* 1290 */ 2128, 400, 2108, 1572, 1575, 1991, 2144, 1754, 1215, 110, - /* 1300 */ 2110, 670, 2112, 2113, 665, 618, 660, 1491, 150, 1489, - /* 1310 */ 632, 152, 1925, 2197, 63, 2108, 47, 2196, 2193, 2144, - /* 1320 */ 47, 674, 168, 2110, 670, 2112, 2113, 665, 2109, 660, - /* 1330 */ 2227, 1418, 150, 621, 1216, 285, 1494, 1495, 642, 1545, - /* 1340 */ 1548, 1549, 1550, 1551, 1552, 1553, 1554, 1555, 662, 658, - /* 1350 */ 1564, 1565, 1567, 1568, 1569, 1570, 2, 711, 626, 270, - /* 1360 */ 267, 2127, 1, 589, 2234, 289, 4, 1502, 1299, 667, - /* 1370 */ 152, 1608, 135, 1557, 2077, 403, 666, 305, 1327, 1232, - /* 1380 */ 150, 1760, 408, 350, 1441, 300, 437, 195, 1518, 1331, - /* 1390 */ 1501, 1992, 441, 446, 474, 459, 1513, 1984, 2109, 466, - /* 1400 */ 484, 473, 475, 485, 482, 199, 487, 2108, 488, 200, - /* 1410 */ 202, 2144, 490, 492, 109, 2110, 670, 2112, 2113, 665, - /* 1420 */ 1519, 660, 493, 1592, 502, 1521, 2288, 1338, 2197, 1336, - /* 1430 */ 752, 2127, 383, 2193, 505, 208, 1516, 155, 210, 667, - /* 1440 */ 1520, 506, 507, 2244, 2077, 1522, 666, 508, 510, 2109, - /* 1450 */ 213, 215, 84, 1188, 514, 531, 85, 219, 532, 533, - /* 1460 */ 535, 1874, 225, 1870, 227, 157, 111, 340, 158, 572, - /* 1470 */ 88, 2053, 2050, 578, 574, 148, 579, 2108, 242, 244, - /* 1480 */ 301, 2144, 2127, 1872, 109, 2110, 670, 2112, 2113, 665, - /* 1490 */ 667, 660, 577, 1868, 159, 2077, 2288, 666, 2197, 386, - /* 1500 */ 385, 2049, 383, 2193, 160, 246, 1448, 585, 2228, 1504, - /* 1510 */ 582, 602, 2238, 595, 7, 592, 2243, 640, 611, 583, - /* 1520 */ 1571, 598, 1497, 372, 2109, 605, 593, 373, 2108, 591, - /* 1530 */ 2242, 253, 2144, 261, 256, 109, 2110, 670, 2112, 2113, - /* 1540 */ 665, 2219, 660, 622, 590, 1566, 2267, 2288, 619, 2197, - /* 1550 */ 1634, 2291, 269, 383, 2193, 138, 1517, 2127, 1496, 266, - /* 1560 */ 630, 2213, 376, 275, 2262, 667, 95, 1523, 1997, 302, - /* 1570 */ 2077, 638, 666, 172, 639, 2011, 2010, 2009, 303, 643, - /* 1580 */ 379, 644, 60, 264, 656, 97, 304, 1885, 263, 99, - /* 1590 */ 2178, 1929, 101, 265, 1847, 672, 296, 307, 753, 342, - /* 1600 */ 50, 754, 331, 2108, 756, 343, 316, 2144, 330, 576, - /* 1610 */ 109, 2110, 670, 2112, 2113, 665, 311, 660, 320, 309, - /* 1620 */ 2069, 2068, 2288, 2067, 2197, 77, 2064, 760, 383, 2193, - /* 1630 */ 405, 406, 1481, 1482, 188, 410, 2062, 412, 413, 2216, - /* 1640 */ 414, 299, 2061, 2109, 351, 2059, 418, 2058, 2057, 420, - /* 1650 */ 422, 78, 1444, 1443, 2023, 2022, 175, 2021, 429, 430, - /* 1660 */ 2020, 2019, 750, 746, 742, 738, 297, 1975, 1974, 1395, - /* 1670 */ 1972, 144, 1971, 1505, 1970, 1500, 2127, 193, 447, 1963, - /* 1680 */ 449, 1977, 1962, 1961, 667, 1973, 1969, 1968, 1966, 2077, - /* 1690 */ 1965, 666, 1964, 1960, 1959, 1958, 1957, 1956, 1955, 1954, - /* 1700 */ 1953, 1952, 1508, 1510, 1951, 1950, 1949, 107, 2109, 1948, - /* 1710 */ 290, 146, 1947, 1946, 1945, 658, 1564, 1565, 1567, 1568, - /* 1720 */ 1569, 1570, 2108, 1976, 1944, 1943, 2144, 1942, 1941, 110, - /* 1730 */ 2110, 670, 2112, 2113, 665, 1940, 660, 1397, 1939, 477, - /* 1740 */ 1938, 2127, 646, 2197, 1270, 338, 339, 655, 2193, 667, - /* 1750 */ 1789, 1274, 1266, 1788, 2077, 1787, 666, 1785, 1749, 75, - /* 1760 */ 201, 206, 177, 2097, 1748, 203, 204, 2040, 1174, 76, - /* 1770 */ 1173, 2030, 2018, 2109, 207, 212, 2017, 277, 498, 500, - /* 1780 */ 1995, 1863, 276, 214, 1784, 1782, 1208, 668, 517, 516, - /* 1790 */ 1780, 2144, 520, 1778, 110, 2110, 670, 2112, 2113, 665, - /* 1800 */ 515, 660, 243, 519, 523, 521, 2127, 524, 2197, 525, - /* 1810 */ 1776, 528, 345, 2193, 667, 1763, 529, 527, 1762, 2077, - /* 1820 */ 1745, 666, 1865, 1343, 2109, 1342, 1864, 1257, 1244, 1255, - /* 1830 */ 725, 1253, 1252, 1251, 1250, 62, 1249, 1246, 1774, 1245, - /* 1840 */ 224, 363, 1243, 2109, 727, 1767, 364, 1765, 559, 365, - /* 1850 */ 1744, 561, 2108, 1743, 1742, 556, 2144, 2127, 563, 110, - /* 1860 */ 2110, 670, 2112, 2113, 665, 667, 660, 1468, 565, 112, - /* 1870 */ 2077, 1470, 666, 2197, 1467, 26, 2127, 2039, 2194, 1472, - /* 1880 */ 1452, 2029, 1454, 66, 667, 580, 1450, 2016, 2014, 2077, - /* 1890 */ 16, 666, 2273, 20, 64, 17, 19, 2015, 2013, 162, - /* 1900 */ 58, 59, 1697, 2108, 596, 55, 259, 2144, 28, 247, - /* 1910 */ 168, 2110, 670, 2112, 2113, 665, 581, 660, 2109, 369, - /* 1920 */ 586, 252, 2108, 260, 5, 2098, 2144, 594, 30, 322, - /* 1930 */ 2110, 670, 2112, 2113, 665, 1682, 660, 2109, 6, 21, - /* 1940 */ 1696, 254, 1664, 170, 258, 374, 1701, 1700, 29, 375, - /* 1950 */ 272, 2127, 2235, 57, 173, 2012, 1656, 1994, 641, 667, - /* 1960 */ 91, 56, 1702, 94, 2077, 1703, 666, 1631, 278, 1630, - /* 1970 */ 2127, 93, 22, 610, 279, 378, 1662, 1993, 667, 281, - /* 1980 */ 286, 96, 67, 2077, 102, 666, 288, 291, 23, 10, - /* 1990 */ 12, 1583, 98, 1506, 1582, 1561, 174, 2108, 186, 2147, - /* 2000 */ 659, 2144, 1559, 1558, 169, 2110, 670, 2112, 2113, 665, - /* 2010 */ 2109, 660, 1538, 36, 673, 15, 2108, 24, 391, 1530, - /* 2020 */ 2144, 677, 25, 329, 2110, 670, 2112, 2113, 665, 1593, - /* 2030 */ 660, 671, 1328, 675, 680, 1325, 2109, 678, 683, 1322, - /* 2040 */ 1316, 681, 686, 2127, 1314, 684, 687, 669, 1320, 1319, - /* 2050 */ 1318, 664, 1317, 1305, 103, 294, 2077, 2290, 666, 693, - /* 2060 */ 104, 1337, 74, 1333, 1240, 1206, 703, 1239, 1238, 2127, - /* 2070 */ 1237, 1236, 1264, 1235, 388, 1233, 1224, 667, 1231, 1230, - /* 2080 */ 1229, 1227, 2077, 295, 666, 715, 1226, 1225, 1223, 2108, - /* 2090 */ 1222, 1221, 1261, 2144, 1259, 1218, 328, 2110, 670, 2112, - /* 2100 */ 2113, 665, 1217, 660, 1214, 2163, 1213, 2109, 1212, 1211, - /* 2110 */ 1781, 735, 736, 1779, 737, 2108, 739, 741, 740, 2144, - /* 2120 */ 1777, 744, 329, 2110, 670, 2112, 2113, 665, 2109, 660, - /* 2130 */ 743, 745, 1775, 748, 747, 749, 1761, 751, 1163, 1741, - /* 2140 */ 2127, 298, 755, 759, 1492, 390, 308, 758, 667, 1716, - /* 2150 */ 1716, 1716, 1716, 2077, 1716, 666, 1716, 1716, 2109, 1716, - /* 2160 */ 1716, 2127, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 667, - /* 2170 */ 1716, 1716, 1716, 1716, 2077, 1716, 666, 1716, 1716, 1716, - /* 2180 */ 1716, 1716, 1716, 1716, 1716, 1716, 2108, 1716, 1716, 1716, - /* 2190 */ 2144, 2127, 1716, 329, 2110, 670, 2112, 2113, 665, 667, - /* 2200 */ 660, 1716, 1716, 1716, 2077, 1716, 666, 571, 1716, 1716, - /* 2210 */ 1716, 2144, 1716, 1716, 324, 2110, 670, 2112, 2113, 665, - /* 2220 */ 1716, 660, 1716, 2109, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2230 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 2108, 1716, 1716, - /* 2240 */ 1716, 2144, 1716, 1716, 313, 2110, 670, 2112, 2113, 665, - /* 2250 */ 1716, 660, 2109, 1716, 1716, 1716, 2127, 1716, 1716, 1716, - /* 2260 */ 1716, 1716, 1716, 1716, 667, 1716, 1716, 1716, 1716, 2077, - /* 2270 */ 1716, 666, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2280 */ 1716, 1716, 1716, 1716, 1716, 2127, 2109, 1716, 1716, 1716, - /* 2290 */ 1716, 1716, 1716, 667, 1716, 1716, 1716, 1716, 2077, 1716, - /* 2300 */ 666, 1716, 2108, 1716, 1716, 1716, 2144, 1716, 1716, 314, - /* 2310 */ 2110, 670, 2112, 2113, 665, 1716, 660, 1716, 1716, 2127, - /* 2320 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 667, 1716, 1716, - /* 2330 */ 1716, 2108, 2077, 1716, 666, 2144, 1716, 1716, 315, 2110, - /* 2340 */ 670, 2112, 2113, 665, 1716, 660, 1716, 1716, 1716, 1716, - /* 2350 */ 2109, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2360 */ 1716, 1716, 1716, 1716, 1716, 2108, 1716, 1716, 1716, 2144, - /* 2370 */ 2109, 1716, 321, 2110, 670, 2112, 2113, 665, 1716, 660, - /* 2380 */ 1716, 1716, 1716, 2127, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2390 */ 1716, 667, 1716, 1716, 1716, 1716, 2077, 1716, 666, 1716, - /* 2400 */ 1716, 1716, 1716, 2127, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2410 */ 1716, 667, 1716, 1716, 1716, 1716, 2077, 1716, 666, 1716, - /* 2420 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 2108, - /* 2430 */ 1716, 2109, 1716, 2144, 1716, 1716, 325, 2110, 670, 2112, - /* 2440 */ 2113, 665, 1716, 660, 1716, 1716, 1716, 1716, 1716, 2108, - /* 2450 */ 1716, 1716, 1716, 2144, 1716, 1716, 317, 2110, 670, 2112, - /* 2460 */ 2113, 665, 1716, 660, 2127, 1716, 1716, 1716, 1716, 1716, - /* 2470 */ 1716, 1716, 667, 1716, 1716, 1716, 1716, 2077, 1716, 666, - /* 2480 */ 1716, 1716, 2109, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2490 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2500 */ 1716, 1716, 1716, 2109, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2510 */ 2108, 1716, 1716, 1716, 2144, 2127, 1716, 326, 2110, 670, - /* 2520 */ 2112, 2113, 665, 667, 660, 1716, 1716, 1716, 2077, 1716, - /* 2530 */ 666, 1716, 1716, 2109, 1716, 1716, 2127, 1716, 1716, 1716, - /* 2540 */ 1716, 1716, 1716, 1716, 667, 1716, 1716, 1716, 1716, 2077, - /* 2550 */ 1716, 666, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2560 */ 1716, 2108, 1716, 1716, 1716, 2144, 2127, 1716, 318, 2110, - /* 2570 */ 670, 2112, 2113, 665, 667, 660, 1716, 1716, 1716, 2077, - /* 2580 */ 1716, 666, 2108, 1716, 1716, 1716, 2144, 1716, 1716, 327, - /* 2590 */ 2110, 670, 2112, 2113, 665, 1716, 660, 1716, 2109, 1716, - /* 2600 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2610 */ 1716, 1716, 2108, 1716, 1716, 1716, 2144, 1716, 1716, 319, - /* 2620 */ 2110, 670, 2112, 2113, 665, 1716, 660, 2109, 1716, 1716, - /* 2630 */ 1716, 2127, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 667, - /* 2640 */ 1716, 1716, 1716, 1716, 2077, 1716, 666, 1716, 1716, 1716, - /* 2650 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2660 */ 2127, 2109, 1716, 1716, 1716, 1716, 1716, 1716, 667, 1716, - /* 2670 */ 1716, 1716, 1716, 2077, 1716, 666, 1716, 2108, 1716, 1716, - /* 2680 */ 1716, 2144, 1716, 1716, 332, 2110, 670, 2112, 2113, 665, - /* 2690 */ 1716, 660, 1716, 1716, 2127, 1716, 1716, 1716, 1716, 1716, - /* 2700 */ 1716, 1716, 667, 1716, 1716, 1716, 2108, 2077, 1716, 666, - /* 2710 */ 2144, 1716, 1716, 333, 2110, 670, 2112, 2113, 665, 1716, - /* 2720 */ 660, 1716, 1716, 1716, 1716, 2109, 1716, 1716, 1716, 1716, - /* 2730 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2740 */ 2108, 1716, 1716, 1716, 2144, 2109, 1716, 2121, 2110, 670, - /* 2750 */ 2112, 2113, 665, 1716, 660, 1716, 1716, 1716, 2127, 1716, - /* 2760 */ 1716, 1716, 1716, 1716, 1716, 1716, 667, 1716, 1716, 1716, - /* 2770 */ 1716, 2077, 1716, 666, 1716, 1716, 1716, 1716, 2127, 1716, - /* 2780 */ 1716, 1716, 1716, 1716, 1716, 1716, 667, 1716, 1716, 1716, - /* 2790 */ 1716, 2077, 1716, 666, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2800 */ 1716, 1716, 1716, 1716, 2108, 1716, 2109, 1716, 2144, 1716, - /* 2810 */ 1716, 2120, 2110, 670, 2112, 2113, 665, 1716, 660, 1716, - /* 2820 */ 1716, 1716, 1716, 1716, 2108, 1716, 1716, 1716, 2144, 1716, - /* 2830 */ 1716, 2119, 2110, 670, 2112, 2113, 665, 1716, 660, 2127, - /* 2840 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 667, 1716, 1716, - /* 2850 */ 1716, 1716, 2077, 1716, 666, 1716, 1716, 2109, 1716, 1716, - /* 2860 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2870 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 2109, 1716, - /* 2880 */ 1716, 1716, 1716, 1716, 1716, 2108, 1716, 1716, 1716, 2144, - /* 2890 */ 2127, 1716, 347, 2110, 670, 2112, 2113, 665, 667, 660, - /* 2900 */ 1716, 1716, 1716, 2077, 1716, 666, 1716, 1716, 2109, 1716, - /* 2910 */ 1716, 2127, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 667, - /* 2920 */ 1716, 1716, 1716, 1716, 2077, 1716, 666, 1716, 1716, 1716, - /* 2930 */ 1716, 1716, 1716, 1716, 1716, 1716, 2108, 1716, 1716, 1716, - /* 2940 */ 2144, 2127, 1716, 348, 2110, 670, 2112, 2113, 665, 667, - /* 2950 */ 660, 1716, 1716, 1716, 2077, 1716, 666, 2108, 1716, 1716, - /* 2960 */ 1716, 2144, 1716, 1716, 344, 2110, 670, 2112, 2113, 665, - /* 2970 */ 1716, 660, 1716, 2109, 1716, 1716, 1716, 1716, 1716, 1716, - /* 2980 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 2108, 1716, 1716, - /* 2990 */ 1716, 2144, 1716, 1716, 349, 2110, 670, 2112, 2113, 665, - /* 3000 */ 1716, 660, 2109, 1716, 1716, 1716, 2127, 1716, 1716, 1716, - /* 3010 */ 1716, 1716, 1716, 1716, 667, 1716, 1716, 1716, 1716, 2077, - /* 3020 */ 1716, 666, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 3030 */ 1716, 1716, 1716, 1716, 1716, 2127, 1716, 1716, 1716, 1716, - /* 3040 */ 1716, 1716, 1716, 667, 1716, 1716, 1716, 1716, 2077, 1716, - /* 3050 */ 666, 1716, 668, 1716, 1716, 1716, 2144, 1716, 1716, 324, - /* 3060 */ 2110, 670, 2112, 2113, 665, 1716, 660, 1716, 1716, 1716, - /* 3070 */ 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, 1716, - /* 3080 */ 1716, 2108, 1716, 1716, 1716, 2144, 1716, 1716, 323, 2110, - /* 3090 */ 670, 2112, 2113, 665, 1716, 660, + /* 0 */ 2116, 209, 2010, 1881, 432, 502, 1883, 1765, 431, 2102, + /* 10 */ 670, 2010, 48, 46, 1646, 1722, 2098, 2008, 640, 2098, + /* 20 */ 393, 1522, 1495, 384, 41, 40, 2007, 640, 47, 45, + /* 30 */ 44, 43, 42, 1576, 1791, 1493, 2134, 180, 1520, 132, + /* 40 */ 131, 130, 129, 128, 127, 126, 125, 124, 2084, 1931, + /* 50 */ 669, 591, 2094, 2100, 2275, 2094, 2100, 388, 398, 1571, + /* 60 */ 30, 1937, 1939, 663, 499, 19, 663, 500, 1758, 2281, + /* 70 */ 184, 168, 1501, 1734, 2276, 617, 1520, 628, 140, 1869, + /* 80 */ 507, 2115, 107, 500, 1758, 2151, 38, 296, 169, 2117, + /* 90 */ 673, 2119, 2120, 668, 655, 663, 2176, 758, 141, 9, + /* 100 */ 15, 735, 734, 733, 732, 403, 1884, 731, 730, 144, + /* 110 */ 725, 724, 723, 722, 721, 720, 719, 157, 715, 714, + /* 120 */ 713, 402, 401, 710, 709, 708, 707, 706, 592, 2241, + /* 130 */ 62, 2219, 1320, 1938, 1939, 123, 1578, 1579, 122, 121, + /* 140 */ 120, 119, 118, 117, 116, 115, 114, 1311, 695, 694, + /* 150 */ 693, 1315, 692, 1317, 1318, 691, 688, 2216, 1326, 685, + /* 160 */ 1328, 1329, 682, 679, 504, 652, 1551, 1561, 2280, 277, + /* 170 */ 501, 2275, 1577, 1580, 1944, 653, 1892, 630, 182, 2212, + /* 180 */ 2213, 356, 138, 2217, 1409, 1410, 1496, 2279, 1494, 1942, + /* 190 */ 1720, 2276, 2278, 133, 287, 288, 1989, 41, 40, 286, + /* 200 */ 537, 47, 45, 44, 43, 42, 278, 52, 703, 155, + /* 210 */ 154, 700, 699, 698, 152, 1499, 1500, 1794, 1550, 1553, + /* 220 */ 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, + /* 230 */ 1570, 1572, 1573, 1574, 1575, 2, 48, 46, 425, 1169, + /* 240 */ 192, 341, 62, 1518, 393, 51, 1495, 62, 87, 93, + /* 250 */ 469, 2116, 616, 483, 350, 2275, 482, 1576, 177, 1493, + /* 260 */ 406, 670, 427, 423, 405, 47, 45, 44, 43, 42, + /* 270 */ 615, 184, 452, 1888, 484, 2276, 617, 454, 1171, 1994, + /* 280 */ 1174, 1175, 187, 1571, 555, 554, 553, 2134, 1723, 19, + /* 290 */ 106, 545, 137, 549, 551, 550, 1501, 548, 1605, 2084, + /* 300 */ 103, 669, 547, 552, 366, 365, 1521, 591, 546, 123, + /* 310 */ 2275, 1639, 122, 121, 120, 119, 118, 117, 116, 115, + /* 320 */ 114, 758, 359, 101, 15, 2281, 184, 430, 2280, 429, + /* 330 */ 2276, 617, 2115, 1191, 442, 1190, 2151, 653, 1892, 110, + /* 340 */ 2117, 673, 2119, 2120, 668, 1522, 663, 1885, 226, 143, + /* 350 */ 438, 150, 2175, 2204, 1606, 133, 428, 387, 2200, 187, + /* 360 */ 1578, 1579, 542, 480, 652, 1192, 474, 473, 472, 471, + /* 370 */ 468, 467, 466, 465, 464, 460, 459, 458, 457, 340, + /* 380 */ 449, 448, 447, 177, 444, 443, 357, 653, 1892, 638, + /* 390 */ 1551, 1561, 2280, 338, 187, 2275, 1577, 1580, 165, 187, + /* 400 */ 555, 554, 553, 358, 1993, 189, 1895, 545, 137, 549, + /* 410 */ 1496, 2279, 1494, 548, 606, 2276, 2277, 2070, 547, 552, + /* 420 */ 366, 365, 1354, 1355, 546, 2279, 1708, 1264, 37, 391, + /* 430 */ 1600, 1601, 1602, 1603, 1604, 1608, 1609, 1610, 1611, 1499, + /* 440 */ 1500, 1974, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, + /* 450 */ 1560, 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, + /* 460 */ 12, 48, 46, 227, 1266, 1495, 413, 1521, 610, 393, + /* 470 */ 2116, 1495, 652, 196, 167, 1674, 516, 1520, 1493, 172, + /* 480 */ 631, 1833, 1576, 476, 1493, 533, 529, 525, 521, 224, + /* 490 */ 253, 41, 40, 277, 2134, 47, 45, 44, 43, 42, + /* 500 */ 612, 607, 600, 1523, 611, 66, 2134, 2103, 1571, 12, + /* 510 */ 1191, 10, 1190, 84, 19, 1501, 83, 2098, 2084, 639, + /* 520 */ 669, 1501, 603, 602, 1672, 1673, 1675, 1676, 1677, 88, + /* 530 */ 41, 40, 222, 91, 47, 45, 44, 43, 42, 1868, + /* 540 */ 758, 211, 1192, 200, 199, 502, 758, 1765, 609, 15, + /* 550 */ 1650, 2115, 51, 2094, 2100, 2151, 1520, 62, 110, 2117, + /* 560 */ 673, 2119, 2120, 668, 663, 663, 475, 2051, 166, 514, + /* 570 */ 181, 2003, 2204, 316, 41, 40, 387, 2200, 47, 45, + /* 580 */ 44, 43, 42, 1426, 1427, 1578, 1579, 314, 73, 186, + /* 590 */ 657, 72, 2176, 718, 560, 1854, 456, 2230, 1745, 221, + /* 600 */ 215, 62, 705, 705, 220, 455, 512, 14, 13, 570, + /* 610 */ 207, 495, 493, 490, 244, 1551, 1561, 396, 385, 1425, + /* 620 */ 1428, 1577, 1580, 240, 213, 162, 165, 653, 1892, 1496, + /* 630 */ 1177, 1494, 372, 1894, 1894, 1496, 1519, 1494, 616, 563, + /* 640 */ 1942, 2275, 653, 1892, 557, 57, 2084, 1519, 1944, 239, + /* 650 */ 62, 255, 44, 43, 42, 371, 615, 184, 1499, 1500, + /* 660 */ 436, 2276, 617, 1942, 1499, 1500, 1523, 1550, 1553, 1554, + /* 670 */ 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, + /* 680 */ 1572, 1573, 1574, 1575, 2, 48, 46, 1581, 109, 70, + /* 690 */ 1944, 2116, 69, 393, 619, 1495, 1744, 381, 653, 1892, + /* 700 */ 1715, 631, 1552, 1221, 187, 1942, 1576, 1619, 1493, 187, + /* 710 */ 2219, 34, 729, 727, 628, 140, 437, 41, 40, 653, + /* 720 */ 1892, 47, 45, 44, 43, 42, 578, 2134, 81, 80, + /* 730 */ 435, 541, 1571, 191, 164, 540, 2215, 446, 639, 2084, + /* 740 */ 1222, 669, 41, 40, 2084, 1501, 47, 45, 44, 43, + /* 750 */ 42, 653, 1892, 187, 339, 142, 697, 421, 2175, 1935, + /* 760 */ 419, 415, 411, 408, 428, 12, 243, 653, 1892, 461, + /* 770 */ 758, 1501, 2115, 49, 653, 1892, 2151, 2116, 639, 110, + /* 780 */ 2117, 673, 2119, 2120, 668, 462, 663, 670, 637, 1767, + /* 790 */ 2003, 181, 515, 2204, 36, 396, 87, 387, 2200, 1714, + /* 800 */ 41, 40, 187, 165, 47, 45, 44, 43, 42, 1578, + /* 810 */ 1579, 1894, 360, 2134, 193, 275, 2212, 627, 2231, 134, + /* 820 */ 626, 1887, 2275, 628, 140, 2084, 696, 669, 648, 1944, + /* 830 */ 2003, 653, 1892, 1468, 1469, 1685, 386, 615, 184, 1551, + /* 840 */ 1561, 1989, 2276, 617, 1942, 1577, 1580, 41, 40, 1889, + /* 850 */ 717, 47, 45, 44, 43, 42, 1867, 590, 2115, 1496, + /* 860 */ 1275, 1494, 2151, 653, 1892, 110, 2117, 673, 2119, 2120, + /* 870 */ 668, 1944, 663, 1274, 242, 1944, 1989, 2295, 241, 2204, + /* 880 */ 363, 245, 397, 387, 2200, 194, 1943, 634, 1499, 1500, + /* 890 */ 1942, 1550, 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, + /* 900 */ 665, 661, 1569, 1570, 1572, 1573, 1574, 1575, 2, 48, + /* 910 */ 46, 2116, 1743, 1174, 1175, 653, 1892, 393, 1742, 1495, + /* 920 */ 198, 670, 1877, 2238, 183, 2212, 2213, 2067, 138, 2217, + /* 930 */ 1576, 2116, 1493, 587, 591, 485, 1523, 2275, 90, 345, + /* 940 */ 1662, 667, 370, 1643, 571, 1741, 364, 2134, 362, 361, + /* 950 */ 1740, 539, 2281, 184, 1879, 569, 1571, 2276, 617, 2084, + /* 960 */ 2084, 669, 653, 1892, 653, 1892, 2084, 2134, 567, 1501, + /* 970 */ 565, 701, 541, 1875, 1935, 702, 540, 55, 1935, 2084, + /* 980 */ 632, 669, 636, 2219, 1896, 703, 155, 154, 700, 699, + /* 990 */ 698, 152, 2115, 2084, 758, 2077, 2151, 49, 2084, 110, + /* 1000 */ 2117, 673, 2119, 2120, 668, 2116, 663, 2078, 54, 2214, + /* 1010 */ 3, 2295, 2115, 2204, 1520, 670, 2151, 387, 2200, 332, + /* 1020 */ 2117, 673, 2119, 2120, 668, 666, 663, 654, 2169, 653, + /* 1030 */ 1892, 41, 40, 1578, 1579, 47, 45, 44, 43, 42, + /* 1040 */ 399, 2134, 591, 1739, 249, 2275, 1586, 291, 165, 653, + /* 1050 */ 1892, 153, 1520, 2084, 591, 669, 1894, 2275, 653, 1892, + /* 1060 */ 2281, 184, 1279, 1551, 1561, 2276, 617, 650, 1607, 1577, + /* 1070 */ 1580, 232, 2281, 184, 230, 1278, 651, 2276, 617, 2224, + /* 1080 */ 1639, 653, 1892, 1496, 254, 1494, 2115, 1738, 653, 1892, + /* 1090 */ 2151, 2084, 439, 170, 2117, 673, 2119, 2120, 668, 297, + /* 1100 */ 663, 576, 1737, 628, 140, 440, 400, 1736, 56, 146, + /* 1110 */ 1733, 135, 1499, 1500, 1732, 1550, 1553, 1554, 1555, 1556, + /* 1120 */ 1557, 1558, 1559, 1560, 665, 661, 1569, 1570, 1572, 1573, + /* 1130 */ 1574, 1575, 2, 48, 46, 2084, 420, 487, 1870, 1731, + /* 1140 */ 1730, 393, 35, 1495, 618, 2296, 1729, 2116, 591, 623, + /* 1150 */ 2084, 2275, 1612, 1728, 1576, 2084, 1493, 670, 2084, 2251, + /* 1160 */ 1552, 620, 2084, 1727, 1726, 1725, 2281, 184, 74, 543, + /* 1170 */ 310, 2276, 617, 1921, 2116, 234, 236, 1504, 233, 235, + /* 1180 */ 1571, 544, 148, 2134, 670, 1781, 598, 2084, 2084, 153, + /* 1190 */ 252, 1262, 1642, 1501, 2084, 2084, 238, 669, 1552, 237, + /* 1200 */ 573, 2084, 572, 1260, 185, 2212, 2213, 556, 138, 2217, + /* 1210 */ 2134, 2084, 2084, 2084, 153, 50, 50, 82, 758, 1774, + /* 1220 */ 259, 15, 2084, 1772, 669, 1717, 1718, 664, 2115, 1834, + /* 1230 */ 14, 13, 2151, 153, 2116, 110, 2117, 673, 2119, 2120, + /* 1240 */ 668, 558, 663, 1503, 670, 561, 1463, 2295, 50, 2204, + /* 1250 */ 2105, 284, 660, 387, 2200, 2115, 71, 1578, 1579, 2151, + /* 1260 */ 151, 153, 110, 2117, 673, 2119, 2120, 668, 2244, 663, + /* 1270 */ 2134, 1466, 1671, 1670, 2295, 64, 2204, 261, 50, 1735, + /* 1280 */ 387, 2200, 2084, 225, 669, 272, 604, 1551, 1561, 2135, + /* 1290 */ 635, 266, 404, 1577, 1580, 1998, 703, 155, 154, 700, + /* 1300 */ 699, 698, 152, 2107, 1759, 1423, 1932, 1496, 289, 1494, + /* 1310 */ 2234, 50, 711, 645, 677, 2115, 151, 293, 1305, 2151, + /* 1320 */ 1768, 153, 169, 2117, 673, 2119, 2120, 668, 1764, 663, + /* 1330 */ 1507, 629, 1613, 136, 1240, 1562, 1499, 1500, 151, 1550, + /* 1340 */ 1553, 1554, 1555, 1556, 1557, 1558, 1559, 1560, 665, 661, + /* 1350 */ 1569, 1570, 1572, 1573, 1574, 1575, 2, 274, 271, 390, + /* 1360 */ 389, 712, 1, 2242, 5, 407, 412, 2116, 309, 1509, + /* 1370 */ 753, 1332, 354, 1336, 1446, 304, 197, 670, 1343, 2269, + /* 1380 */ 1576, 624, 1502, 1238, 441, 1523, 1999, 445, 450, 478, + /* 1390 */ 1341, 621, 1518, 463, 1991, 156, 1506, 2116, 470, 477, + /* 1400 */ 479, 488, 1597, 2134, 489, 486, 1571, 670, 201, 2223, + /* 1410 */ 202, 491, 492, 1524, 497, 2084, 204, 669, 494, 1501, + /* 1420 */ 496, 4, 498, 505, 506, 1526, 508, 212, 1521, 509, + /* 1430 */ 214, 1525, 1527, 2134, 510, 513, 511, 217, 1194, 517, + /* 1440 */ 112, 534, 535, 219, 659, 2084, 85, 669, 2115, 86, + /* 1450 */ 223, 536, 2151, 538, 344, 110, 2117, 673, 2119, 2120, + /* 1460 */ 668, 575, 663, 2060, 89, 2116, 577, 2295, 305, 2204, + /* 1470 */ 1882, 229, 1878, 387, 2200, 670, 231, 158, 2115, 246, + /* 1480 */ 159, 1880, 2151, 1876, 160, 110, 2117, 673, 2119, 2120, + /* 1490 */ 668, 161, 663, 581, 2057, 580, 250, 2295, 1453, 2204, + /* 1500 */ 588, 2134, 605, 387, 2200, 2056, 585, 595, 643, 601, + /* 1510 */ 2250, 2235, 582, 2084, 2249, 669, 248, 149, 2245, 8, + /* 1520 */ 376, 257, 608, 614, 586, 260, 596, 594, 2226, 593, + /* 1530 */ 267, 265, 377, 1510, 625, 1505, 2298, 173, 622, 1639, + /* 1540 */ 268, 139, 1522, 633, 380, 279, 2115, 96, 1528, 2004, + /* 1550 */ 2151, 270, 646, 110, 2117, 673, 2119, 2120, 668, 306, + /* 1560 */ 663, 641, 1513, 1515, 2220, 2179, 647, 2204, 98, 642, + /* 1570 */ 307, 387, 2200, 2116, 2018, 661, 1569, 1570, 1572, 1573, + /* 1580 */ 1574, 1575, 1893, 670, 100, 61, 269, 2017, 2185, 102, + /* 1590 */ 2016, 383, 308, 2116, 675, 1936, 1855, 300, 311, 2274, + /* 1600 */ 754, 757, 273, 670, 755, 53, 335, 320, 346, 2134, + /* 1610 */ 347, 313, 315, 2076, 334, 324, 2075, 2074, 78, 2071, + /* 1620 */ 409, 2084, 410, 669, 1486, 1487, 190, 414, 2069, 2134, + /* 1630 */ 416, 417, 418, 2068, 355, 2066, 422, 2065, 2064, 426, + /* 1640 */ 424, 2084, 79, 669, 1449, 1448, 2030, 2029, 2028, 433, + /* 1650 */ 434, 2027, 2026, 1400, 2115, 1982, 1981, 1979, 2151, 1978, + /* 1660 */ 1977, 110, 2117, 673, 2119, 2120, 668, 145, 663, 1980, + /* 1670 */ 1976, 1975, 1973, 2177, 2115, 2204, 195, 2116, 2151, 387, + /* 1680 */ 2200, 110, 2117, 673, 2119, 2120, 668, 670, 663, 1972, + /* 1690 */ 1971, 1970, 453, 656, 1984, 2204, 451, 1969, 1968, 387, + /* 1700 */ 2200, 2116, 1967, 1966, 1965, 1964, 1963, 1962, 1961, 1960, + /* 1710 */ 1959, 670, 1958, 2134, 1957, 1956, 1955, 147, 1954, 1953, + /* 1720 */ 1952, 1983, 1951, 1950, 1949, 2084, 1948, 669, 1947, 1946, + /* 1730 */ 1945, 481, 342, 1402, 343, 1276, 1797, 2134, 203, 1280, + /* 1740 */ 1796, 205, 1795, 1793, 206, 1754, 2104, 179, 1176, 2084, + /* 1750 */ 218, 669, 1753, 2047, 2037, 2025, 2024, 2002, 2115, 1871, + /* 1760 */ 1272, 1214, 2151, 1792, 1790, 111, 2117, 673, 2119, 2120, + /* 1770 */ 668, 76, 663, 208, 503, 210, 77, 178, 2116, 2204, + /* 1780 */ 216, 518, 2115, 2203, 2200, 1788, 2151, 519, 670, 111, + /* 1790 */ 2117, 673, 2119, 2120, 668, 522, 663, 520, 523, 1786, + /* 1800 */ 524, 2116, 526, 2204, 528, 1784, 530, 658, 2200, 1771, + /* 1810 */ 1770, 670, 532, 1750, 2134, 527, 1873, 531, 1348, 1347, + /* 1820 */ 1872, 1263, 1261, 1259, 2116, 1258, 2084, 1257, 669, 1256, + /* 1830 */ 1255, 1252, 1782, 726, 670, 728, 1251, 2134, 1249, 1250, + /* 1840 */ 367, 1775, 1773, 368, 559, 369, 562, 2116, 63, 2084, + /* 1850 */ 1749, 669, 1748, 1747, 228, 568, 113, 670, 2046, 671, + /* 1860 */ 2134, 564, 566, 2151, 1473, 374, 111, 2117, 673, 2119, + /* 1870 */ 2120, 668, 2084, 663, 669, 1475, 1477, 29, 1472, 67, + /* 1880 */ 2204, 1459, 2115, 2134, 349, 2200, 2151, 1457, 375, 111, + /* 1890 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1455, 2036, + /* 1900 */ 583, 2023, 2021, 2204, 20, 2115, 2280, 31, 2201, 2151, + /* 1910 */ 2116, 599, 333, 2117, 673, 2119, 2120, 668, 251, 663, + /* 1920 */ 670, 17, 6, 1687, 256, 7, 589, 264, 2115, 597, + /* 1930 */ 58, 258, 2151, 163, 21, 333, 2117, 673, 2119, 2120, + /* 1940 */ 668, 22, 663, 2105, 584, 1669, 2134, 171, 373, 262, + /* 1950 */ 263, 32, 33, 65, 24, 1661, 92, 23, 2084, 1702, + /* 1960 */ 669, 1701, 1707, 1708, 378, 1706, 1705, 379, 276, 1636, + /* 1970 */ 1635, 60, 174, 2022, 2020, 2116, 2019, 2001, 95, 94, + /* 1980 */ 282, 25, 2000, 283, 285, 670, 1667, 290, 68, 99, + /* 1990 */ 644, 2115, 97, 295, 292, 2151, 2116, 13, 326, 2117, + /* 2000 */ 673, 2119, 2120, 668, 103, 663, 670, 26, 1511, 2116, + /* 2010 */ 1588, 2134, 1587, 1598, 175, 11, 2154, 188, 1566, 667, + /* 2020 */ 662, 1543, 39, 2084, 59, 669, 674, 676, 18, 1564, + /* 2030 */ 1338, 672, 2134, 395, 298, 1563, 16, 382, 27, 680, + /* 2040 */ 28, 613, 1535, 1333, 2084, 2134, 669, 678, 683, 1330, + /* 2050 */ 1327, 681, 684, 686, 689, 1321, 2115, 2084, 687, 669, + /* 2060 */ 2151, 1319, 690, 170, 2117, 673, 2119, 2120, 668, 104, + /* 2070 */ 663, 105, 1342, 1246, 704, 75, 1212, 2115, 1325, 1324, + /* 2080 */ 1245, 2151, 1244, 2116, 333, 2117, 673, 2119, 2120, 668, + /* 2090 */ 2115, 663, 1243, 670, 2151, 1242, 1241, 332, 2117, 673, + /* 2100 */ 2119, 2120, 668, 2116, 663, 1323, 2170, 1322, 1239, 1237, + /* 2110 */ 1236, 1235, 1270, 670, 299, 2297, 716, 1233, 1232, 2134, + /* 2120 */ 1231, 1230, 1229, 1228, 392, 1227, 1267, 1265, 1224, 1223, + /* 2130 */ 1220, 2084, 1219, 669, 1218, 1217, 1789, 736, 737, 2134, + /* 2140 */ 738, 1787, 740, 742, 394, 741, 1785, 745, 744, 746, + /* 2150 */ 1783, 2084, 748, 669, 749, 750, 1769, 752, 1166, 1746, + /* 2160 */ 302, 756, 1721, 759, 2115, 1497, 312, 760, 2151, 1721, + /* 2170 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2180 */ 1721, 1721, 1721, 1721, 2115, 1721, 579, 1721, 2151, 1721, + /* 2190 */ 1721, 333, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2200 */ 2116, 1721, 1721, 1721, 761, 1721, 1721, 1721, 1721, 1721, + /* 2210 */ 670, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 303, 1721, + /* 2220 */ 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2230 */ 1721, 1721, 1721, 1721, 176, 1721, 2134, 1721, 1721, 1721, + /* 2240 */ 751, 747, 743, 739, 301, 1721, 2116, 1721, 2084, 1721, + /* 2250 */ 669, 1721, 2134, 1721, 1721, 1721, 670, 1721, 1721, 1721, + /* 2260 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, + /* 2270 */ 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, + /* 2280 */ 1721, 574, 2134, 1721, 108, 2151, 670, 294, 328, 2117, + /* 2290 */ 673, 2119, 2120, 668, 2084, 663, 669, 2115, 1721, 1721, + /* 2300 */ 1721, 2151, 1721, 1721, 317, 2117, 673, 2119, 2120, 668, + /* 2310 */ 1721, 663, 2134, 1721, 1721, 1721, 1721, 1721, 1721, 649, + /* 2320 */ 1721, 1721, 1721, 1721, 2084, 1721, 669, 2115, 1721, 1721, + /* 2330 */ 1721, 2151, 1721, 1721, 318, 2117, 673, 2119, 2120, 668, + /* 2340 */ 2116, 663, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2350 */ 670, 1721, 1721, 2116, 281, 1721, 1721, 2115, 1721, 280, + /* 2360 */ 1721, 2151, 1721, 670, 319, 2117, 673, 2119, 2120, 668, + /* 2370 */ 1721, 663, 1721, 1721, 1721, 1721, 2134, 1721, 1721, 247, + /* 2380 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2390 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2400 */ 1721, 2084, 1721, 669, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2410 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2420 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 670, 325, 2117, + /* 2430 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2440 */ 2134, 329, 2117, 673, 2119, 2120, 668, 1721, 663, 1721, + /* 2450 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2460 */ 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, + /* 2470 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2480 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 670, 2151, + /* 2490 */ 1721, 1721, 321, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2500 */ 1721, 1721, 2151, 1721, 1721, 330, 2117, 673, 2119, 2120, + /* 2510 */ 668, 1721, 663, 1721, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2520 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 2116, + /* 2530 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, + /* 2540 */ 1721, 1721, 1721, 1721, 2116, 1721, 1721, 1721, 1721, 1721, + /* 2550 */ 1721, 1721, 1721, 1721, 670, 1721, 1721, 1721, 1721, 2115, + /* 2560 */ 1721, 1721, 1721, 2151, 1721, 2134, 322, 2117, 673, 2119, + /* 2570 */ 2120, 668, 1721, 663, 1721, 1721, 1721, 2084, 1721, 669, + /* 2580 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2590 */ 2116, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, 1721, + /* 2600 */ 670, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2610 */ 2115, 1721, 1721, 1721, 2151, 1721, 1721, 331, 2117, 673, + /* 2620 */ 2119, 2120, 668, 1721, 663, 2115, 2134, 1721, 1721, 2151, + /* 2630 */ 1721, 1721, 323, 2117, 673, 2119, 2120, 668, 2084, 663, + /* 2640 */ 669, 2116, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2650 */ 1721, 670, 1721, 1721, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2660 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 670, 1721, + /* 2670 */ 1721, 2115, 1721, 1721, 1721, 2151, 1721, 2134, 336, 2117, + /* 2680 */ 673, 2119, 2120, 668, 1721, 663, 1721, 2116, 1721, 2084, + /* 2690 */ 1721, 669, 1721, 1721, 2134, 1721, 1721, 670, 1721, 1721, + /* 2700 */ 1721, 1721, 1721, 1721, 1721, 1721, 2084, 1721, 669, 1721, + /* 2710 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2720 */ 1721, 1721, 2115, 2134, 1721, 1721, 2151, 1721, 1721, 337, + /* 2730 */ 2117, 673, 2119, 2120, 668, 2084, 663, 669, 1721, 2115, + /* 2740 */ 1721, 1721, 1721, 2151, 1721, 1721, 2128, 2117, 673, 2119, + /* 2750 */ 2120, 668, 1721, 663, 1721, 2116, 1721, 1721, 1721, 1721, + /* 2760 */ 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, 2115, 1721, + /* 2770 */ 1721, 1721, 2151, 1721, 2116, 2127, 2117, 673, 2119, 2120, + /* 2780 */ 668, 1721, 663, 1721, 670, 1721, 1721, 2116, 1721, 1721, + /* 2790 */ 1721, 2134, 1721, 1721, 1721, 1721, 1721, 670, 1721, 1721, + /* 2800 */ 1721, 1721, 1721, 2084, 1721, 669, 1721, 1721, 1721, 1721, + /* 2810 */ 2134, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2820 */ 1721, 1721, 2084, 2134, 669, 1721, 1721, 1721, 1721, 1721, + /* 2830 */ 1721, 1721, 1721, 1721, 1721, 2084, 2115, 669, 1721, 1721, + /* 2840 */ 2151, 1721, 1721, 2126, 2117, 673, 2119, 2120, 668, 1721, + /* 2850 */ 663, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 2860 */ 2116, 1721, 351, 2117, 673, 2119, 2120, 668, 2115, 663, + /* 2870 */ 670, 1721, 2151, 2116, 1721, 352, 2117, 673, 2119, 2120, + /* 2880 */ 668, 1721, 663, 670, 1721, 1721, 1721, 1721, 2116, 1721, + /* 2890 */ 1721, 1721, 1721, 1721, 1721, 1721, 2134, 1721, 670, 1721, + /* 2900 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 2084, 2134, + /* 2910 */ 669, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 2920 */ 1721, 2084, 1721, 669, 2134, 1721, 1721, 1721, 1721, 1721, + /* 2930 */ 1721, 1721, 1721, 1721, 2116, 1721, 2084, 1721, 669, 1721, + /* 2940 */ 1721, 2115, 1721, 1721, 670, 2151, 1721, 1721, 348, 2117, + /* 2950 */ 673, 2119, 2120, 668, 2115, 663, 1721, 1721, 2151, 1721, + /* 2960 */ 1721, 353, 2117, 673, 2119, 2120, 668, 1721, 663, 671, + /* 2970 */ 2134, 1721, 1721, 2151, 1721, 1721, 328, 2117, 673, 2119, + /* 2980 */ 2120, 668, 2084, 663, 669, 1721, 1721, 1721, 1721, 1721, + /* 2990 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3000 */ 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, 1721, + /* 3010 */ 1721, 1721, 1721, 1721, 1721, 2115, 1721, 1721, 1721, 2151, + /* 3020 */ 1721, 1721, 327, 2117, 673, 2119, 2120, 668, 1721, 663, }; static const YYCODETYPE yy_lookahead[] = { - /* 0 */ 367, 333, 381, 398, 368, 367, 398, 402, 368, 332, - /* 10 */ 341, 334, 12, 13, 14, 379, 366, 396, 397, 379, - /* 20 */ 20, 445, 22, 8, 9, 449, 357, 12, 13, 14, - /* 30 */ 15, 16, 366, 33, 366, 35, 0, 387, 388, 373, - /* 40 */ 464, 465, 374, 374, 345, 469, 470, 379, 382, 381, - /* 50 */ 445, 415, 416, 445, 449, 415, 416, 449, 58, 20, - /* 60 */ 361, 445, 426, 365, 64, 449, 426, 381, 369, 464, - /* 70 */ 465, 71, 464, 465, 469, 470, 378, 469, 470, 393, - /* 80 */ 412, 465, 396, 397, 416, 469, 470, 419, 420, 421, - /* 90 */ 422, 423, 424, 425, 426, 427, 428, 97, 35, 20, - /* 100 */ 100, 14, 66, 67, 68, 69, 70, 20, 72, 73, - /* 110 */ 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, - /* 120 */ 84, 85, 86, 87, 88, 89, 90, 91, 92, 20, - /* 130 */ 337, 97, 339, 340, 71, 21, 136, 137, 24, 25, - /* 140 */ 26, 27, 28, 29, 30, 31, 32, 113, 114, 115, - /* 150 */ 116, 117, 118, 119, 120, 121, 122, 20, 124, 125, - /* 160 */ 126, 127, 128, 129, 341, 100, 166, 167, 3, 20, - /* 170 */ 8, 9, 172, 173, 12, 13, 14, 15, 16, 100, - /* 180 */ 357, 337, 366, 339, 340, 20, 186, 364, 188, 373, - /* 190 */ 20, 434, 435, 178, 20, 8, 9, 374, 382, 12, - /* 200 */ 13, 14, 15, 16, 131, 8, 9, 168, 135, 12, - /* 210 */ 13, 14, 15, 16, 333, 215, 216, 20, 218, 219, + /* 0 */ 333, 338, 383, 370, 400, 342, 371, 344, 404, 371, + /* 10 */ 343, 383, 12, 13, 14, 0, 381, 398, 399, 381, + /* 20 */ 20, 20, 22, 395, 8, 9, 398, 399, 12, 13, + /* 30 */ 14, 15, 16, 33, 0, 35, 369, 368, 20, 24, + /* 40 */ 25, 26, 27, 28, 29, 30, 31, 32, 381, 380, + /* 50 */ 383, 447, 417, 418, 450, 417, 418, 419, 379, 59, + /* 60 */ 44, 382, 383, 428, 337, 65, 428, 340, 341, 465, + /* 70 */ 466, 332, 72, 334, 470, 471, 20, 342, 343, 0, + /* 80 */ 337, 414, 348, 340, 341, 418, 436, 437, 421, 422, + /* 90 */ 423, 424, 425, 426, 432, 428, 434, 97, 364, 39, + /* 100 */ 100, 67, 68, 69, 70, 71, 372, 73, 74, 75, + /* 110 */ 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, + /* 120 */ 86, 87, 88, 89, 90, 91, 92, 93, 461, 462, + /* 130 */ 100, 420, 97, 382, 383, 21, 136, 137, 24, 25, + /* 140 */ 26, 27, 28, 29, 30, 31, 32, 112, 113, 114, + /* 150 */ 115, 116, 117, 118, 119, 120, 121, 446, 123, 124, + /* 160 */ 125, 126, 127, 128, 14, 20, 166, 167, 447, 168, + /* 170 */ 20, 450, 172, 173, 369, 342, 343, 442, 443, 444, + /* 180 */ 445, 376, 447, 448, 166, 167, 186, 466, 188, 384, + /* 190 */ 330, 470, 471, 360, 130, 131, 343, 8, 9, 135, + /* 200 */ 367, 12, 13, 14, 15, 16, 59, 100, 129, 130, + /* 210 */ 131, 132, 133, 134, 135, 215, 216, 0, 218, 219, /* 220 */ 220, 221, 222, 223, 224, 225, 226, 227, 228, 229, - /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 348, 63, - /* 240 */ 341, 18, 341, 20, 20, 355, 22, 366, 333, 100, - /* 250 */ 27, 64, 333, 30, 341, 374, 33, 33, 362, 35, - /* 260 */ 379, 445, 381, 101, 398, 449, 101, 100, 195, 196, - /* 270 */ 100, 48, 199, 50, 201, 374, 53, 20, 341, 368, - /* 280 */ 464, 465, 58, 1, 2, 469, 470, 374, 64, 390, - /* 290 */ 379, 392, 277, 412, 379, 71, 109, 416, 379, 20, - /* 300 */ 419, 420, 421, 422, 423, 424, 410, 426, 20, 58, - /* 310 */ 22, 445, 431, 347, 433, 449, 333, 252, 437, 438, - /* 320 */ 383, 97, 99, 35, 100, 347, 415, 416, 417, 363, - /* 330 */ 464, 465, 451, 168, 111, 469, 470, 426, 372, 51, - /* 340 */ 459, 440, 441, 442, 443, 0, 445, 446, 20, 366, - /* 350 */ 372, 100, 165, 102, 441, 442, 443, 374, 445, 446, - /* 360 */ 136, 137, 379, 140, 381, 168, 143, 144, 145, 146, + /* 230 */ 230, 231, 232, 233, 234, 235, 12, 13, 181, 4, + /* 240 */ 387, 18, 100, 20, 20, 100, 22, 100, 350, 102, + /* 250 */ 27, 333, 447, 30, 65, 450, 33, 33, 369, 35, + /* 260 */ 400, 343, 205, 206, 404, 12, 13, 14, 15, 16, + /* 270 */ 465, 466, 49, 375, 51, 470, 471, 54, 43, 390, + /* 280 */ 45, 46, 252, 59, 67, 68, 69, 369, 0, 65, + /* 290 */ 100, 74, 75, 76, 355, 356, 72, 80, 109, 381, + /* 300 */ 110, 383, 85, 86, 87, 88, 20, 447, 91, 21, + /* 310 */ 450, 251, 24, 25, 26, 27, 28, 29, 30, 31, + /* 320 */ 32, 97, 99, 348, 100, 465, 466, 185, 3, 187, + /* 330 */ 470, 471, 414, 20, 111, 22, 418, 342, 343, 421, + /* 340 */ 422, 423, 424, 425, 426, 20, 428, 372, 35, 431, + /* 350 */ 342, 433, 434, 435, 165, 360, 214, 439, 440, 252, + /* 360 */ 136, 137, 367, 140, 20, 52, 143, 144, 145, 146, /* 370 */ 147, 148, 149, 150, 151, 152, 153, 154, 155, 156, - /* 380 */ 157, 158, 159, 101, 161, 162, 163, 380, 381, 44, - /* 390 */ 166, 167, 333, 136, 137, 412, 172, 173, 21, 416, - /* 400 */ 358, 252, 419, 420, 421, 422, 423, 424, 366, 426, - /* 410 */ 186, 34, 188, 36, 431, 373, 433, 375, 171, 252, - /* 420 */ 437, 438, 252, 430, 382, 432, 239, 240, 241, 242, - /* 430 */ 243, 244, 245, 246, 247, 248, 249, 341, 379, 215, - /* 440 */ 216, 63, 218, 219, 220, 221, 222, 223, 224, 225, + /* 380 */ 157, 158, 159, 369, 161, 162, 163, 342, 343, 20, + /* 390 */ 166, 167, 447, 385, 252, 450, 172, 173, 369, 252, + /* 400 */ 67, 68, 69, 389, 390, 360, 377, 74, 75, 76, + /* 410 */ 186, 466, 188, 80, 171, 470, 471, 0, 85, 86, + /* 420 */ 87, 88, 136, 137, 91, 3, 101, 35, 239, 240, + /* 430 */ 241, 242, 243, 244, 245, 246, 247, 248, 249, 215, + /* 440 */ 216, 0, 218, 219, 220, 221, 222, 223, 224, 225, /* 450 */ 226, 227, 228, 229, 230, 231, 232, 233, 234, 235, - /* 460 */ 236, 12, 13, 377, 136, 137, 380, 381, 20, 20, - /* 470 */ 374, 22, 66, 67, 68, 215, 352, 353, 374, 73, - /* 480 */ 74, 75, 33, 130, 35, 79, 333, 33, 333, 385, - /* 490 */ 84, 85, 86, 87, 338, 20, 90, 341, 342, 0, - /* 500 */ 172, 173, 48, 252, 257, 258, 259, 58, 54, 55, - /* 510 */ 56, 57, 58, 64, 12, 13, 14, 15, 16, 341, - /* 520 */ 71, 366, 262, 263, 264, 265, 266, 267, 268, 374, - /* 530 */ 345, 341, 379, 4, 379, 357, 381, 441, 442, 443, - /* 540 */ 338, 445, 446, 341, 342, 449, 97, 100, 366, 100, - /* 550 */ 197, 198, 374, 99, 369, 373, 102, 14, 352, 353, - /* 560 */ 464, 465, 366, 20, 382, 469, 470, 412, 18, 0, - /* 570 */ 374, 416, 0, 23, 419, 420, 421, 422, 423, 424, - /* 580 */ 390, 426, 392, 3, 112, 136, 137, 37, 38, 181, - /* 590 */ 21, 41, 0, 24, 25, 26, 27, 28, 29, 30, - /* 600 */ 31, 32, 130, 131, 132, 133, 134, 135, 111, 59, - /* 610 */ 60, 61, 62, 205, 206, 166, 167, 421, 14, 15, - /* 620 */ 16, 172, 173, 169, 170, 236, 471, 472, 174, 100, - /* 630 */ 176, 112, 185, 366, 187, 186, 358, 188, 66, 67, - /* 640 */ 68, 166, 167, 341, 366, 73, 74, 75, 194, 368, - /* 650 */ 100, 79, 133, 375, 155, 388, 84, 85, 86, 87, - /* 660 */ 379, 214, 90, 164, 215, 216, 71, 218, 219, 220, + /* 460 */ 236, 12, 13, 33, 72, 22, 49, 20, 343, 20, + /* 470 */ 333, 22, 20, 59, 351, 215, 64, 20, 35, 49, + /* 480 */ 343, 358, 33, 81, 35, 55, 56, 57, 58, 59, + /* 490 */ 59, 8, 9, 168, 369, 12, 13, 14, 15, 16, + /* 500 */ 257, 258, 259, 20, 20, 4, 369, 371, 59, 236, + /* 510 */ 20, 238, 22, 99, 65, 72, 102, 381, 381, 342, + /* 520 */ 383, 72, 262, 263, 264, 265, 266, 267, 268, 99, + /* 530 */ 8, 9, 102, 102, 12, 13, 14, 15, 16, 0, + /* 540 */ 97, 338, 52, 141, 142, 342, 97, 344, 423, 100, + /* 550 */ 14, 414, 100, 417, 418, 418, 20, 100, 421, 422, + /* 560 */ 423, 424, 425, 426, 428, 428, 164, 365, 18, 392, + /* 570 */ 433, 394, 435, 23, 8, 9, 439, 440, 12, 13, + /* 580 */ 14, 15, 16, 136, 137, 136, 137, 37, 38, 452, + /* 590 */ 432, 41, 434, 357, 4, 359, 155, 460, 333, 169, + /* 600 */ 170, 100, 64, 64, 174, 164, 176, 1, 2, 19, + /* 610 */ 60, 61, 62, 63, 412, 166, 167, 361, 361, 172, + /* 620 */ 173, 172, 173, 33, 194, 369, 369, 342, 343, 186, + /* 630 */ 14, 188, 376, 377, 377, 186, 20, 188, 447, 49, + /* 640 */ 384, 450, 342, 343, 54, 360, 381, 20, 369, 59, + /* 650 */ 100, 168, 14, 15, 16, 376, 465, 466, 215, 216, + /* 660 */ 360, 470, 471, 384, 215, 216, 20, 218, 219, 220, /* 670 */ 221, 222, 223, 224, 225, 226, 227, 228, 229, 230, - /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 0, - /* 690 */ 333, 333, 390, 20, 392, 22, 415, 416, 417, 252, - /* 700 */ 236, 333, 238, 330, 112, 367, 33, 426, 35, 166, - /* 710 */ 20, 4, 22, 24, 25, 26, 27, 28, 29, 30, - /* 720 */ 31, 32, 130, 131, 132, 133, 134, 135, 178, 179, - /* 730 */ 180, 58, 341, 183, 366, 341, 379, 379, 45, 46, - /* 740 */ 333, 51, 374, 341, 71, 111, 80, 379, 357, 381, - /* 750 */ 43, 357, 45, 46, 204, 364, 22, 207, 71, 357, - /* 760 */ 210, 211, 212, 213, 214, 374, 368, 366, 374, 35, - /* 770 */ 97, 398, 22, 100, 373, 402, 374, 379, 0, 333, - /* 780 */ 412, 252, 47, 382, 416, 35, 379, 419, 420, 421, - /* 790 */ 422, 423, 424, 430, 426, 432, 367, 429, 14, 431, - /* 800 */ 432, 433, 252, 341, 20, 437, 438, 141, 142, 136, - /* 810 */ 137, 341, 366, 415, 416, 417, 366, 366, 445, 357, - /* 820 */ 374, 71, 449, 373, 426, 379, 375, 381, 130, 131, - /* 830 */ 164, 97, 382, 135, 58, 100, 374, 464, 465, 166, - /* 840 */ 167, 63, 469, 470, 374, 172, 173, 97, 341, 8, - /* 850 */ 9, 20, 272, 12, 13, 14, 15, 16, 412, 186, - /* 860 */ 22, 188, 416, 358, 357, 419, 420, 421, 422, 423, - /* 870 */ 424, 366, 426, 35, 333, 99, 20, 431, 102, 433, - /* 880 */ 375, 374, 333, 437, 438, 44, 398, 165, 215, 216, - /* 890 */ 4, 218, 219, 220, 221, 222, 223, 224, 225, 226, + /* 680 */ 231, 232, 233, 234, 235, 12, 13, 14, 138, 99, + /* 690 */ 369, 333, 102, 20, 272, 22, 333, 376, 342, 343, + /* 700 */ 178, 343, 166, 35, 252, 384, 33, 101, 35, 252, + /* 710 */ 420, 2, 355, 356, 342, 343, 360, 8, 9, 342, + /* 720 */ 343, 12, 13, 14, 15, 16, 111, 369, 178, 179, + /* 730 */ 180, 129, 59, 183, 168, 133, 446, 360, 342, 381, + /* 740 */ 72, 383, 8, 9, 381, 72, 12, 13, 14, 15, + /* 750 */ 16, 342, 343, 252, 204, 431, 378, 207, 434, 381, + /* 760 */ 210, 211, 212, 213, 214, 236, 130, 342, 343, 360, + /* 770 */ 97, 72, 414, 100, 342, 343, 418, 333, 342, 421, + /* 780 */ 422, 423, 424, 425, 426, 360, 428, 343, 392, 345, + /* 790 */ 394, 433, 360, 435, 2, 361, 350, 439, 440, 277, + /* 800 */ 8, 9, 252, 369, 12, 13, 14, 15, 16, 136, + /* 810 */ 137, 377, 366, 369, 168, 443, 444, 445, 460, 447, + /* 820 */ 448, 375, 450, 342, 343, 381, 111, 383, 392, 369, + /* 830 */ 394, 342, 343, 197, 198, 101, 376, 465, 466, 166, + /* 840 */ 167, 343, 470, 471, 384, 172, 173, 8, 9, 360, + /* 850 */ 72, 12, 13, 14, 15, 16, 0, 48, 414, 186, + /* 860 */ 22, 188, 418, 342, 343, 421, 422, 423, 424, 425, + /* 870 */ 426, 369, 428, 35, 131, 369, 343, 433, 135, 435, + /* 880 */ 37, 360, 376, 439, 440, 387, 384, 400, 215, 216, + /* 890 */ 384, 218, 219, 220, 221, 222, 223, 224, 225, 226, /* 900 */ 227, 228, 229, 230, 231, 232, 233, 234, 235, 12, - /* 910 */ 13, 441, 442, 443, 358, 445, 446, 20, 341, 22, - /* 920 */ 379, 333, 366, 403, 250, 251, 341, 333, 379, 111, - /* 930 */ 33, 375, 35, 445, 357, 97, 186, 449, 188, 8, - /* 940 */ 9, 4, 357, 12, 13, 14, 15, 16, 418, 367, - /* 950 */ 166, 374, 464, 465, 366, 58, 19, 469, 470, 374, - /* 960 */ 333, 239, 374, 418, 0, 215, 216, 379, 71, 381, - /* 970 */ 33, 249, 333, 379, 444, 8, 9, 374, 160, 12, - /* 980 */ 13, 14, 15, 16, 354, 48, 356, 341, 385, 444, - /* 990 */ 53, 333, 39, 44, 97, 58, 333, 100, 333, 168, - /* 1000 */ 412, 20, 333, 357, 416, 333, 379, 419, 420, 421, - /* 1010 */ 422, 423, 424, 333, 426, 333, 333, 2, 379, 431, - /* 1020 */ 374, 433, 166, 8, 9, 437, 438, 12, 13, 14, - /* 1030 */ 15, 16, 101, 136, 137, 0, 99, 379, 366, 102, - /* 1040 */ 429, 341, 379, 432, 379, 374, 374, 459, 379, 341, - /* 1050 */ 101, 379, 366, 381, 22, 418, 385, 357, 42, 379, - /* 1060 */ 44, 379, 379, 166, 167, 357, 2, 35, 382, 172, - /* 1070 */ 173, 341, 8, 9, 374, 44, 12, 13, 14, 15, - /* 1080 */ 16, 444, 374, 186, 412, 188, 376, 357, 416, 379, - /* 1090 */ 0, 419, 420, 421, 422, 423, 424, 376, 426, 376, - /* 1100 */ 379, 44, 379, 431, 374, 433, 44, 333, 359, 437, - /* 1110 */ 438, 362, 215, 216, 13, 218, 219, 220, 221, 222, + /* 910 */ 13, 333, 333, 45, 46, 342, 343, 20, 333, 22, + /* 920 */ 387, 343, 370, 345, 443, 444, 445, 0, 447, 448, + /* 930 */ 33, 333, 35, 360, 447, 97, 20, 450, 195, 196, + /* 940 */ 101, 343, 199, 4, 201, 333, 103, 369, 105, 106, + /* 950 */ 333, 108, 465, 466, 370, 21, 59, 470, 471, 381, + /* 960 */ 381, 383, 342, 343, 342, 343, 381, 369, 34, 72, + /* 970 */ 36, 378, 129, 370, 381, 378, 133, 168, 381, 381, + /* 980 */ 360, 383, 360, 420, 370, 129, 130, 131, 132, 133, + /* 990 */ 134, 135, 414, 381, 97, 400, 418, 100, 381, 421, + /* 1000 */ 422, 423, 424, 425, 426, 333, 428, 400, 42, 446, + /* 1010 */ 44, 433, 414, 435, 20, 343, 418, 439, 440, 421, + /* 1020 */ 422, 423, 424, 425, 426, 427, 428, 429, 430, 342, + /* 1030 */ 343, 8, 9, 136, 137, 12, 13, 14, 15, 16, + /* 1040 */ 361, 369, 447, 333, 370, 450, 14, 360, 369, 342, + /* 1050 */ 343, 44, 20, 381, 447, 383, 377, 450, 342, 343, + /* 1060 */ 465, 466, 22, 166, 167, 470, 471, 360, 165, 172, + /* 1070 */ 173, 104, 465, 466, 107, 35, 360, 470, 471, 250, + /* 1080 */ 251, 342, 343, 186, 168, 188, 414, 333, 342, 343, + /* 1090 */ 418, 381, 22, 421, 422, 423, 424, 425, 426, 360, + /* 1100 */ 428, 400, 333, 342, 343, 35, 360, 333, 101, 42, + /* 1110 */ 333, 44, 215, 216, 333, 218, 219, 220, 221, 222, /* 1120 */ 223, 224, 225, 226, 227, 228, 229, 230, 231, 232, - /* 1130 */ 233, 234, 235, 12, 13, 168, 35, 96, 48, 253, - /* 1140 */ 366, 20, 341, 22, 200, 341, 202, 112, 374, 168, - /* 1150 */ 42, 44, 44, 379, 33, 381, 35, 104, 357, 398, - /* 1160 */ 107, 357, 367, 101, 367, 130, 131, 132, 133, 134, - /* 1170 */ 135, 104, 37, 209, 107, 374, 8, 9, 374, 58, - /* 1180 */ 12, 13, 14, 15, 16, 104, 412, 341, 107, 58, - /* 1190 */ 416, 44, 71, 419, 420, 421, 422, 423, 424, 341, - /* 1200 */ 426, 341, 355, 357, 251, 431, 445, 433, 101, 168, - /* 1210 */ 449, 437, 438, 333, 35, 357, 44, 357, 97, 341, - /* 1220 */ 374, 100, 448, 0, 104, 464, 465, 107, 0, 44, - /* 1230 */ 469, 470, 374, 102, 374, 357, 333, 35, 103, 100, - /* 1240 */ 105, 106, 0, 108, 13, 22, 366, 112, 101, 110, - /* 1250 */ 22, 445, 374, 44, 374, 449, 334, 136, 137, 379, - /* 1260 */ 389, 381, 136, 137, 22, 473, 35, 13, 133, 366, - /* 1270 */ 462, 465, 64, 101, 44, 469, 470, 374, 44, 1, - /* 1280 */ 2, 44, 379, 343, 381, 456, 101, 166, 167, 35, - /* 1290 */ 366, 343, 412, 172, 173, 389, 416, 340, 35, 419, - /* 1300 */ 420, 421, 422, 423, 424, 274, 426, 186, 44, 188, - /* 1310 */ 101, 44, 378, 433, 44, 412, 44, 437, 438, 416, - /* 1320 */ 44, 44, 419, 420, 421, 422, 423, 424, 333, 426, - /* 1330 */ 389, 101, 44, 276, 71, 101, 215, 216, 101, 218, + /* 1130 */ 233, 234, 235, 12, 13, 381, 209, 97, 0, 333, + /* 1140 */ 333, 20, 239, 22, 472, 473, 333, 333, 447, 44, + /* 1150 */ 381, 450, 249, 333, 33, 381, 35, 343, 381, 345, + /* 1160 */ 166, 44, 381, 333, 333, 333, 465, 466, 111, 13, + /* 1170 */ 362, 470, 471, 365, 333, 104, 104, 35, 107, 107, + /* 1180 */ 59, 13, 44, 369, 343, 0, 345, 381, 381, 44, + /* 1190 */ 405, 35, 253, 72, 381, 381, 104, 383, 166, 107, + /* 1200 */ 200, 381, 202, 35, 443, 444, 445, 22, 447, 448, + /* 1210 */ 369, 381, 381, 381, 44, 44, 44, 160, 97, 0, + /* 1220 */ 44, 100, 381, 0, 383, 136, 137, 370, 414, 358, + /* 1230 */ 1, 2, 418, 44, 333, 421, 422, 423, 424, 425, + /* 1240 */ 426, 22, 428, 35, 343, 22, 101, 433, 44, 435, + /* 1250 */ 47, 44, 65, 439, 440, 414, 44, 136, 137, 418, + /* 1260 */ 44, 44, 421, 422, 423, 424, 425, 426, 391, 428, + /* 1270 */ 369, 101, 101, 101, 433, 44, 435, 101, 44, 334, + /* 1280 */ 439, 440, 381, 346, 383, 474, 463, 166, 167, 369, + /* 1290 */ 101, 457, 346, 172, 173, 391, 129, 130, 131, 132, + /* 1300 */ 133, 134, 135, 100, 341, 101, 380, 186, 101, 188, + /* 1310 */ 391, 44, 13, 101, 44, 414, 44, 101, 101, 418, + /* 1320 */ 0, 44, 421, 422, 423, 424, 425, 426, 343, 428, + /* 1330 */ 188, 449, 101, 44, 35, 101, 215, 216, 44, 218, /* 1340 */ 219, 220, 221, 222, 223, 224, 225, 226, 227, 228, - /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 13, 447, 466, - /* 1360 */ 439, 366, 450, 460, 461, 101, 254, 188, 101, 374, - /* 1370 */ 44, 101, 44, 101, 379, 414, 381, 101, 101, 35, - /* 1380 */ 44, 0, 48, 413, 184, 400, 386, 42, 20, 101, - /* 1390 */ 188, 389, 386, 384, 165, 341, 20, 341, 333, 386, - /* 1400 */ 98, 384, 384, 351, 95, 350, 94, 412, 349, 341, - /* 1410 */ 341, 416, 341, 341, 419, 420, 421, 422, 423, 424, - /* 1420 */ 20, 426, 335, 215, 335, 20, 431, 101, 433, 101, - /* 1430 */ 49, 366, 437, 438, 407, 347, 20, 101, 347, 374, - /* 1440 */ 20, 381, 342, 448, 379, 20, 381, 399, 342, 333, - /* 1450 */ 347, 347, 347, 52, 341, 344, 347, 347, 344, 335, - /* 1460 */ 366, 366, 366, 366, 366, 366, 341, 335, 366, 203, - /* 1470 */ 100, 379, 379, 192, 411, 409, 406, 412, 345, 405, - /* 1480 */ 407, 416, 366, 366, 419, 420, 421, 422, 423, 424, - /* 1490 */ 374, 426, 191, 366, 366, 379, 431, 381, 433, 12, - /* 1500 */ 13, 379, 437, 438, 366, 345, 190, 341, 389, 22, - /* 1510 */ 381, 261, 389, 448, 269, 379, 455, 260, 177, 404, - /* 1520 */ 33, 379, 35, 379, 333, 379, 271, 278, 412, 270, - /* 1530 */ 455, 394, 416, 457, 394, 419, 420, 421, 422, 423, - /* 1540 */ 424, 458, 426, 275, 255, 58, 468, 431, 273, 433, - /* 1550 */ 251, 474, 467, 437, 438, 374, 20, 366, 71, 414, - /* 1560 */ 341, 418, 342, 345, 448, 374, 345, 20, 392, 394, - /* 1570 */ 379, 379, 381, 455, 379, 379, 379, 379, 394, 170, - /* 1580 */ 379, 391, 100, 453, 97, 345, 362, 374, 454, 345, - /* 1590 */ 436, 379, 100, 452, 356, 370, 345, 341, 36, 395, - /* 1600 */ 401, 336, 408, 412, 335, 395, 360, 416, 360, 1, - /* 1610 */ 419, 420, 421, 422, 423, 424, 331, 426, 360, 346, - /* 1620 */ 0, 0, 431, 0, 433, 42, 0, 19, 437, 438, - /* 1630 */ 35, 208, 35, 35, 35, 208, 0, 35, 35, 448, - /* 1640 */ 208, 33, 0, 333, 208, 0, 35, 0, 0, 22, - /* 1650 */ 35, 195, 188, 186, 0, 0, 48, 0, 182, 181, - /* 1660 */ 0, 0, 54, 55, 56, 57, 58, 0, 0, 47, - /* 1670 */ 0, 42, 0, 186, 0, 188, 366, 155, 35, 0, - /* 1680 */ 155, 0, 0, 0, 374, 0, 0, 0, 0, 379, - /* 1690 */ 0, 381, 0, 0, 0, 0, 0, 0, 0, 0, - /* 1700 */ 0, 0, 215, 216, 0, 0, 0, 99, 333, 0, - /* 1710 */ 102, 42, 0, 0, 0, 228, 229, 230, 231, 232, - /* 1720 */ 233, 234, 412, 0, 0, 0, 416, 0, 0, 419, - /* 1730 */ 420, 421, 422, 423, 424, 0, 426, 22, 0, 139, - /* 1740 */ 0, 366, 134, 433, 22, 96, 96, 437, 438, 374, - /* 1750 */ 0, 22, 35, 0, 379, 0, 381, 0, 0, 39, - /* 1760 */ 58, 42, 44, 47, 0, 58, 58, 0, 14, 39, - /* 1770 */ 14, 0, 0, 333, 40, 39, 0, 169, 47, 47, - /* 1780 */ 0, 0, 174, 177, 0, 0, 65, 412, 39, 48, - /* 1790 */ 0, 416, 48, 0, 419, 420, 421, 422, 423, 424, - /* 1800 */ 35, 426, 194, 35, 35, 39, 366, 48, 433, 39, - /* 1810 */ 0, 48, 437, 438, 374, 0, 39, 35, 0, 379, - /* 1820 */ 0, 381, 0, 35, 333, 22, 0, 35, 22, 35, - /* 1830 */ 44, 35, 35, 35, 35, 109, 35, 35, 0, 35, - /* 1840 */ 107, 22, 35, 333, 44, 0, 22, 0, 35, 22, - /* 1850 */ 0, 35, 412, 0, 0, 50, 416, 366, 35, 419, - /* 1860 */ 420, 421, 422, 423, 424, 374, 426, 35, 22, 20, - /* 1870 */ 379, 35, 381, 433, 35, 100, 366, 0, 438, 101, - /* 1880 */ 22, 0, 193, 100, 374, 22, 35, 0, 0, 379, - /* 1890 */ 256, 381, 3, 256, 3, 256, 44, 0, 0, 189, - /* 1900 */ 44, 44, 35, 412, 95, 168, 44, 416, 100, 170, - /* 1910 */ 419, 420, 421, 422, 423, 424, 168, 426, 333, 168, - /* 1920 */ 175, 100, 412, 47, 96, 47, 416, 98, 44, 419, - /* 1930 */ 420, 421, 422, 423, 424, 101, 426, 333, 96, 44, - /* 1940 */ 35, 101, 101, 100, 100, 35, 35, 35, 100, 35, - /* 1950 */ 47, 366, 461, 44, 47, 0, 101, 0, 171, 374, - /* 1960 */ 100, 250, 101, 39, 379, 101, 381, 101, 47, 101, - /* 1970 */ 366, 100, 100, 463, 101, 371, 101, 0, 374, 100, - /* 1980 */ 100, 39, 100, 379, 110, 381, 169, 47, 44, 237, - /* 1990 */ 2, 98, 100, 22, 98, 101, 47, 412, 47, 100, - /* 2000 */ 100, 416, 101, 101, 419, 420, 421, 422, 423, 424, - /* 2010 */ 333, 426, 22, 100, 35, 100, 412, 100, 35, 101, - /* 2020 */ 416, 35, 100, 419, 420, 421, 422, 423, 424, 215, - /* 2030 */ 426, 111, 101, 100, 35, 101, 333, 100, 35, 101, - /* 2040 */ 101, 100, 35, 366, 101, 100, 100, 217, 123, 123, - /* 2050 */ 123, 374, 123, 22, 100, 44, 379, 472, 381, 112, - /* 2060 */ 100, 35, 100, 22, 35, 65, 64, 35, 35, 366, - /* 2070 */ 35, 35, 71, 35, 371, 35, 22, 374, 35, 35, - /* 2080 */ 35, 35, 379, 44, 381, 93, 35, 35, 35, 412, - /* 2090 */ 35, 35, 71, 416, 35, 35, 419, 420, 421, 422, - /* 2100 */ 423, 424, 35, 426, 35, 428, 35, 333, 22, 35, - /* 2110 */ 0, 35, 48, 0, 39, 412, 35, 39, 48, 416, - /* 2120 */ 0, 48, 419, 420, 421, 422, 423, 424, 333, 426, - /* 2130 */ 35, 39, 0, 48, 35, 39, 0, 35, 35, 0, - /* 2140 */ 366, 22, 21, 20, 22, 371, 22, 21, 374, 475, - /* 2150 */ 475, 475, 475, 379, 475, 381, 475, 475, 333, 475, - /* 2160 */ 475, 366, 475, 475, 475, 475, 475, 475, 475, 374, - /* 2170 */ 475, 475, 475, 475, 379, 475, 381, 475, 475, 475, - /* 2180 */ 475, 475, 475, 475, 475, 475, 412, 475, 475, 475, - /* 2190 */ 416, 366, 475, 419, 420, 421, 422, 423, 424, 374, - /* 2200 */ 426, 475, 475, 475, 379, 475, 381, 412, 475, 475, - /* 2210 */ 475, 416, 475, 475, 419, 420, 421, 422, 423, 424, - /* 2220 */ 475, 426, 475, 333, 475, 475, 475, 475, 475, 475, - /* 2230 */ 475, 475, 475, 475, 475, 475, 475, 412, 475, 475, - /* 2240 */ 475, 416, 475, 475, 419, 420, 421, 422, 423, 424, - /* 2250 */ 475, 426, 333, 475, 475, 475, 366, 475, 475, 475, - /* 2260 */ 475, 475, 475, 475, 374, 475, 475, 475, 475, 379, - /* 2270 */ 475, 381, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2280 */ 475, 475, 475, 475, 475, 366, 333, 475, 475, 475, - /* 2290 */ 475, 475, 475, 374, 475, 475, 475, 475, 379, 475, - /* 2300 */ 381, 475, 412, 475, 475, 475, 416, 475, 475, 419, - /* 2310 */ 420, 421, 422, 423, 424, 475, 426, 475, 475, 366, - /* 2320 */ 475, 475, 475, 475, 475, 475, 475, 374, 475, 475, - /* 2330 */ 475, 412, 379, 475, 381, 416, 475, 475, 419, 420, - /* 2340 */ 421, 422, 423, 424, 475, 426, 475, 475, 475, 475, - /* 2350 */ 333, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2360 */ 475, 475, 475, 475, 475, 412, 475, 475, 475, 416, - /* 2370 */ 333, 475, 419, 420, 421, 422, 423, 424, 475, 426, - /* 2380 */ 475, 475, 475, 366, 475, 475, 475, 475, 475, 475, - /* 2390 */ 475, 374, 475, 475, 475, 475, 379, 475, 381, 475, - /* 2400 */ 475, 475, 475, 366, 475, 475, 475, 475, 475, 475, - /* 2410 */ 475, 374, 475, 475, 475, 475, 379, 475, 381, 475, - /* 2420 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 412, - /* 2430 */ 475, 333, 475, 416, 475, 475, 419, 420, 421, 422, - /* 2440 */ 423, 424, 475, 426, 475, 475, 475, 475, 475, 412, - /* 2450 */ 475, 475, 475, 416, 475, 475, 419, 420, 421, 422, - /* 2460 */ 423, 424, 475, 426, 366, 475, 475, 475, 475, 475, - /* 2470 */ 475, 475, 374, 475, 475, 475, 475, 379, 475, 381, - /* 2480 */ 475, 475, 333, 475, 475, 475, 475, 475, 475, 475, - /* 2490 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2500 */ 475, 475, 475, 333, 475, 475, 475, 475, 475, 475, - /* 2510 */ 412, 475, 475, 475, 416, 366, 475, 419, 420, 421, - /* 2520 */ 422, 423, 424, 374, 426, 475, 475, 475, 379, 475, - /* 2530 */ 381, 475, 475, 333, 475, 475, 366, 475, 475, 475, - /* 2540 */ 475, 475, 475, 475, 374, 475, 475, 475, 475, 379, - /* 2550 */ 475, 381, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2560 */ 475, 412, 475, 475, 475, 416, 366, 475, 419, 420, - /* 2570 */ 421, 422, 423, 424, 374, 426, 475, 475, 475, 379, - /* 2580 */ 475, 381, 412, 475, 475, 475, 416, 475, 475, 419, - /* 2590 */ 420, 421, 422, 423, 424, 475, 426, 475, 333, 475, - /* 2600 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2610 */ 475, 475, 412, 475, 475, 475, 416, 475, 475, 419, - /* 2620 */ 420, 421, 422, 423, 424, 475, 426, 333, 475, 475, - /* 2630 */ 475, 366, 475, 475, 475, 475, 475, 475, 475, 374, - /* 2640 */ 475, 475, 475, 475, 379, 475, 381, 475, 475, 475, - /* 2650 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2660 */ 366, 333, 475, 475, 475, 475, 475, 475, 374, 475, - /* 2670 */ 475, 475, 475, 379, 475, 381, 475, 412, 475, 475, - /* 2680 */ 475, 416, 475, 475, 419, 420, 421, 422, 423, 424, - /* 2690 */ 475, 426, 475, 475, 366, 475, 475, 475, 475, 475, - /* 2700 */ 475, 475, 374, 475, 475, 475, 412, 379, 475, 381, - /* 2710 */ 416, 475, 475, 419, 420, 421, 422, 423, 424, 475, - /* 2720 */ 426, 475, 475, 475, 475, 333, 475, 475, 475, 475, - /* 2730 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2740 */ 412, 475, 475, 475, 416, 333, 475, 419, 420, 421, - /* 2750 */ 422, 423, 424, 475, 426, 475, 475, 475, 366, 475, - /* 2760 */ 475, 475, 475, 475, 475, 475, 374, 475, 475, 475, - /* 2770 */ 475, 379, 475, 381, 475, 475, 475, 475, 366, 475, - /* 2780 */ 475, 475, 475, 475, 475, 475, 374, 475, 475, 475, - /* 2790 */ 475, 379, 475, 381, 475, 475, 475, 475, 475, 475, - /* 2800 */ 475, 475, 475, 475, 412, 475, 333, 475, 416, 475, - /* 2810 */ 475, 419, 420, 421, 422, 423, 424, 475, 426, 475, - /* 2820 */ 475, 475, 475, 475, 412, 475, 475, 475, 416, 475, - /* 2830 */ 475, 419, 420, 421, 422, 423, 424, 475, 426, 366, - /* 2840 */ 475, 475, 475, 475, 475, 475, 475, 374, 475, 475, - /* 2850 */ 475, 475, 379, 475, 381, 475, 475, 333, 475, 475, - /* 2860 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 2870 */ 475, 475, 475, 475, 475, 475, 475, 475, 333, 475, - /* 2880 */ 475, 475, 475, 475, 475, 412, 475, 475, 475, 416, - /* 2890 */ 366, 475, 419, 420, 421, 422, 423, 424, 374, 426, - /* 2900 */ 475, 475, 475, 379, 475, 381, 475, 475, 333, 475, - /* 2910 */ 475, 366, 475, 475, 475, 475, 475, 475, 475, 374, - /* 2920 */ 475, 475, 475, 475, 379, 475, 381, 475, 475, 475, - /* 2930 */ 475, 475, 475, 475, 475, 475, 412, 475, 475, 475, - /* 2940 */ 416, 366, 475, 419, 420, 421, 422, 423, 424, 374, - /* 2950 */ 426, 475, 475, 475, 379, 475, 381, 412, 475, 475, - /* 2960 */ 475, 416, 475, 475, 419, 420, 421, 422, 423, 424, - /* 2970 */ 475, 426, 475, 333, 475, 475, 475, 475, 475, 475, - /* 2980 */ 475, 475, 475, 475, 475, 475, 475, 412, 475, 475, - /* 2990 */ 475, 416, 475, 475, 419, 420, 421, 422, 423, 424, - /* 3000 */ 475, 426, 333, 475, 475, 475, 366, 475, 475, 475, - /* 3010 */ 475, 475, 475, 475, 374, 475, 475, 475, 475, 379, - /* 3020 */ 475, 381, 475, 475, 475, 475, 475, 475, 475, 475, - /* 3030 */ 475, 475, 475, 475, 475, 366, 475, 475, 475, 475, - /* 3040 */ 475, 475, 475, 374, 475, 475, 475, 475, 379, 475, - /* 3050 */ 381, 475, 412, 475, 475, 475, 416, 475, 475, 419, - /* 3060 */ 420, 421, 422, 423, 424, 475, 426, 475, 475, 475, - /* 3070 */ 475, 475, 475, 475, 475, 475, 475, 475, 475, 475, - /* 3080 */ 475, 412, 475, 475, 475, 416, 475, 475, 419, 420, - /* 3090 */ 421, 422, 423, 424, 475, 426, + /* 1350 */ 229, 230, 231, 232, 233, 234, 235, 467, 441, 12, + /* 1360 */ 13, 13, 451, 462, 254, 416, 49, 333, 101, 22, + /* 1370 */ 50, 101, 415, 101, 184, 402, 42, 343, 101, 345, + /* 1380 */ 33, 276, 35, 35, 388, 20, 391, 388, 386, 165, + /* 1390 */ 101, 274, 20, 342, 342, 101, 188, 333, 388, 386, + /* 1400 */ 386, 98, 215, 369, 354, 96, 59, 343, 353, 345, + /* 1410 */ 342, 95, 352, 20, 335, 381, 342, 383, 342, 72, + /* 1420 */ 342, 48, 339, 335, 339, 20, 409, 350, 20, 383, + /* 1430 */ 350, 20, 20, 369, 344, 344, 401, 350, 53, 342, + /* 1440 */ 342, 347, 347, 350, 97, 381, 350, 383, 414, 350, + /* 1450 */ 350, 335, 418, 369, 335, 421, 422, 423, 424, 425, + /* 1460 */ 426, 203, 428, 381, 100, 333, 413, 433, 409, 435, + /* 1470 */ 369, 369, 369, 439, 440, 343, 369, 369, 414, 348, + /* 1480 */ 369, 369, 418, 369, 369, 421, 422, 423, 424, 425, + /* 1490 */ 426, 369, 428, 192, 381, 191, 348, 433, 190, 435, + /* 1500 */ 342, 369, 261, 439, 440, 381, 383, 381, 260, 381, + /* 1510 */ 456, 391, 408, 381, 456, 383, 407, 411, 391, 269, + /* 1520 */ 381, 396, 381, 177, 406, 396, 271, 270, 459, 255, + /* 1530 */ 455, 458, 278, 186, 275, 188, 475, 456, 273, 251, + /* 1540 */ 454, 343, 20, 342, 344, 348, 414, 348, 20, 394, + /* 1550 */ 418, 416, 170, 421, 422, 423, 424, 425, 426, 396, + /* 1560 */ 428, 381, 215, 216, 420, 433, 393, 435, 348, 381, + /* 1570 */ 396, 439, 440, 333, 381, 228, 229, 230, 231, 232, + /* 1580 */ 233, 234, 343, 343, 348, 100, 453, 381, 438, 100, + /* 1590 */ 381, 381, 365, 333, 373, 381, 359, 348, 342, 469, + /* 1600 */ 36, 335, 468, 343, 336, 403, 410, 363, 397, 369, + /* 1610 */ 397, 349, 331, 0, 363, 363, 0, 0, 42, 0, + /* 1620 */ 35, 381, 208, 383, 35, 35, 35, 208, 0, 369, + /* 1630 */ 35, 35, 208, 0, 208, 0, 35, 0, 0, 35, + /* 1640 */ 22, 381, 195, 383, 188, 186, 0, 0, 0, 182, + /* 1650 */ 181, 0, 0, 47, 414, 0, 0, 0, 418, 0, + /* 1660 */ 0, 421, 422, 423, 424, 425, 426, 42, 428, 0, + /* 1670 */ 0, 0, 0, 433, 414, 435, 155, 333, 418, 439, + /* 1680 */ 440, 421, 422, 423, 424, 425, 426, 343, 428, 0, + /* 1690 */ 0, 0, 155, 433, 0, 435, 35, 0, 0, 439, + /* 1700 */ 440, 333, 0, 0, 0, 0, 0, 0, 0, 0, + /* 1710 */ 0, 343, 0, 369, 0, 0, 0, 42, 0, 0, + /* 1720 */ 0, 0, 0, 0, 0, 381, 0, 383, 0, 0, + /* 1730 */ 0, 139, 48, 22, 48, 22, 0, 369, 59, 22, + /* 1740 */ 0, 59, 0, 0, 59, 0, 47, 47, 14, 381, + /* 1750 */ 177, 383, 0, 0, 0, 0, 0, 0, 414, 0, + /* 1760 */ 35, 66, 418, 0, 0, 421, 422, 423, 424, 425, + /* 1770 */ 426, 39, 428, 42, 47, 40, 39, 44, 333, 435, + /* 1780 */ 39, 35, 414, 439, 440, 0, 418, 49, 343, 421, + /* 1790 */ 422, 423, 424, 425, 426, 35, 428, 39, 49, 0, + /* 1800 */ 39, 333, 35, 435, 39, 0, 35, 439, 440, 0, + /* 1810 */ 0, 343, 39, 0, 369, 49, 0, 49, 35, 22, + /* 1820 */ 0, 35, 35, 35, 333, 35, 381, 35, 383, 35, + /* 1830 */ 35, 35, 0, 44, 343, 44, 35, 369, 35, 22, + /* 1840 */ 22, 0, 0, 22, 51, 22, 35, 333, 109, 381, + /* 1850 */ 0, 383, 0, 0, 107, 22, 20, 343, 0, 414, + /* 1860 */ 369, 35, 35, 418, 35, 374, 421, 422, 423, 424, + /* 1870 */ 425, 426, 381, 428, 383, 35, 101, 100, 35, 100, + /* 1880 */ 435, 193, 414, 369, 439, 440, 418, 22, 374, 421, + /* 1890 */ 422, 423, 424, 425, 426, 381, 428, 383, 35, 0, + /* 1900 */ 22, 0, 0, 435, 44, 414, 3, 100, 440, 418, + /* 1910 */ 333, 96, 421, 422, 423, 424, 425, 426, 170, 428, + /* 1920 */ 343, 256, 48, 101, 100, 48, 175, 47, 414, 98, + /* 1930 */ 168, 101, 418, 189, 44, 421, 422, 423, 424, 425, + /* 1940 */ 426, 44, 428, 47, 168, 101, 369, 100, 168, 100, + /* 1950 */ 44, 100, 44, 3, 44, 101, 100, 256, 381, 35, + /* 1960 */ 383, 35, 101, 101, 35, 35, 35, 35, 47, 101, + /* 1970 */ 101, 44, 47, 0, 0, 333, 0, 0, 39, 100, + /* 1980 */ 47, 100, 0, 101, 100, 343, 101, 100, 100, 100, + /* 1990 */ 171, 414, 39, 47, 169, 418, 333, 2, 421, 422, + /* 2000 */ 423, 424, 425, 426, 110, 428, 343, 44, 22, 333, + /* 2010 */ 98, 369, 98, 215, 47, 237, 100, 47, 101, 343, + /* 2020 */ 100, 22, 100, 381, 250, 383, 111, 35, 256, 101, + /* 2030 */ 22, 217, 369, 35, 44, 101, 100, 374, 100, 35, + /* 2040 */ 100, 464, 101, 101, 381, 369, 383, 100, 35, 101, + /* 2050 */ 101, 100, 100, 35, 35, 101, 414, 381, 100, 383, + /* 2060 */ 418, 101, 100, 421, 422, 423, 424, 425, 426, 100, + /* 2070 */ 428, 100, 35, 35, 65, 100, 66, 414, 122, 122, + /* 2080 */ 35, 418, 35, 333, 421, 422, 423, 424, 425, 426, + /* 2090 */ 414, 428, 35, 343, 418, 35, 35, 421, 422, 423, + /* 2100 */ 424, 425, 426, 333, 428, 122, 430, 122, 35, 35, + /* 2110 */ 35, 35, 72, 343, 44, 473, 94, 35, 35, 369, + /* 2120 */ 35, 22, 35, 35, 374, 35, 72, 35, 35, 35, + /* 2130 */ 35, 381, 35, 383, 22, 35, 0, 35, 49, 369, + /* 2140 */ 39, 0, 35, 39, 374, 49, 0, 49, 35, 39, + /* 2150 */ 0, 381, 35, 383, 49, 39, 0, 35, 35, 0, + /* 2160 */ 22, 21, 476, 21, 414, 22, 22, 20, 418, 476, + /* 2170 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2180 */ 476, 476, 476, 476, 414, 476, 1, 476, 418, 476, + /* 2190 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2200 */ 333, 476, 476, 476, 19, 476, 476, 476, 476, 476, + /* 2210 */ 343, 476, 476, 476, 476, 476, 333, 476, 33, 476, + /* 2220 */ 476, 476, 476, 476, 476, 476, 343, 476, 476, 476, + /* 2230 */ 476, 476, 476, 476, 49, 476, 369, 476, 476, 476, + /* 2240 */ 55, 56, 57, 58, 59, 476, 333, 476, 381, 476, + /* 2250 */ 383, 476, 369, 476, 476, 476, 343, 476, 476, 476, + /* 2260 */ 476, 476, 476, 476, 381, 476, 383, 476, 476, 476, + /* 2270 */ 476, 476, 476, 476, 476, 476, 333, 476, 476, 476, + /* 2280 */ 476, 414, 369, 476, 99, 418, 343, 102, 421, 422, + /* 2290 */ 423, 424, 425, 426, 381, 428, 383, 414, 476, 476, + /* 2300 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2310 */ 476, 428, 369, 476, 476, 476, 476, 476, 476, 134, + /* 2320 */ 476, 476, 476, 476, 381, 476, 383, 414, 476, 476, + /* 2330 */ 476, 418, 476, 476, 421, 422, 423, 424, 425, 426, + /* 2340 */ 333, 428, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2350 */ 343, 476, 476, 333, 169, 476, 476, 414, 476, 174, + /* 2360 */ 476, 418, 476, 343, 421, 422, 423, 424, 425, 426, + /* 2370 */ 476, 428, 476, 476, 476, 476, 369, 476, 476, 194, + /* 2380 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2390 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2400 */ 476, 381, 476, 383, 333, 476, 476, 476, 476, 476, + /* 2410 */ 476, 476, 476, 476, 343, 476, 476, 333, 476, 476, + /* 2420 */ 476, 414, 476, 476, 476, 418, 476, 343, 421, 422, + /* 2430 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2440 */ 369, 421, 422, 423, 424, 425, 426, 476, 428, 476, + /* 2450 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2460 */ 476, 476, 476, 476, 476, 381, 476, 383, 476, 476, + /* 2470 */ 476, 476, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2480 */ 476, 476, 476, 476, 476, 414, 476, 476, 343, 418, + /* 2490 */ 476, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2500 */ 476, 476, 418, 476, 476, 421, 422, 423, 424, 425, + /* 2510 */ 426, 476, 428, 476, 369, 476, 476, 476, 476, 476, + /* 2520 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 333, + /* 2530 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 343, + /* 2540 */ 476, 476, 476, 476, 333, 476, 476, 476, 476, 476, + /* 2550 */ 476, 476, 476, 476, 343, 476, 476, 476, 476, 414, + /* 2560 */ 476, 476, 476, 418, 476, 369, 421, 422, 423, 424, + /* 2570 */ 425, 426, 476, 428, 476, 476, 476, 381, 476, 383, + /* 2580 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2590 */ 333, 476, 381, 476, 383, 476, 476, 476, 476, 476, + /* 2600 */ 343, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2610 */ 414, 476, 476, 476, 418, 476, 476, 421, 422, 423, + /* 2620 */ 424, 425, 426, 476, 428, 414, 369, 476, 476, 418, + /* 2630 */ 476, 476, 421, 422, 423, 424, 425, 426, 381, 428, + /* 2640 */ 383, 333, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2650 */ 476, 343, 476, 476, 476, 476, 476, 476, 333, 476, + /* 2660 */ 476, 476, 476, 476, 476, 476, 476, 476, 343, 476, + /* 2670 */ 476, 414, 476, 476, 476, 418, 476, 369, 421, 422, + /* 2680 */ 423, 424, 425, 426, 476, 428, 476, 333, 476, 381, + /* 2690 */ 476, 383, 476, 476, 369, 476, 476, 343, 476, 476, + /* 2700 */ 476, 476, 476, 476, 476, 476, 381, 476, 383, 476, + /* 2710 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2720 */ 476, 476, 414, 369, 476, 476, 418, 476, 476, 421, + /* 2730 */ 422, 423, 424, 425, 426, 381, 428, 383, 476, 414, + /* 2740 */ 476, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2750 */ 425, 426, 476, 428, 476, 333, 476, 476, 476, 476, + /* 2760 */ 476, 476, 476, 476, 476, 343, 476, 476, 414, 476, + /* 2770 */ 476, 476, 418, 476, 333, 421, 422, 423, 424, 425, + /* 2780 */ 426, 476, 428, 476, 343, 476, 476, 333, 476, 476, + /* 2790 */ 476, 369, 476, 476, 476, 476, 476, 343, 476, 476, + /* 2800 */ 476, 476, 476, 381, 476, 383, 476, 476, 476, 476, + /* 2810 */ 369, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2820 */ 476, 476, 381, 369, 383, 476, 476, 476, 476, 476, + /* 2830 */ 476, 476, 476, 476, 476, 381, 414, 383, 476, 476, + /* 2840 */ 418, 476, 476, 421, 422, 423, 424, 425, 426, 476, + /* 2850 */ 428, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 2860 */ 333, 476, 421, 422, 423, 424, 425, 426, 414, 428, + /* 2870 */ 343, 476, 418, 333, 476, 421, 422, 423, 424, 425, + /* 2880 */ 426, 476, 428, 343, 476, 476, 476, 476, 333, 476, + /* 2890 */ 476, 476, 476, 476, 476, 476, 369, 476, 343, 476, + /* 2900 */ 476, 476, 476, 476, 476, 476, 476, 476, 381, 369, + /* 2910 */ 383, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 2920 */ 476, 381, 476, 383, 369, 476, 476, 476, 476, 476, + /* 2930 */ 476, 476, 476, 476, 333, 476, 381, 476, 383, 476, + /* 2940 */ 476, 414, 476, 476, 343, 418, 476, 476, 421, 422, + /* 2950 */ 423, 424, 425, 426, 414, 428, 476, 476, 418, 476, + /* 2960 */ 476, 421, 422, 423, 424, 425, 426, 476, 428, 414, + /* 2970 */ 369, 476, 476, 418, 476, 476, 421, 422, 423, 424, + /* 2980 */ 425, 426, 381, 428, 383, 476, 476, 476, 476, 476, + /* 2990 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3000 */ 476, 476, 476, 476, 476, 476, 476, 476, 476, 476, + /* 3010 */ 476, 476, 476, 476, 476, 414, 476, 476, 476, 418, + /* 3020 */ 476, 476, 421, 422, 423, 424, 425, 426, 476, 428, }; -#define YY_SHIFT_COUNT (760) +#define YY_SHIFT_COUNT (761) #define YY_SHIFT_MIN (0) -#define YY_SHIFT_MAX (2139) +#define YY_SHIFT_MAX (2185) static const unsigned short int yy_shift_ofst[] = { /* 0 */ 550, 0, 224, 0, 449, 449, 449, 449, 449, 449, - /* 10 */ 449, 449, 449, 449, 449, 673, 897, 897, 1121, 897, + /* 10 */ 449, 449, 449, 449, 449, 449, 673, 897, 897, 1121, /* 20 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, /* 30 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, - /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 149, 170, - /* 50 */ 447, 79, 251, 65, 167, 65, 79, 79, 1487, 1487, - /* 60 */ 1487, 65, 1487, 1487, 529, 65, 109, 328, 137, 137, - /* 70 */ 328, 707, 707, 475, 257, 87, 87, 137, 137, 137, - /* 80 */ 137, 137, 137, 137, 174, 137, 137, 176, 109, 137, - /* 90 */ 137, 279, 137, 109, 137, 174, 137, 174, 109, 137, - /* 100 */ 137, 109, 137, 109, 109, 109, 137, 378, 223, 187, - /* 110 */ 187, 406, 114, 750, 750, 750, 750, 750, 750, 750, - /* 120 */ 750, 750, 750, 750, 750, 750, 750, 750, 750, 750, - /* 130 */ 750, 750, 1135, 165, 475, 257, 63, 39, 39, 39, - /* 140 */ 778, 464, 464, 63, 448, 448, 448, 176, 497, 389, - /* 150 */ 109, 595, 109, 595, 595, 634, 687, 34, 34, 34, - /* 160 */ 34, 34, 34, 34, 34, 1608, 572, 569, 197, 15, - /* 170 */ 260, 288, 247, 543, 784, 690, 831, 693, 519, 981, - /* 180 */ 674, 953, 580, 674, 1016, 886, 856, 1112, 1334, 1200, - /* 190 */ 1345, 1368, 1345, 1229, 1376, 1376, 1345, 1229, 1229, 1302, - /* 200 */ 1309, 1376, 1312, 1376, 1376, 1376, 1400, 1400, 1405, 176, - /* 210 */ 1416, 176, 1420, 1425, 176, 1420, 176, 176, 176, 1376, - /* 220 */ 176, 1401, 1401, 1400, 109, 109, 109, 109, 109, 109, - /* 230 */ 109, 109, 109, 109, 109, 1376, 1400, 595, 595, 595, - /* 240 */ 1266, 1370, 1405, 378, 1281, 1301, 1416, 378, 1316, 1376, - /* 250 */ 1368, 1368, 595, 1250, 1257, 595, 1250, 1257, 595, 595, - /* 260 */ 109, 1245, 1341, 1250, 1255, 1259, 1289, 1112, 1249, 1268, - /* 270 */ 1275, 1299, 448, 1536, 1376, 1420, 378, 378, 1547, 1257, - /* 280 */ 595, 595, 595, 595, 595, 1257, 595, 1409, 378, 634, - /* 290 */ 378, 448, 1482, 1492, 595, 687, 1376, 378, 1562, 1400, - /* 300 */ 3096, 3096, 3096, 3096, 3096, 3096, 3096, 3096, 3096, 36, - /* 310 */ 454, 689, 937, 162, 841, 931, 592, 1015, 1064, 967, - /* 320 */ 1035, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, 1168, - /* 330 */ 472, 73, 502, 502, 666, 408, 499, 776, 734, 838, - /* 340 */ 377, 353, 698, 698, 604, 282, 722, 604, 604, 604, - /* 350 */ 1090, 964, 949, 1032, 1108, 818, 345, 1053, 1067, 1081, - /* 360 */ 1120, 1101, 1231, 1223, 1228, 1242, 944, 1062, 1107, 1131, - /* 370 */ 1147, 1172, 1185, 1126, 1031, 1057, 1041, 1209, 1230, 1234, - /* 380 */ 1237, 1264, 1267, 1278, 1270, 1179, 1202, 1208, 1272, 735, - /* 390 */ 1276, 1277, 1288, 1326, 1328, 1336, 1139, 1254, 1344, 1263, - /* 400 */ 1381, 1620, 1621, 1623, 1583, 1626, 1595, 1423, 1597, 1598, - /* 410 */ 1599, 1427, 1636, 1602, 1603, 1432, 1642, 1436, 1645, 1611, - /* 420 */ 1647, 1627, 1648, 1615, 1456, 1464, 1467, 1654, 1655, 1657, - /* 430 */ 1476, 1478, 1660, 1661, 1622, 1667, 1668, 1670, 1629, 1672, - /* 440 */ 1674, 1685, 1686, 1687, 1688, 1690, 1692, 1522, 1643, 1679, - /* 450 */ 1525, 1681, 1682, 1683, 1693, 1694, 1695, 1696, 1697, 1698, - /* 460 */ 1699, 1700, 1701, 1704, 1705, 1706, 1709, 1669, 1712, 1713, - /* 470 */ 1714, 1723, 1724, 1725, 1715, 1727, 1728, 1735, 1600, 1738, - /* 480 */ 1740, 1722, 1649, 1729, 1650, 1750, 1702, 1717, 1753, 1707, - /* 490 */ 1755, 1708, 1757, 1758, 1719, 1720, 1718, 1716, 1754, 1731, - /* 500 */ 1756, 1732, 1764, 1734, 1730, 1767, 1771, 1772, 1736, 1606, - /* 510 */ 1776, 1780, 1781, 1721, 1784, 1785, 1765, 1741, 1749, 1790, - /* 520 */ 1768, 1744, 1766, 1793, 1769, 1759, 1770, 1810, 1782, 1763, - /* 530 */ 1777, 1815, 1818, 1820, 1822, 1726, 1733, 1788, 1803, 1826, - /* 540 */ 1792, 1794, 1796, 1797, 1798, 1799, 1801, 1786, 1800, 1802, - /* 550 */ 1804, 1806, 1807, 1838, 1819, 1845, 1824, 1805, 1847, 1827, - /* 560 */ 1813, 1850, 1816, 1853, 1823, 1854, 1846, 1849, 1832, 1836, - /* 570 */ 1839, 1778, 1775, 1877, 1737, 1783, 1689, 1851, 1858, 1881, - /* 580 */ 1710, 1863, 1748, 1739, 1887, 1888, 1751, 1745, 1889, 1852, - /* 590 */ 1634, 1808, 1834, 1821, 1828, 1829, 1842, 1809, 1840, 1856, - /* 600 */ 1857, 1841, 1843, 1844, 1848, 1855, 1862, 1876, 1878, 1860, - /* 610 */ 1884, 1637, 1861, 1864, 1891, 1895, 1639, 1867, 1905, 1910, - /* 620 */ 1911, 1912, 1914, 1866, 1868, 1903, 1711, 1909, 1907, 1897, - /* 630 */ 1898, 1955, 1957, 1871, 1924, 1716, 1921, 1872, 1873, 1875, - /* 640 */ 1879, 1880, 1787, 1882, 1977, 1942, 1817, 1892, 1874, 1716, - /* 650 */ 1940, 1944, 1893, 1752, 1896, 1988, 1971, 1814, 1899, 1894, - /* 660 */ 1900, 1901, 1913, 1902, 1949, 1915, 1917, 1951, 1918, 1990, - /* 670 */ 1830, 1922, 1920, 1931, 1979, 1983, 1933, 1934, 1986, 1937, - /* 680 */ 1938, 1999, 1941, 1939, 2003, 1945, 1943, 2007, 1946, 1925, - /* 690 */ 1926, 1927, 1929, 2031, 1947, 1954, 2011, 1960, 2026, 1962, - /* 700 */ 2011, 2011, 2041, 2000, 2002, 2029, 2032, 2033, 2035, 2036, - /* 710 */ 2038, 2040, 2043, 2044, 2045, 2001, 1992, 2039, 2046, 2051, - /* 720 */ 2052, 2054, 2053, 2055, 2056, 2021, 1786, 2059, 1800, 2060, - /* 730 */ 2067, 2069, 2071, 2086, 2074, 2110, 2076, 2064, 2075, 2113, - /* 740 */ 2081, 2070, 2078, 2120, 2095, 2073, 2092, 2132, 2099, 2085, - /* 750 */ 2096, 2136, 2102, 2103, 2139, 2119, 2121, 2122, 2124, 2126, - /* 760 */ 2123, + /* 40 */ 897, 897, 897, 897, 897, 897, 897, 897, 897, 897, + /* 50 */ 897, 452, 457, 142, 145, 147, 30, 107, 30, 145, + /* 60 */ 145, 1347, 30, 1347, 1347, 501, 30, 56, 447, 344, + /* 70 */ 344, 447, 235, 235, 18, 286, 150, 150, 344, 344, + /* 80 */ 344, 344, 344, 344, 344, 369, 344, 344, 412, 56, + /* 90 */ 344, 344, 484, 344, 56, 344, 369, 344, 369, 56, + /* 100 */ 344, 344, 56, 344, 56, 56, 56, 344, 538, 223, + /* 110 */ 189, 189, 333, 114, 443, 443, 443, 443, 443, 443, + /* 120 */ 443, 443, 443, 443, 443, 443, 443, 443, 443, 443, + /* 130 */ 443, 443, 443, 843, 325, 18, 286, 392, 1, 1, + /* 140 */ 1, 539, 273, 273, 392, 627, 627, 627, 412, 615, + /* 150 */ 529, 56, 699, 56, 699, 699, 715, 778, 35, 35, + /* 160 */ 35, 35, 35, 35, 35, 35, 2185, 217, 288, 483, + /* 170 */ 522, 260, 313, 243, 536, 1032, 490, 646, 868, 616, + /* 180 */ 602, 916, 829, 60, 422, 829, 966, 939, 994, 1110, + /* 190 */ 1317, 1190, 1334, 1365, 1334, 1224, 1372, 1372, 1334, 1224, + /* 200 */ 1224, 1303, 1309, 1372, 1316, 1372, 1372, 1372, 1393, 1373, + /* 210 */ 1393, 1373, 1405, 412, 1408, 412, 1411, 1412, 412, 1411, + /* 220 */ 412, 412, 412, 1372, 412, 1385, 1385, 1393, 56, 56, + /* 230 */ 56, 56, 56, 56, 56, 56, 56, 56, 56, 1372, + /* 240 */ 1393, 699, 699, 699, 1258, 1364, 1405, 538, 1301, 1304, + /* 250 */ 1408, 538, 1308, 1372, 1365, 1365, 699, 1241, 1248, 699, + /* 260 */ 1241, 1248, 699, 699, 56, 1250, 1346, 1241, 1255, 1257, + /* 270 */ 1274, 1110, 1254, 1259, 1265, 1288, 627, 1522, 1372, 1411, + /* 280 */ 538, 538, 1528, 1248, 699, 699, 699, 699, 699, 1248, + /* 290 */ 699, 1382, 538, 715, 538, 627, 1485, 1489, 699, 778, + /* 300 */ 1372, 538, 1564, 1393, 3030, 3030, 3030, 3030, 3030, 3030, + /* 310 */ 3030, 3030, 3030, 34, 430, 15, 590, 734, 16, 839, + /* 320 */ 79, 709, 792, 566, 856, 1023, 1023, 1023, 1023, 1023, + /* 330 */ 1023, 1023, 1023, 1023, 1167, 743, 253, 253, 402, 57, + /* 340 */ 441, 414, 838, 1040, 934, 636, 64, 64, 638, 606, + /* 350 */ 903, 638, 638, 638, 417, 927, 1007, 1070, 1067, 1057, + /* 360 */ 1138, 967, 1071, 1072, 1092, 1156, 1168, 1185, 1219, 1223, + /* 370 */ 1000, 1145, 1170, 431, 1171, 1172, 1176, 1089, 1117, 1105, + /* 380 */ 809, 1189, 1204, 1207, 1212, 1216, 1217, 1229, 1231, 1142, + /* 390 */ 1208, 1187, 1234, 1203, 1267, 1270, 1272, 1277, 1289, 1294, + /* 400 */ 190, 1299, 1348, 668, 1320, 1613, 1616, 1617, 1576, 1619, + /* 410 */ 1585, 1414, 1589, 1590, 1591, 1419, 1628, 1595, 1596, 1424, + /* 420 */ 1633, 1426, 1635, 1601, 1637, 1618, 1638, 1604, 1447, 1456, + /* 430 */ 1459, 1646, 1647, 1648, 1467, 1469, 1651, 1652, 1606, 1655, + /* 440 */ 1656, 1657, 1625, 1659, 1660, 1669, 1670, 1671, 1672, 1689, + /* 450 */ 1690, 1521, 1661, 1691, 1537, 1694, 1697, 1698, 1702, 1703, + /* 460 */ 1704, 1705, 1706, 1707, 1708, 1709, 1710, 1712, 1714, 1715, + /* 470 */ 1716, 1675, 1718, 1719, 1720, 1721, 1722, 1723, 1711, 1724, + /* 480 */ 1726, 1728, 1592, 1729, 1730, 1713, 1684, 1717, 1686, 1736, + /* 490 */ 1679, 1725, 1740, 1682, 1742, 1685, 1743, 1745, 1731, 1732, + /* 500 */ 1733, 1699, 1700, 1734, 1727, 1752, 1735, 1737, 1753, 1754, + /* 510 */ 1755, 1741, 1573, 1756, 1757, 1759, 1695, 1763, 1764, 1746, + /* 520 */ 1738, 1758, 1785, 1760, 1749, 1761, 1799, 1767, 1766, 1765, + /* 530 */ 1805, 1771, 1768, 1773, 1809, 1810, 1813, 1816, 1739, 1747, + /* 540 */ 1783, 1797, 1820, 1786, 1787, 1788, 1790, 1792, 1794, 1795, + /* 550 */ 1789, 1791, 1796, 1801, 1817, 1803, 1832, 1818, 1841, 1821, + /* 560 */ 1793, 1842, 1823, 1811, 1850, 1826, 1852, 1827, 1853, 1833, + /* 570 */ 1836, 1829, 1840, 1843, 1775, 1777, 1858, 1762, 1779, 1688, + /* 580 */ 1863, 1865, 1899, 1744, 1878, 1776, 1748, 1901, 1902, 1780, + /* 590 */ 1751, 1903, 1860, 1665, 1807, 1822, 1824, 1874, 1831, 1877, + /* 600 */ 1815, 1830, 1890, 1897, 1844, 1847, 1849, 1851, 1854, 1906, + /* 610 */ 1880, 1896, 1856, 1908, 1701, 1861, 1862, 1950, 1910, 1772, + /* 620 */ 1924, 1926, 1929, 1930, 1931, 1932, 1868, 1869, 1921, 1774, + /* 630 */ 1927, 1925, 1973, 1974, 1976, 1977, 1879, 1939, 1699, 1933, + /* 640 */ 1881, 1882, 1885, 1884, 1887, 1819, 1888, 1982, 1953, 1825, + /* 650 */ 1889, 1894, 1699, 1946, 1963, 1912, 1778, 1914, 1995, 1986, + /* 660 */ 1798, 1916, 1917, 1920, 1928, 1922, 1934, 1967, 1936, 1938, + /* 670 */ 1970, 1941, 1999, 1814, 1940, 1915, 1942, 1992, 1998, 1947, + /* 680 */ 1948, 2004, 1951, 1949, 2013, 1952, 1954, 2018, 1958, 1960, + /* 690 */ 2019, 1962, 1956, 1957, 1983, 1985, 1969, 1990, 1971, 2037, + /* 700 */ 1975, 1990, 1990, 2008, 2010, 2009, 2038, 2045, 2047, 2057, + /* 710 */ 2060, 2061, 2073, 2074, 2075, 2076, 2040, 2022, 2070, 2082, + /* 720 */ 2083, 2085, 2099, 2087, 2088, 2090, 2054, 1789, 2092, 1791, + /* 730 */ 2093, 2094, 2095, 2097, 2112, 2100, 2136, 2102, 2089, 2101, + /* 740 */ 2141, 2107, 2096, 2104, 2146, 2113, 2098, 2110, 2150, 2117, + /* 750 */ 2105, 2116, 2156, 2122, 2123, 2159, 2138, 2140, 2143, 2144, + /* 760 */ 2142, 2147, }; -#define YY_REDUCE_COUNT (308) -#define YY_REDUCE_MIN (-424) -#define YY_REDUCE_MAX (2669) +#define YY_REDUCE_COUNT (312) +#define YY_REDUCE_MIN (-396) +#define YY_REDUCE_MAX (2601) static const short yy_reduce_ofst[] = { - /* 0 */ 373, -119, 368, 588, 774, 995, 1065, 1116, 1191, -17, - /* 10 */ 446, 672, 880, 1310, 1375, -332, 903, 155, 1440, 1491, - /* 20 */ 1510, 1585, 1604, 1677, 1703, 1774, 1795, 1825, 1890, 1919, - /* 30 */ 1953, 2017, 2037, 2098, 2149, 2170, 2200, 2265, 2294, 2328, - /* 40 */ 2392, 2412, 2473, 2524, 2545, 2575, 2640, 2669, 96, -184, - /* 50 */ -395, -99, -392, -134, 488, 761, -87, 470, -89, 281, - /* 60 */ 398, -424, -364, -360, -384, 806, 42, -314, -177, 391, - /* 70 */ -379, -207, -156, -350, 86, 156, 202, -331, 178, 394, - /* 80 */ 402, 462, 507, 577, -101, 585, 646, -34, -334, 700, - /* 90 */ 708, 196, 730, 182, 801, 190, 804, 302, 278, 846, - /* 100 */ 858, 401, 860, 505, 450, 556, 878, -301, -63, -243, - /* 110 */ -243, -110, -323, -85, -81, 59, 153, 357, 358, 407, - /* 120 */ 541, 549, 594, 627, 639, 658, 663, 665, 669, 680, - /* 130 */ 682, 683, -302, 530, 267, 7, 124, 530, 545, 637, - /* 140 */ 185, -7, 363, 206, 104, 603, 671, -22, -104, 611, - /* 150 */ 451, 710, 686, 721, 723, 749, 630, -367, -362, 338, - /* 160 */ 429, 582, 795, 797, 582, 520, 847, 922, 871, 792, - /* 170 */ 808, 940, 829, 924, 924, 948, 906, 957, 934, 941, - /* 180 */ 911, 911, 893, 911, 921, 912, 924, 961, 970, 985, - /* 190 */ 1000, 1002, 1006, 1009, 1054, 1056, 1013, 1017, 1018, 1052, - /* 200 */ 1055, 1068, 1059, 1069, 1071, 1072, 1087, 1089, 1027, 1088, - /* 210 */ 1060, 1091, 1100, 1048, 1103, 1106, 1104, 1105, 1109, 1113, - /* 220 */ 1110, 1111, 1114, 1124, 1094, 1095, 1096, 1097, 1098, 1099, - /* 230 */ 1102, 1117, 1127, 1128, 1138, 1125, 1132, 1092, 1093, 1122, - /* 240 */ 1063, 1066, 1073, 1133, 1070, 1074, 1129, 1160, 1115, 1166, - /* 250 */ 1119, 1123, 1136, 1061, 1137, 1142, 1075, 1140, 1144, 1146, - /* 260 */ 924, 1083, 1076, 1118, 1134, 1130, 1141, 1145, 1077, 1078, - /* 270 */ 1085, 911, 1181, 1143, 1219, 1220, 1218, 1221, 1176, 1175, - /* 280 */ 1192, 1195, 1196, 1197, 1198, 1184, 1201, 1190, 1240, 1224, - /* 290 */ 1244, 1213, 1154, 1225, 1212, 1238, 1256, 1251, 1265, 1269, - /* 300 */ 1199, 1194, 1204, 1210, 1246, 1248, 1258, 1273, 1285, + /* 0 */ -140, 137, -82, 358, 444, 578, 814, 841, 1034, 1064, + /* 10 */ 1132, 1240, 1260, 1344, 1368, 1445, 598, -333, 672, 1468, + /* 20 */ 901, 1491, 1514, 1577, 1642, 1663, 1676, 1750, 1770, 1867, + /* 30 */ 1883, 1913, 1943, 2007, 2020, 2071, 2084, 2145, 2196, 2211, + /* 40 */ 2257, 2308, 2325, 2354, 2422, 2441, 2454, 2527, 2540, 2555, + /* 50 */ 2601, 372, -195, -396, -265, 487, 595, 607, 701, 481, + /* 60 */ 761, -362, 191, -365, 136, -279, -55, 256, -372, -167, + /* 70 */ -5, -381, -273, -257, 14, -321, -337, 203, 45, 285, + /* 80 */ 300, 356, 377, 409, 425, 177, 432, 489, 446, 279, + /* 90 */ 521, 573, 125, 620, 321, 622, 396, 687, 436, 257, + /* 100 */ 707, 716, 460, 739, 434, 506, 679, 746, -266, 8, + /* 110 */ -350, -350, 123, -261, 265, 363, 579, 585, 612, 617, + /* 120 */ 710, 754, 769, 774, 777, 781, 806, 807, 813, 820, + /* 130 */ 830, 831, 832, -331, -289, -111, -249, -61, -289, 290, + /* 140 */ 563, -25, -338, 158, 357, -147, 498, 533, -102, 202, + /* 150 */ 324, 29, 378, 502, 593, 597, 808, 236, -367, 552, + /* 160 */ 584, 603, 614, 674, 857, 614, 785, 871, 945, 877, + /* 170 */ 811, 823, 937, 834, 920, 920, 946, 904, 963, 985, + /* 180 */ 926, 919, 882, 882, 890, 882, 917, 911, 920, 949, + /* 190 */ 957, 973, 996, 995, 999, 1002, 1051, 1052, 1010, 1013, + /* 200 */ 1014, 1050, 1055, 1068, 1060, 1074, 1076, 1078, 1079, 1083, + /* 210 */ 1088, 1085, 1017, 1077, 1046, 1080, 1090, 1035, 1087, 1091, + /* 220 */ 1093, 1096, 1099, 1097, 1100, 1094, 1095, 1116, 1084, 1101, + /* 230 */ 1102, 1103, 1107, 1108, 1111, 1112, 1114, 1115, 1122, 1098, + /* 240 */ 1119, 1082, 1113, 1124, 1053, 1106, 1059, 1131, 1104, 1109, + /* 250 */ 1123, 1148, 1118, 1158, 1120, 1127, 1126, 1054, 1125, 1128, + /* 260 */ 1058, 1129, 1139, 1141, 920, 1069, 1073, 1081, 1075, 1086, + /* 270 */ 1133, 1135, 1061, 1130, 1134, 882, 1198, 1144, 1201, 1200, + /* 280 */ 1197, 1199, 1155, 1163, 1180, 1188, 1193, 1206, 1209, 1174, + /* 290 */ 1210, 1173, 1220, 1227, 1236, 1239, 1150, 1221, 1214, 1237, + /* 300 */ 1256, 1249, 1268, 1266, 1202, 1196, 1211, 1213, 1244, 1251, + /* 310 */ 1252, 1262, 1281, }; static const YYACTIONTYPE yy_default[] = { - /* 0 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 10 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 20 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 30 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 40 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 50 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 60 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 70 */ 1714, 1714, 1714, 1985, 1714, 1714, 1714, 1714, 1714, 1714, - /* 80 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1793, 1714, 1714, - /* 90 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 100 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1791, 1978, 2199, - /* 110 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 120 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 130 */ 1714, 1714, 1714, 2211, 1714, 1714, 1714, 2211, 2211, 2211, - /* 140 */ 1791, 2171, 2171, 1714, 1714, 1714, 1714, 1793, 2043, 1714, - /* 150 */ 1714, 1714, 1714, 1714, 1714, 1913, 1714, 1714, 1714, 1714, - /* 160 */ 1714, 1937, 1714, 1714, 1714, 2037, 1714, 1714, 2236, 2292, - /* 170 */ 1714, 1714, 2239, 1714, 1714, 1714, 1990, 1714, 1866, 2226, - /* 180 */ 2203, 2217, 2276, 2204, 2201, 2220, 1714, 2230, 1714, 2024, - /* 190 */ 1983, 1714, 1983, 1980, 1714, 1714, 1983, 1980, 1980, 1855, - /* 200 */ 1851, 1714, 1849, 1714, 1714, 1714, 1714, 1714, 1714, 1793, - /* 210 */ 1714, 1793, 1714, 1714, 1793, 1714, 1793, 1793, 1793, 1714, - /* 220 */ 1793, 1771, 1771, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 230 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 240 */ 2055, 2041, 1714, 1791, 2035, 2033, 1714, 1791, 2031, 1714, - /* 250 */ 1714, 1714, 1714, 2247, 2245, 1714, 2247, 2245, 1714, 1714, - /* 260 */ 1714, 2261, 2257, 2247, 2265, 2263, 2232, 2230, 2295, 2282, - /* 270 */ 2278, 2217, 1714, 1714, 1714, 1714, 1791, 1791, 1714, 2245, - /* 280 */ 1714, 1714, 1714, 1714, 1714, 2245, 1714, 1714, 1791, 1714, - /* 290 */ 1791, 1714, 1714, 1882, 1714, 1714, 1714, 1791, 1746, 1714, - /* 300 */ 2026, 2046, 2008, 2008, 1916, 1916, 1916, 1794, 1719, 1714, - /* 310 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 320 */ 1714, 2260, 2259, 2126, 1714, 2175, 2174, 2173, 2164, 2125, - /* 330 */ 1878, 1714, 2124, 2123, 1714, 1714, 1714, 1714, 1714, 1714, - /* 340 */ 1714, 1714, 1999, 1998, 2117, 1714, 1714, 2118, 2116, 2115, - /* 350 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 360 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 370 */ 1714, 1714, 1714, 1714, 2279, 2283, 1714, 1714, 1714, 1714, - /* 380 */ 1714, 1714, 1714, 2200, 1714, 1714, 1714, 1714, 1714, 2099, - /* 390 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 400 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 410 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 420 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 430 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 440 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 450 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 460 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 470 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 480 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 490 */ 1714, 1714, 1714, 1714, 1714, 1714, 1751, 2104, 1714, 1714, - /* 500 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 510 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 520 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 530 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 540 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1832, 1831, 1714, - /* 550 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 560 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 570 */ 1714, 2108, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 580 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 2275, 2233, - /* 590 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 600 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 2099, 1714, - /* 610 */ 2258, 1714, 1714, 2273, 1714, 2277, 1714, 1714, 1714, 1714, - /* 620 */ 1714, 1714, 1714, 2210, 2206, 1714, 1714, 2202, 1714, 1714, - /* 630 */ 1714, 1714, 1714, 1714, 1714, 2107, 1714, 1714, 1714, 1714, - /* 640 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 2098, - /* 650 */ 1714, 2161, 1714, 1714, 1714, 2195, 1714, 1714, 2146, 1714, - /* 660 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 2108, 1714, - /* 670 */ 2111, 1714, 1714, 1714, 1714, 1714, 1910, 1714, 1714, 1714, - /* 680 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1895, - /* 690 */ 1893, 1892, 1891, 1714, 1888, 1714, 1923, 1714, 1714, 1714, - /* 700 */ 1919, 1918, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 710 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1812, 1714, 1714, - /* 720 */ 1714, 1714, 1714, 1714, 1714, 1714, 1804, 1714, 1803, 1714, - /* 730 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 740 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 750 */ 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, 1714, - /* 760 */ 1714, + /* 0 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 10 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 20 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 30 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 40 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 50 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 60 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 70 */ 1719, 1719, 1719, 1719, 1992, 1719, 1719, 1719, 1719, 1719, + /* 80 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1801, 1719, + /* 90 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 100 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1799, 1985, + /* 110 */ 2206, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 120 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 130 */ 1719, 1719, 1719, 1719, 2218, 1719, 1719, 1719, 2218, 2218, + /* 140 */ 2218, 1799, 2178, 2178, 1719, 1719, 1719, 1719, 1801, 2050, + /* 150 */ 1719, 1719, 1719, 1719, 1719, 1719, 1920, 1719, 1719, 1719, + /* 160 */ 1719, 1719, 1944, 1719, 1719, 1719, 2044, 1719, 1719, 2243, + /* 170 */ 2299, 1719, 1719, 2246, 1719, 1719, 1719, 1997, 1719, 1719, + /* 180 */ 1874, 2233, 2210, 2224, 2283, 2211, 2208, 2227, 1719, 2237, + /* 190 */ 1719, 2031, 1990, 1719, 1990, 1987, 1719, 1719, 1990, 1987, + /* 200 */ 1987, 1863, 1859, 1719, 1857, 1719, 1719, 1719, 1719, 1766, + /* 210 */ 1719, 1766, 1719, 1801, 1719, 1801, 1719, 1719, 1801, 1719, + /* 220 */ 1801, 1801, 1801, 1719, 1801, 1779, 1779, 1719, 1719, 1719, + /* 230 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 240 */ 1719, 1719, 1719, 1719, 2062, 2048, 1719, 1799, 2042, 2040, + /* 250 */ 1719, 1799, 2038, 1719, 1719, 1719, 1719, 2254, 2252, 1719, + /* 260 */ 2254, 2252, 1719, 1719, 1719, 2268, 2264, 2254, 2272, 2270, + /* 270 */ 2239, 2237, 2302, 2289, 2285, 2224, 1719, 1719, 1719, 1719, + /* 280 */ 1799, 1799, 1719, 2252, 1719, 1719, 1719, 1719, 1719, 2252, + /* 290 */ 1719, 1719, 1799, 1719, 1799, 1719, 1719, 1890, 1719, 1719, + /* 300 */ 1719, 1799, 1751, 1719, 2033, 2053, 2015, 2015, 1923, 1923, + /* 310 */ 1923, 1802, 1724, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 320 */ 1719, 1719, 1719, 1719, 1719, 2267, 2266, 2133, 1719, 2182, + /* 330 */ 2181, 2180, 2171, 2132, 1886, 1719, 2131, 2130, 1719, 1719, + /* 340 */ 1719, 1719, 1719, 1719, 1719, 1719, 2006, 2005, 2124, 1719, + /* 350 */ 1719, 2125, 2123, 2122, 1719, 1719, 1719, 1719, 1719, 1719, + /* 360 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 370 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2286, 2290, + /* 380 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2207, 1719, 1719, + /* 390 */ 1719, 1719, 1719, 2106, 1719, 1719, 1719, 1719, 1719, 1719, + /* 400 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 410 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 420 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 430 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 440 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 450 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 460 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 470 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 480 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 490 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 500 */ 1756, 2111, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 510 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 520 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 530 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 540 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 550 */ 1840, 1839, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 560 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 570 */ 1719, 1719, 1719, 1719, 2115, 1719, 1719, 1719, 1719, 1719, + /* 580 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 590 */ 1719, 2282, 2240, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 600 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 610 */ 1719, 2106, 1719, 2265, 1719, 1719, 2280, 1719, 2284, 1719, + /* 620 */ 1719, 1719, 1719, 1719, 1719, 1719, 2217, 2213, 1719, 1719, + /* 630 */ 2209, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 2114, 1719, + /* 640 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 650 */ 1719, 1719, 2105, 1719, 2168, 1719, 1719, 1719, 2202, 1719, + /* 660 */ 1719, 2153, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 670 */ 1719, 2115, 1719, 2118, 1719, 1719, 1719, 1719, 1719, 1917, + /* 680 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 690 */ 1719, 1719, 1902, 1900, 1899, 1898, 1719, 1930, 1719, 1719, + /* 700 */ 1719, 1926, 1925, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 710 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1820, 1719, + /* 720 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1812, 1719, 1811, + /* 730 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 740 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 750 */ 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, 1719, + /* 760 */ 1719, 1719, }; /********** End of lemon-generated parsing tables *****************************/ @@ -1104,6 +1092,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* READ => nothing */ 0, /* WRITE => nothing */ 0, /* NK_DOT => nothing */ + 0, /* WITH => nothing */ 0, /* DNODE => nothing */ 0, /* PORT => nothing */ 0, /* DNODES => nothing */ @@ -1152,7 +1141,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* NK_COLON => nothing */ 0, /* MAX_SPEED => nothing */ 0, /* START => nothing */ - 0, /* WITH => nothing */ 0, /* TIMESTAMP => nothing */ 279, /* END => ABORT */ 0, /* TABLE => nothing */ @@ -1168,7 +1156,6 @@ static const YYCODETYPE yyFallback[] = { 0, /* NK_EQ => nothing */ 0, /* USING => nothing */ 0, /* TAGS => nothing */ - 0, /* COMMENT => nothing */ 0, /* BOOL => nothing */ 0, /* TINYINT => nothing */ 0, /* SMALLINT => nothing */ @@ -1186,6 +1173,7 @@ static const YYCODETYPE yyFallback[] = { 0, /* BLOB => nothing */ 0, /* VARBINARY => nothing */ 0, /* DECIMAL => nothing */ + 0, /* COMMENT => nothing */ 0, /* MAX_DELAY => nothing */ 0, /* WATERMARK => nothing */ 0, /* ROLLUP => nothing */ @@ -1521,55 +1509,55 @@ static const char *const yyTokenName[] = { /* 45 */ "READ", /* 46 */ "WRITE", /* 47 */ "NK_DOT", - /* 48 */ "DNODE", - /* 49 */ "PORT", - /* 50 */ "DNODES", - /* 51 */ "NK_IPTOKEN", - /* 52 */ "FORCE", - /* 53 */ "LOCAL", - /* 54 */ "QNODE", - /* 55 */ "BNODE", - /* 56 */ "SNODE", - /* 57 */ "MNODE", - /* 58 */ "DATABASE", - /* 59 */ "USE", - /* 60 */ "FLUSH", - /* 61 */ "TRIM", - /* 62 */ "COMPACT", - /* 63 */ "IF", - /* 64 */ "NOT", - /* 65 */ "EXISTS", - /* 66 */ "BUFFER", - /* 67 */ "CACHEMODEL", - /* 68 */ "CACHESIZE", - /* 69 */ "COMP", - /* 70 */ "DURATION", - /* 71 */ "NK_VARIABLE", - /* 72 */ "MAXROWS", - /* 73 */ "MINROWS", - /* 74 */ "KEEP", - /* 75 */ "PAGES", - /* 76 */ "PAGESIZE", - /* 77 */ "TSDB_PAGESIZE", - /* 78 */ "PRECISION", - /* 79 */ "REPLICA", - /* 80 */ "VGROUPS", - /* 81 */ "SINGLE_STABLE", - /* 82 */ "RETENTIONS", - /* 83 */ "SCHEMALESS", - /* 84 */ "WAL_LEVEL", - /* 85 */ "WAL_FSYNC_PERIOD", - /* 86 */ "WAL_RETENTION_PERIOD", - /* 87 */ "WAL_RETENTION_SIZE", - /* 88 */ "WAL_ROLL_PERIOD", - /* 89 */ "WAL_SEGMENT_SIZE", - /* 90 */ "STT_TRIGGER", - /* 91 */ "TABLE_PREFIX", - /* 92 */ "TABLE_SUFFIX", - /* 93 */ "NK_COLON", - /* 94 */ "MAX_SPEED", - /* 95 */ "START", - /* 96 */ "WITH", + /* 48 */ "WITH", + /* 49 */ "DNODE", + /* 50 */ "PORT", + /* 51 */ "DNODES", + /* 52 */ "NK_IPTOKEN", + /* 53 */ "FORCE", + /* 54 */ "LOCAL", + /* 55 */ "QNODE", + /* 56 */ "BNODE", + /* 57 */ "SNODE", + /* 58 */ "MNODE", + /* 59 */ "DATABASE", + /* 60 */ "USE", + /* 61 */ "FLUSH", + /* 62 */ "TRIM", + /* 63 */ "COMPACT", + /* 64 */ "IF", + /* 65 */ "NOT", + /* 66 */ "EXISTS", + /* 67 */ "BUFFER", + /* 68 */ "CACHEMODEL", + /* 69 */ "CACHESIZE", + /* 70 */ "COMP", + /* 71 */ "DURATION", + /* 72 */ "NK_VARIABLE", + /* 73 */ "MAXROWS", + /* 74 */ "MINROWS", + /* 75 */ "KEEP", + /* 76 */ "PAGES", + /* 77 */ "PAGESIZE", + /* 78 */ "TSDB_PAGESIZE", + /* 79 */ "PRECISION", + /* 80 */ "REPLICA", + /* 81 */ "VGROUPS", + /* 82 */ "SINGLE_STABLE", + /* 83 */ "RETENTIONS", + /* 84 */ "SCHEMALESS", + /* 85 */ "WAL_LEVEL", + /* 86 */ "WAL_FSYNC_PERIOD", + /* 87 */ "WAL_RETENTION_PERIOD", + /* 88 */ "WAL_RETENTION_SIZE", + /* 89 */ "WAL_ROLL_PERIOD", + /* 90 */ "WAL_SEGMENT_SIZE", + /* 91 */ "STT_TRIGGER", + /* 92 */ "TABLE_PREFIX", + /* 93 */ "TABLE_SUFFIX", + /* 94 */ "NK_COLON", + /* 95 */ "MAX_SPEED", + /* 96 */ "START", /* 97 */ "TIMESTAMP", /* 98 */ "END", /* 99 */ "TABLE", @@ -1585,24 +1573,24 @@ static const char *const yyTokenName[] = { /* 109 */ "NK_EQ", /* 110 */ "USING", /* 111 */ "TAGS", - /* 112 */ "COMMENT", - /* 113 */ "BOOL", - /* 114 */ "TINYINT", - /* 115 */ "SMALLINT", - /* 116 */ "INT", - /* 117 */ "INTEGER", - /* 118 */ "BIGINT", - /* 119 */ "FLOAT", - /* 120 */ "DOUBLE", - /* 121 */ "BINARY", - /* 122 */ "NCHAR", - /* 123 */ "UNSIGNED", - /* 124 */ "JSON", - /* 125 */ "VARCHAR", - /* 126 */ "MEDIUMBLOB", - /* 127 */ "BLOB", - /* 128 */ "VARBINARY", - /* 129 */ "DECIMAL", + /* 112 */ "BOOL", + /* 113 */ "TINYINT", + /* 114 */ "SMALLINT", + /* 115 */ "INT", + /* 116 */ "INTEGER", + /* 117 */ "BIGINT", + /* 118 */ "FLOAT", + /* 119 */ "DOUBLE", + /* 120 */ "BINARY", + /* 121 */ "NCHAR", + /* 122 */ "UNSIGNED", + /* 123 */ "JSON", + /* 124 */ "VARCHAR", + /* 125 */ "MEDIUMBLOB", + /* 126 */ "BLOB", + /* 127 */ "VARBINARY", + /* 128 */ "DECIMAL", + /* 129 */ "COMMENT", /* 130 */ "MAX_DELAY", /* 131 */ "WATERMARK", /* 132 */ "ROLLUP", @@ -1812,142 +1800,143 @@ static const char *const yyTokenName[] = { /* 336 */ "sysinfo_opt", /* 337 */ "privileges", /* 338 */ "priv_level", - /* 339 */ "priv_type_list", - /* 340 */ "priv_type", - /* 341 */ "db_name", - /* 342 */ "topic_name", - /* 343 */ "dnode_endpoint", - /* 344 */ "force_opt", - /* 345 */ "not_exists_opt", - /* 346 */ "db_options", - /* 347 */ "exists_opt", - /* 348 */ "alter_db_options", - /* 349 */ "speed_opt", - /* 350 */ "start_opt", - /* 351 */ "end_opt", - /* 352 */ "integer_list", - /* 353 */ "variable_list", - /* 354 */ "retention_list", - /* 355 */ "alter_db_option", - /* 356 */ "retention", - /* 357 */ "full_table_name", - /* 358 */ "column_def_list", - /* 359 */ "tags_def_opt", - /* 360 */ "table_options", - /* 361 */ "multi_create_clause", - /* 362 */ "tags_def", - /* 363 */ "multi_drop_clause", - /* 364 */ "alter_table_clause", - /* 365 */ "alter_table_options", - /* 366 */ "column_name", - /* 367 */ "type_name", - /* 368 */ "signed_literal", - /* 369 */ "create_subtable_clause", - /* 370 */ "specific_cols_opt", - /* 371 */ "expression_list", - /* 372 */ "drop_table_clause", - /* 373 */ "col_name_list", - /* 374 */ "table_name", - /* 375 */ "column_def", - /* 376 */ "duration_list", - /* 377 */ "rollup_func_list", - /* 378 */ "alter_table_option", - /* 379 */ "duration_literal", - /* 380 */ "rollup_func_name", - /* 381 */ "function_name", - /* 382 */ "col_name", - /* 383 */ "db_name_cond_opt", - /* 384 */ "like_pattern_opt", - /* 385 */ "table_name_cond", - /* 386 */ "from_db_opt", - /* 387 */ "tag_list_opt", - /* 388 */ "tag_item", - /* 389 */ "column_alias", - /* 390 */ "full_index_name", - /* 391 */ "index_options", - /* 392 */ "index_name", - /* 393 */ "func_list", - /* 394 */ "sliding_opt", - /* 395 */ "sma_stream_opt", - /* 396 */ "func", - /* 397 */ "sma_func_name", - /* 398 */ "query_or_subquery", - /* 399 */ "cgroup_name", - /* 400 */ "analyze_opt", - /* 401 */ "explain_options", - /* 402 */ "insert_query", - /* 403 */ "or_replace_opt", - /* 404 */ "agg_func_opt", - /* 405 */ "bufsize_opt", - /* 406 */ "language_opt", - /* 407 */ "stream_name", - /* 408 */ "stream_options", - /* 409 */ "col_list_opt", - /* 410 */ "tag_def_or_ref_opt", - /* 411 */ "subtable_opt", - /* 412 */ "expression", - /* 413 */ "dnode_list", - /* 414 */ "where_clause_opt", - /* 415 */ "signed", - /* 416 */ "literal_func", - /* 417 */ "literal_list", - /* 418 */ "table_alias", - /* 419 */ "expr_or_subquery", - /* 420 */ "pseudo_column", - /* 421 */ "column_reference", - /* 422 */ "function_expression", - /* 423 */ "case_when_expression", - /* 424 */ "star_func", - /* 425 */ "star_func_para_list", - /* 426 */ "noarg_func", - /* 427 */ "other_para_list", - /* 428 */ "star_func_para", - /* 429 */ "when_then_list", - /* 430 */ "case_when_else_opt", - /* 431 */ "common_expression", - /* 432 */ "when_then_expr", - /* 433 */ "predicate", - /* 434 */ "compare_op", - /* 435 */ "in_op", - /* 436 */ "in_predicate_value", - /* 437 */ "boolean_value_expression", - /* 438 */ "boolean_primary", - /* 439 */ "from_clause_opt", - /* 440 */ "table_reference_list", - /* 441 */ "table_reference", - /* 442 */ "table_primary", - /* 443 */ "joined_table", - /* 444 */ "alias_opt", - /* 445 */ "subquery", - /* 446 */ "parenthesized_joined_table", - /* 447 */ "join_type", - /* 448 */ "search_condition", - /* 449 */ "query_specification", - /* 450 */ "set_quantifier_opt", - /* 451 */ "select_list", - /* 452 */ "partition_by_clause_opt", - /* 453 */ "range_opt", - /* 454 */ "every_opt", - /* 455 */ "fill_opt", - /* 456 */ "twindow_clause_opt", - /* 457 */ "group_by_clause_opt", - /* 458 */ "having_clause_opt", - /* 459 */ "select_item", - /* 460 */ "partition_list", - /* 461 */ "partition_item", - /* 462 */ "fill_mode", - /* 463 */ "group_by_list", - /* 464 */ "query_expression", - /* 465 */ "query_simple", - /* 466 */ "order_by_clause_opt", - /* 467 */ "slimit_clause_opt", - /* 468 */ "limit_clause_opt", - /* 469 */ "union_query_expression", - /* 470 */ "query_simple_or_subquery", - /* 471 */ "sort_specification_list", - /* 472 */ "sort_specification", - /* 473 */ "ordering_specification_opt", - /* 474 */ "null_ordering_opt", + /* 339 */ "with_opt", + /* 340 */ "priv_type_list", + /* 341 */ "priv_type", + /* 342 */ "db_name", + /* 343 */ "table_name", + /* 344 */ "topic_name", + /* 345 */ "search_condition", + /* 346 */ "dnode_endpoint", + /* 347 */ "force_opt", + /* 348 */ "not_exists_opt", + /* 349 */ "db_options", + /* 350 */ "exists_opt", + /* 351 */ "alter_db_options", + /* 352 */ "speed_opt", + /* 353 */ "start_opt", + /* 354 */ "end_opt", + /* 355 */ "integer_list", + /* 356 */ "variable_list", + /* 357 */ "retention_list", + /* 358 */ "alter_db_option", + /* 359 */ "retention", + /* 360 */ "full_table_name", + /* 361 */ "column_def_list", + /* 362 */ "tags_def_opt", + /* 363 */ "table_options", + /* 364 */ "multi_create_clause", + /* 365 */ "tags_def", + /* 366 */ "multi_drop_clause", + /* 367 */ "alter_table_clause", + /* 368 */ "alter_table_options", + /* 369 */ "column_name", + /* 370 */ "type_name", + /* 371 */ "signed_literal", + /* 372 */ "create_subtable_clause", + /* 373 */ "specific_cols_opt", + /* 374 */ "expression_list", + /* 375 */ "drop_table_clause", + /* 376 */ "col_name_list", + /* 377 */ "column_def", + /* 378 */ "duration_list", + /* 379 */ "rollup_func_list", + /* 380 */ "alter_table_option", + /* 381 */ "duration_literal", + /* 382 */ "rollup_func_name", + /* 383 */ "function_name", + /* 384 */ "col_name", + /* 385 */ "db_name_cond_opt", + /* 386 */ "like_pattern_opt", + /* 387 */ "table_name_cond", + /* 388 */ "from_db_opt", + /* 389 */ "tag_list_opt", + /* 390 */ "tag_item", + /* 391 */ "column_alias", + /* 392 */ "full_index_name", + /* 393 */ "index_options", + /* 394 */ "index_name", + /* 395 */ "func_list", + /* 396 */ "sliding_opt", + /* 397 */ "sma_stream_opt", + /* 398 */ "func", + /* 399 */ "sma_func_name", + /* 400 */ "query_or_subquery", + /* 401 */ "cgroup_name", + /* 402 */ "analyze_opt", + /* 403 */ "explain_options", + /* 404 */ "insert_query", + /* 405 */ "or_replace_opt", + /* 406 */ "agg_func_opt", + /* 407 */ "bufsize_opt", + /* 408 */ "language_opt", + /* 409 */ "stream_name", + /* 410 */ "stream_options", + /* 411 */ "col_list_opt", + /* 412 */ "tag_def_or_ref_opt", + /* 413 */ "subtable_opt", + /* 414 */ "expression", + /* 415 */ "dnode_list", + /* 416 */ "where_clause_opt", + /* 417 */ "signed", + /* 418 */ "literal_func", + /* 419 */ "literal_list", + /* 420 */ "table_alias", + /* 421 */ "expr_or_subquery", + /* 422 */ "pseudo_column", + /* 423 */ "column_reference", + /* 424 */ "function_expression", + /* 425 */ "case_when_expression", + /* 426 */ "star_func", + /* 427 */ "star_func_para_list", + /* 428 */ "noarg_func", + /* 429 */ "other_para_list", + /* 430 */ "star_func_para", + /* 431 */ "when_then_list", + /* 432 */ "case_when_else_opt", + /* 433 */ "common_expression", + /* 434 */ "when_then_expr", + /* 435 */ "predicate", + /* 436 */ "compare_op", + /* 437 */ "in_op", + /* 438 */ "in_predicate_value", + /* 439 */ "boolean_value_expression", + /* 440 */ "boolean_primary", + /* 441 */ "from_clause_opt", + /* 442 */ "table_reference_list", + /* 443 */ "table_reference", + /* 444 */ "table_primary", + /* 445 */ "joined_table", + /* 446 */ "alias_opt", + /* 447 */ "subquery", + /* 448 */ "parenthesized_joined_table", + /* 449 */ "join_type", + /* 450 */ "query_specification", + /* 451 */ "set_quantifier_opt", + /* 452 */ "select_list", + /* 453 */ "partition_by_clause_opt", + /* 454 */ "range_opt", + /* 455 */ "every_opt", + /* 456 */ "fill_opt", + /* 457 */ "twindow_clause_opt", + /* 458 */ "group_by_clause_opt", + /* 459 */ "having_clause_opt", + /* 460 */ "select_item", + /* 461 */ "partition_list", + /* 462 */ "partition_item", + /* 463 */ "fill_mode", + /* 464 */ "group_by_list", + /* 465 */ "query_expression", + /* 466 */ "query_simple", + /* 467 */ "order_by_clause_opt", + /* 468 */ "slimit_clause_opt", + /* 469 */ "limit_clause_opt", + /* 470 */ "union_query_expression", + /* 471 */ "query_simple_or_subquery", + /* 472 */ "sort_specification_list", + /* 473 */ "sort_specification", + /* 474 */ "ordering_specification_opt", + /* 475 */ "null_ordering_opt", }; #endif /* defined(YYCOVERAGE) || !defined(NDEBUG) */ @@ -1986,8 +1975,8 @@ static const char *const yyRuleName[] = { /* 28 */ "cmd ::= DROP USER user_name", /* 29 */ "sysinfo_opt ::=", /* 30 */ "sysinfo_opt ::= SYSINFO NK_INTEGER", - /* 31 */ "cmd ::= GRANT privileges ON priv_level TO user_name", - /* 32 */ "cmd ::= REVOKE privileges ON priv_level FROM user_name", + /* 31 */ "cmd ::= GRANT privileges ON priv_level with_opt TO user_name", + /* 32 */ "cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name", /* 33 */ "privileges ::= ALL", /* 34 */ "privileges ::= priv_type_list", /* 35 */ "privileges ::= SUBSCRIBE", @@ -1997,545 +1986,547 @@ static const char *const yyRuleName[] = { /* 39 */ "priv_type ::= WRITE", /* 40 */ "priv_level ::= NK_STAR NK_DOT NK_STAR", /* 41 */ "priv_level ::= db_name NK_DOT NK_STAR", - /* 42 */ "priv_level ::= topic_name", - /* 43 */ "cmd ::= CREATE DNODE dnode_endpoint", - /* 44 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", - /* 45 */ "cmd ::= DROP DNODE NK_INTEGER force_opt", - /* 46 */ "cmd ::= DROP DNODE dnode_endpoint force_opt", - /* 47 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", - /* 48 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", - /* 49 */ "cmd ::= ALTER ALL DNODES NK_STRING", - /* 50 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", - /* 51 */ "dnode_endpoint ::= NK_STRING", - /* 52 */ "dnode_endpoint ::= NK_ID", - /* 53 */ "dnode_endpoint ::= NK_IPTOKEN", - /* 54 */ "force_opt ::=", - /* 55 */ "force_opt ::= FORCE", - /* 56 */ "cmd ::= ALTER LOCAL NK_STRING", - /* 57 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", - /* 58 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", - /* 59 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", - /* 60 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", - /* 61 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", - /* 62 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", - /* 63 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", - /* 64 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", - /* 65 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", - /* 66 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", - /* 67 */ "cmd ::= DROP DATABASE exists_opt db_name", - /* 68 */ "cmd ::= USE db_name", - /* 69 */ "cmd ::= ALTER DATABASE db_name alter_db_options", - /* 70 */ "cmd ::= FLUSH DATABASE db_name", - /* 71 */ "cmd ::= TRIM DATABASE db_name speed_opt", - /* 72 */ "cmd ::= COMPACT DATABASE db_name start_opt end_opt", - /* 73 */ "not_exists_opt ::= IF NOT EXISTS", - /* 74 */ "not_exists_opt ::=", - /* 75 */ "exists_opt ::= IF EXISTS", - /* 76 */ "exists_opt ::=", - /* 77 */ "db_options ::=", - /* 78 */ "db_options ::= db_options BUFFER NK_INTEGER", - /* 79 */ "db_options ::= db_options CACHEMODEL NK_STRING", - /* 80 */ "db_options ::= db_options CACHESIZE NK_INTEGER", - /* 81 */ "db_options ::= db_options COMP NK_INTEGER", - /* 82 */ "db_options ::= db_options DURATION NK_INTEGER", - /* 83 */ "db_options ::= db_options DURATION NK_VARIABLE", - /* 84 */ "db_options ::= db_options MAXROWS NK_INTEGER", - /* 85 */ "db_options ::= db_options MINROWS NK_INTEGER", - /* 86 */ "db_options ::= db_options KEEP integer_list", - /* 87 */ "db_options ::= db_options KEEP variable_list", - /* 88 */ "db_options ::= db_options PAGES NK_INTEGER", - /* 89 */ "db_options ::= db_options PAGESIZE NK_INTEGER", - /* 90 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", - /* 91 */ "db_options ::= db_options PRECISION NK_STRING", - /* 92 */ "db_options ::= db_options REPLICA NK_INTEGER", - /* 93 */ "db_options ::= db_options VGROUPS NK_INTEGER", - /* 94 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", - /* 95 */ "db_options ::= db_options RETENTIONS retention_list", - /* 96 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", - /* 97 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", - /* 98 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", - /* 99 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", - /* 100 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 101 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", - /* 102 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 103 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", - /* 104 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", - /* 105 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", - /* 106 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", - /* 107 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", - /* 108 */ "alter_db_options ::= alter_db_option", - /* 109 */ "alter_db_options ::= alter_db_options alter_db_option", - /* 110 */ "alter_db_option ::= BUFFER NK_INTEGER", - /* 111 */ "alter_db_option ::= CACHEMODEL NK_STRING", - /* 112 */ "alter_db_option ::= CACHESIZE NK_INTEGER", - /* 113 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", - /* 114 */ "alter_db_option ::= KEEP integer_list", - /* 115 */ "alter_db_option ::= KEEP variable_list", - /* 116 */ "alter_db_option ::= PAGES NK_INTEGER", - /* 117 */ "alter_db_option ::= REPLICA NK_INTEGER", - /* 118 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", - /* 119 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", - /* 120 */ "alter_db_option ::= MINROWS NK_INTEGER", - /* 121 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER", - /* 122 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", - /* 123 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER", - /* 124 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", - /* 125 */ "integer_list ::= NK_INTEGER", - /* 126 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", - /* 127 */ "variable_list ::= NK_VARIABLE", - /* 128 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", - /* 129 */ "retention_list ::= retention", - /* 130 */ "retention_list ::= retention_list NK_COMMA retention", - /* 131 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", - /* 132 */ "speed_opt ::=", - /* 133 */ "speed_opt ::= MAX_SPEED NK_INTEGER", - /* 134 */ "start_opt ::=", - /* 135 */ "start_opt ::= START WITH NK_INTEGER", - /* 136 */ "start_opt ::= START WITH NK_STRING", - /* 137 */ "start_opt ::= START WITH TIMESTAMP NK_STRING", - /* 138 */ "end_opt ::=", - /* 139 */ "end_opt ::= END WITH NK_INTEGER", - /* 140 */ "end_opt ::= END WITH NK_STRING", - /* 141 */ "end_opt ::= END WITH TIMESTAMP NK_STRING", - /* 142 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", - /* 143 */ "cmd ::= CREATE TABLE multi_create_clause", - /* 144 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", - /* 145 */ "cmd ::= DROP TABLE multi_drop_clause", - /* 146 */ "cmd ::= DROP STABLE exists_opt full_table_name", - /* 147 */ "cmd ::= ALTER TABLE alter_table_clause", - /* 148 */ "cmd ::= ALTER STABLE alter_table_clause", - /* 149 */ "alter_table_clause ::= full_table_name alter_table_options", - /* 150 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", - /* 151 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", - /* 152 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", - /* 153 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", - /* 154 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", - /* 155 */ "alter_table_clause ::= full_table_name DROP TAG column_name", - /* 156 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", - /* 157 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", - /* 158 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", - /* 159 */ "multi_create_clause ::= create_subtable_clause", - /* 160 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", - /* 161 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", - /* 162 */ "multi_drop_clause ::= drop_table_clause", - /* 163 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", - /* 164 */ "drop_table_clause ::= exists_opt full_table_name", - /* 165 */ "specific_cols_opt ::=", - /* 166 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", - /* 167 */ "full_table_name ::= table_name", - /* 168 */ "full_table_name ::= db_name NK_DOT table_name", - /* 169 */ "column_def_list ::= column_def", - /* 170 */ "column_def_list ::= column_def_list NK_COMMA column_def", - /* 171 */ "column_def ::= column_name type_name", - /* 172 */ "column_def ::= column_name type_name COMMENT NK_STRING", - /* 173 */ "type_name ::= BOOL", - /* 174 */ "type_name ::= TINYINT", - /* 175 */ "type_name ::= SMALLINT", - /* 176 */ "type_name ::= INT", - /* 177 */ "type_name ::= INTEGER", - /* 178 */ "type_name ::= BIGINT", - /* 179 */ "type_name ::= FLOAT", - /* 180 */ "type_name ::= DOUBLE", - /* 181 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", - /* 182 */ "type_name ::= TIMESTAMP", - /* 183 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", - /* 184 */ "type_name ::= TINYINT UNSIGNED", - /* 185 */ "type_name ::= SMALLINT UNSIGNED", - /* 186 */ "type_name ::= INT UNSIGNED", - /* 187 */ "type_name ::= BIGINT UNSIGNED", - /* 188 */ "type_name ::= JSON", - /* 189 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", - /* 190 */ "type_name ::= MEDIUMBLOB", - /* 191 */ "type_name ::= BLOB", - /* 192 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", - /* 193 */ "type_name ::= DECIMAL", - /* 194 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", - /* 195 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", - /* 196 */ "tags_def_opt ::=", - /* 197 */ "tags_def_opt ::= tags_def", - /* 198 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", - /* 199 */ "table_options ::=", - /* 200 */ "table_options ::= table_options COMMENT NK_STRING", - /* 201 */ "table_options ::= table_options MAX_DELAY duration_list", - /* 202 */ "table_options ::= table_options WATERMARK duration_list", - /* 203 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", - /* 204 */ "table_options ::= table_options TTL NK_INTEGER", - /* 205 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", - /* 206 */ "table_options ::= table_options DELETE_MARK duration_list", - /* 207 */ "alter_table_options ::= alter_table_option", - /* 208 */ "alter_table_options ::= alter_table_options alter_table_option", - /* 209 */ "alter_table_option ::= COMMENT NK_STRING", - /* 210 */ "alter_table_option ::= TTL NK_INTEGER", - /* 211 */ "duration_list ::= duration_literal", - /* 212 */ "duration_list ::= duration_list NK_COMMA duration_literal", - /* 213 */ "rollup_func_list ::= rollup_func_name", - /* 214 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", - /* 215 */ "rollup_func_name ::= function_name", - /* 216 */ "rollup_func_name ::= FIRST", - /* 217 */ "rollup_func_name ::= LAST", - /* 218 */ "col_name_list ::= col_name", - /* 219 */ "col_name_list ::= col_name_list NK_COMMA col_name", - /* 220 */ "col_name ::= column_name", - /* 221 */ "cmd ::= SHOW DNODES", - /* 222 */ "cmd ::= SHOW USERS", - /* 223 */ "cmd ::= SHOW USER PRIVILEGES", - /* 224 */ "cmd ::= SHOW DATABASES", - /* 225 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", - /* 226 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", - /* 227 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", - /* 228 */ "cmd ::= SHOW MNODES", - /* 229 */ "cmd ::= SHOW QNODES", - /* 230 */ "cmd ::= SHOW FUNCTIONS", - /* 231 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", - /* 232 */ "cmd ::= SHOW STREAMS", - /* 233 */ "cmd ::= SHOW ACCOUNTS", - /* 234 */ "cmd ::= SHOW APPS", - /* 235 */ "cmd ::= SHOW CONNECTIONS", - /* 236 */ "cmd ::= SHOW LICENCES", - /* 237 */ "cmd ::= SHOW GRANTS", - /* 238 */ "cmd ::= SHOW CREATE DATABASE db_name", - /* 239 */ "cmd ::= SHOW CREATE TABLE full_table_name", - /* 240 */ "cmd ::= SHOW CREATE STABLE full_table_name", - /* 241 */ "cmd ::= SHOW QUERIES", - /* 242 */ "cmd ::= SHOW SCORES", - /* 243 */ "cmd ::= SHOW TOPICS", - /* 244 */ "cmd ::= SHOW VARIABLES", - /* 245 */ "cmd ::= SHOW CLUSTER VARIABLES", - /* 246 */ "cmd ::= SHOW LOCAL VARIABLES", - /* 247 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", - /* 248 */ "cmd ::= SHOW BNODES", - /* 249 */ "cmd ::= SHOW SNODES", - /* 250 */ "cmd ::= SHOW CLUSTER", - /* 251 */ "cmd ::= SHOW TRANSACTIONS", - /* 252 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", - /* 253 */ "cmd ::= SHOW CONSUMERS", - /* 254 */ "cmd ::= SHOW SUBSCRIPTIONS", - /* 255 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", - /* 256 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", - /* 257 */ "cmd ::= SHOW VNODES NK_INTEGER", - /* 258 */ "cmd ::= SHOW VNODES NK_STRING", - /* 259 */ "cmd ::= SHOW db_name_cond_opt ALIVE", - /* 260 */ "cmd ::= SHOW CLUSTER ALIVE", - /* 261 */ "db_name_cond_opt ::=", - /* 262 */ "db_name_cond_opt ::= db_name NK_DOT", - /* 263 */ "like_pattern_opt ::=", - /* 264 */ "like_pattern_opt ::= LIKE NK_STRING", - /* 265 */ "table_name_cond ::= table_name", - /* 266 */ "from_db_opt ::=", - /* 267 */ "from_db_opt ::= FROM db_name", - /* 268 */ "tag_list_opt ::=", - /* 269 */ "tag_list_opt ::= tag_item", - /* 270 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", - /* 271 */ "tag_item ::= TBNAME", - /* 272 */ "tag_item ::= QTAGS", - /* 273 */ "tag_item ::= column_name", - /* 274 */ "tag_item ::= column_name column_alias", - /* 275 */ "tag_item ::= column_name AS column_alias", - /* 276 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", - /* 277 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", - /* 278 */ "cmd ::= DROP INDEX exists_opt full_index_name", - /* 279 */ "full_index_name ::= index_name", - /* 280 */ "full_index_name ::= db_name NK_DOT index_name", - /* 281 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", - /* 282 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", - /* 283 */ "func_list ::= func", - /* 284 */ "func_list ::= func_list NK_COMMA func", - /* 285 */ "func ::= sma_func_name NK_LP expression_list NK_RP", - /* 286 */ "sma_func_name ::= function_name", - /* 287 */ "sma_func_name ::= COUNT", - /* 288 */ "sma_func_name ::= FIRST", - /* 289 */ "sma_func_name ::= LAST", - /* 290 */ "sma_func_name ::= LAST_ROW", - /* 291 */ "sma_stream_opt ::=", - /* 292 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", - /* 293 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", - /* 294 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", - /* 295 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", - /* 296 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", - /* 297 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", - /* 298 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", - /* 299 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", - /* 300 */ "cmd ::= DROP TOPIC exists_opt topic_name", - /* 301 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", - /* 302 */ "cmd ::= DESC full_table_name", - /* 303 */ "cmd ::= DESCRIBE full_table_name", - /* 304 */ "cmd ::= RESET QUERY CACHE", - /* 305 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", - /* 306 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", - /* 307 */ "analyze_opt ::=", - /* 308 */ "analyze_opt ::= ANALYZE", - /* 309 */ "explain_options ::=", - /* 310 */ "explain_options ::= explain_options VERBOSE NK_BOOL", - /* 311 */ "explain_options ::= explain_options RATIO NK_FLOAT", - /* 312 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", - /* 313 */ "cmd ::= DROP FUNCTION exists_opt function_name", - /* 314 */ "agg_func_opt ::=", - /* 315 */ "agg_func_opt ::= AGGREGATE", - /* 316 */ "bufsize_opt ::=", - /* 317 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", - /* 318 */ "language_opt ::=", - /* 319 */ "language_opt ::= LANGUAGE NK_STRING", - /* 320 */ "or_replace_opt ::=", - /* 321 */ "or_replace_opt ::= OR REPLACE", - /* 322 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", - /* 323 */ "cmd ::= DROP STREAM exists_opt stream_name", - /* 324 */ "col_list_opt ::=", - /* 325 */ "col_list_opt ::= NK_LP col_name_list NK_RP", - /* 326 */ "tag_def_or_ref_opt ::=", - /* 327 */ "tag_def_or_ref_opt ::= tags_def", - /* 328 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", - /* 329 */ "stream_options ::=", - /* 330 */ "stream_options ::= stream_options TRIGGER AT_ONCE", - /* 331 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", - /* 332 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", - /* 333 */ "stream_options ::= stream_options WATERMARK duration_literal", - /* 334 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", - /* 335 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", - /* 336 */ "stream_options ::= stream_options DELETE_MARK duration_literal", - /* 337 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", - /* 338 */ "subtable_opt ::=", - /* 339 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", - /* 340 */ "cmd ::= KILL CONNECTION NK_INTEGER", - /* 341 */ "cmd ::= KILL QUERY NK_STRING", - /* 342 */ "cmd ::= KILL TRANSACTION NK_INTEGER", - /* 343 */ "cmd ::= BALANCE VGROUP", - /* 344 */ "cmd ::= BALANCE VGROUP LEADER", - /* 345 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", - /* 346 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", - /* 347 */ "cmd ::= SPLIT VGROUP NK_INTEGER", - /* 348 */ "dnode_list ::= DNODE NK_INTEGER", - /* 349 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", - /* 350 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", - /* 351 */ "cmd ::= query_or_subquery", - /* 352 */ "cmd ::= insert_query", - /* 353 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", - /* 354 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", - /* 355 */ "literal ::= NK_INTEGER", - /* 356 */ "literal ::= NK_FLOAT", - /* 357 */ "literal ::= NK_STRING", - /* 358 */ "literal ::= NK_BOOL", - /* 359 */ "literal ::= TIMESTAMP NK_STRING", - /* 360 */ "literal ::= duration_literal", - /* 361 */ "literal ::= NULL", - /* 362 */ "literal ::= NK_QUESTION", - /* 363 */ "duration_literal ::= NK_VARIABLE", - /* 364 */ "signed ::= NK_INTEGER", - /* 365 */ "signed ::= NK_PLUS NK_INTEGER", - /* 366 */ "signed ::= NK_MINUS NK_INTEGER", - /* 367 */ "signed ::= NK_FLOAT", - /* 368 */ "signed ::= NK_PLUS NK_FLOAT", - /* 369 */ "signed ::= NK_MINUS NK_FLOAT", - /* 370 */ "signed_literal ::= signed", - /* 371 */ "signed_literal ::= NK_STRING", - /* 372 */ "signed_literal ::= NK_BOOL", - /* 373 */ "signed_literal ::= TIMESTAMP NK_STRING", - /* 374 */ "signed_literal ::= duration_literal", - /* 375 */ "signed_literal ::= NULL", - /* 376 */ "signed_literal ::= literal_func", - /* 377 */ "signed_literal ::= NK_QUESTION", - /* 378 */ "literal_list ::= signed_literal", - /* 379 */ "literal_list ::= literal_list NK_COMMA signed_literal", - /* 380 */ "db_name ::= NK_ID", - /* 381 */ "table_name ::= NK_ID", - /* 382 */ "column_name ::= NK_ID", - /* 383 */ "function_name ::= NK_ID", - /* 384 */ "table_alias ::= NK_ID", - /* 385 */ "column_alias ::= NK_ID", - /* 386 */ "user_name ::= NK_ID", - /* 387 */ "topic_name ::= NK_ID", - /* 388 */ "stream_name ::= NK_ID", - /* 389 */ "cgroup_name ::= NK_ID", - /* 390 */ "index_name ::= NK_ID", - /* 391 */ "expr_or_subquery ::= expression", - /* 392 */ "expression ::= literal", - /* 393 */ "expression ::= pseudo_column", - /* 394 */ "expression ::= column_reference", - /* 395 */ "expression ::= function_expression", - /* 396 */ "expression ::= case_when_expression", - /* 397 */ "expression ::= NK_LP expression NK_RP", - /* 398 */ "expression ::= NK_PLUS expr_or_subquery", - /* 399 */ "expression ::= NK_MINUS expr_or_subquery", - /* 400 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", - /* 401 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", - /* 402 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", - /* 403 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", - /* 404 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", - /* 405 */ "expression ::= column_reference NK_ARROW NK_STRING", - /* 406 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", - /* 407 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", - /* 408 */ "expression_list ::= expr_or_subquery", - /* 409 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", - /* 410 */ "column_reference ::= column_name", - /* 411 */ "column_reference ::= table_name NK_DOT column_name", - /* 412 */ "pseudo_column ::= ROWTS", - /* 413 */ "pseudo_column ::= TBNAME", - /* 414 */ "pseudo_column ::= table_name NK_DOT TBNAME", - /* 415 */ "pseudo_column ::= QSTART", - /* 416 */ "pseudo_column ::= QEND", - /* 417 */ "pseudo_column ::= QDURATION", - /* 418 */ "pseudo_column ::= WSTART", - /* 419 */ "pseudo_column ::= WEND", - /* 420 */ "pseudo_column ::= WDURATION", - /* 421 */ "pseudo_column ::= IROWTS", - /* 422 */ "pseudo_column ::= ISFILLED", - /* 423 */ "pseudo_column ::= QTAGS", - /* 424 */ "function_expression ::= function_name NK_LP expression_list NK_RP", - /* 425 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", - /* 426 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", - /* 427 */ "function_expression ::= literal_func", - /* 428 */ "literal_func ::= noarg_func NK_LP NK_RP", - /* 429 */ "literal_func ::= NOW", - /* 430 */ "noarg_func ::= NOW", - /* 431 */ "noarg_func ::= TODAY", - /* 432 */ "noarg_func ::= TIMEZONE", - /* 433 */ "noarg_func ::= DATABASE", - /* 434 */ "noarg_func ::= CLIENT_VERSION", - /* 435 */ "noarg_func ::= SERVER_VERSION", - /* 436 */ "noarg_func ::= SERVER_STATUS", - /* 437 */ "noarg_func ::= CURRENT_USER", - /* 438 */ "noarg_func ::= USER", - /* 439 */ "star_func ::= COUNT", - /* 440 */ "star_func ::= FIRST", - /* 441 */ "star_func ::= LAST", - /* 442 */ "star_func ::= LAST_ROW", - /* 443 */ "star_func_para_list ::= NK_STAR", - /* 444 */ "star_func_para_list ::= other_para_list", - /* 445 */ "other_para_list ::= star_func_para", - /* 446 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", - /* 447 */ "star_func_para ::= expr_or_subquery", - /* 448 */ "star_func_para ::= table_name NK_DOT NK_STAR", - /* 449 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", - /* 450 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", - /* 451 */ "when_then_list ::= when_then_expr", - /* 452 */ "when_then_list ::= when_then_list when_then_expr", - /* 453 */ "when_then_expr ::= WHEN common_expression THEN common_expression", - /* 454 */ "case_when_else_opt ::=", - /* 455 */ "case_when_else_opt ::= ELSE common_expression", - /* 456 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", - /* 457 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", - /* 458 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", - /* 459 */ "predicate ::= expr_or_subquery IS NULL", - /* 460 */ "predicate ::= expr_or_subquery IS NOT NULL", - /* 461 */ "predicate ::= expr_or_subquery in_op in_predicate_value", - /* 462 */ "compare_op ::= NK_LT", - /* 463 */ "compare_op ::= NK_GT", - /* 464 */ "compare_op ::= NK_LE", - /* 465 */ "compare_op ::= NK_GE", - /* 466 */ "compare_op ::= NK_NE", - /* 467 */ "compare_op ::= NK_EQ", - /* 468 */ "compare_op ::= LIKE", - /* 469 */ "compare_op ::= NOT LIKE", - /* 470 */ "compare_op ::= MATCH", - /* 471 */ "compare_op ::= NMATCH", - /* 472 */ "compare_op ::= CONTAINS", - /* 473 */ "in_op ::= IN", - /* 474 */ "in_op ::= NOT IN", - /* 475 */ "in_predicate_value ::= NK_LP literal_list NK_RP", - /* 476 */ "boolean_value_expression ::= boolean_primary", - /* 477 */ "boolean_value_expression ::= NOT boolean_primary", - /* 478 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", - /* 479 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", - /* 480 */ "boolean_primary ::= predicate", - /* 481 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", - /* 482 */ "common_expression ::= expr_or_subquery", - /* 483 */ "common_expression ::= boolean_value_expression", - /* 484 */ "from_clause_opt ::=", - /* 485 */ "from_clause_opt ::= FROM table_reference_list", - /* 486 */ "table_reference_list ::= table_reference", - /* 487 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", - /* 488 */ "table_reference ::= table_primary", - /* 489 */ "table_reference ::= joined_table", - /* 490 */ "table_primary ::= table_name alias_opt", - /* 491 */ "table_primary ::= db_name NK_DOT table_name alias_opt", - /* 492 */ "table_primary ::= subquery alias_opt", - /* 493 */ "table_primary ::= parenthesized_joined_table", - /* 494 */ "alias_opt ::=", - /* 495 */ "alias_opt ::= table_alias", - /* 496 */ "alias_opt ::= AS table_alias", - /* 497 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", - /* 498 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", - /* 499 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", - /* 500 */ "join_type ::=", - /* 501 */ "join_type ::= INNER", - /* 502 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", - /* 503 */ "set_quantifier_opt ::=", - /* 504 */ "set_quantifier_opt ::= DISTINCT", - /* 505 */ "set_quantifier_opt ::= ALL", - /* 506 */ "select_list ::= select_item", - /* 507 */ "select_list ::= select_list NK_COMMA select_item", - /* 508 */ "select_item ::= NK_STAR", - /* 509 */ "select_item ::= common_expression", - /* 510 */ "select_item ::= common_expression column_alias", - /* 511 */ "select_item ::= common_expression AS column_alias", - /* 512 */ "select_item ::= table_name NK_DOT NK_STAR", - /* 513 */ "where_clause_opt ::=", - /* 514 */ "where_clause_opt ::= WHERE search_condition", - /* 515 */ "partition_by_clause_opt ::=", - /* 516 */ "partition_by_clause_opt ::= PARTITION BY partition_list", - /* 517 */ "partition_list ::= partition_item", - /* 518 */ "partition_list ::= partition_list NK_COMMA partition_item", - /* 519 */ "partition_item ::= expr_or_subquery", - /* 520 */ "partition_item ::= expr_or_subquery column_alias", - /* 521 */ "partition_item ::= expr_or_subquery AS column_alias", - /* 522 */ "twindow_clause_opt ::=", - /* 523 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", - /* 524 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", - /* 525 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", - /* 526 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", - /* 527 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", - /* 528 */ "sliding_opt ::=", - /* 529 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", - /* 530 */ "fill_opt ::=", - /* 531 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", - /* 532 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP", - /* 533 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP", - /* 534 */ "fill_mode ::= NONE", - /* 535 */ "fill_mode ::= PREV", - /* 536 */ "fill_mode ::= NULL", - /* 537 */ "fill_mode ::= NULL_F", - /* 538 */ "fill_mode ::= LINEAR", - /* 539 */ "fill_mode ::= NEXT", - /* 540 */ "group_by_clause_opt ::=", - /* 541 */ "group_by_clause_opt ::= GROUP BY group_by_list", - /* 542 */ "group_by_list ::= expr_or_subquery", - /* 543 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", - /* 544 */ "having_clause_opt ::=", - /* 545 */ "having_clause_opt ::= HAVING search_condition", - /* 546 */ "range_opt ::=", - /* 547 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", - /* 548 */ "every_opt ::=", - /* 549 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", - /* 550 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", - /* 551 */ "query_simple ::= query_specification", - /* 552 */ "query_simple ::= union_query_expression", - /* 553 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", - /* 554 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", - /* 555 */ "query_simple_or_subquery ::= query_simple", - /* 556 */ "query_simple_or_subquery ::= subquery", - /* 557 */ "query_or_subquery ::= query_expression", - /* 558 */ "query_or_subquery ::= subquery", - /* 559 */ "order_by_clause_opt ::=", - /* 560 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", - /* 561 */ "slimit_clause_opt ::=", - /* 562 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", - /* 563 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", - /* 564 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 565 */ "limit_clause_opt ::=", - /* 566 */ "limit_clause_opt ::= LIMIT NK_INTEGER", - /* 567 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", - /* 568 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", - /* 569 */ "subquery ::= NK_LP query_expression NK_RP", - /* 570 */ "subquery ::= NK_LP subquery NK_RP", - /* 571 */ "search_condition ::= common_expression", - /* 572 */ "sort_specification_list ::= sort_specification", - /* 573 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", - /* 574 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", - /* 575 */ "ordering_specification_opt ::=", - /* 576 */ "ordering_specification_opt ::= ASC", - /* 577 */ "ordering_specification_opt ::= DESC", - /* 578 */ "null_ordering_opt ::=", - /* 579 */ "null_ordering_opt ::= NULLS FIRST", - /* 580 */ "null_ordering_opt ::= NULLS LAST", + /* 42 */ "priv_level ::= db_name NK_DOT table_name", + /* 43 */ "priv_level ::= topic_name", + /* 44 */ "with_opt ::=", + /* 45 */ "with_opt ::= WITH search_condition", + /* 46 */ "cmd ::= CREATE DNODE dnode_endpoint", + /* 47 */ "cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER", + /* 48 */ "cmd ::= DROP DNODE NK_INTEGER force_opt", + /* 49 */ "cmd ::= DROP DNODE dnode_endpoint force_opt", + /* 50 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING", + /* 51 */ "cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING", + /* 52 */ "cmd ::= ALTER ALL DNODES NK_STRING", + /* 53 */ "cmd ::= ALTER ALL DNODES NK_STRING NK_STRING", + /* 54 */ "dnode_endpoint ::= NK_STRING", + /* 55 */ "dnode_endpoint ::= NK_ID", + /* 56 */ "dnode_endpoint ::= NK_IPTOKEN", + /* 57 */ "force_opt ::=", + /* 58 */ "force_opt ::= FORCE", + /* 59 */ "cmd ::= ALTER LOCAL NK_STRING", + /* 60 */ "cmd ::= ALTER LOCAL NK_STRING NK_STRING", + /* 61 */ "cmd ::= CREATE QNODE ON DNODE NK_INTEGER", + /* 62 */ "cmd ::= DROP QNODE ON DNODE NK_INTEGER", + /* 63 */ "cmd ::= CREATE BNODE ON DNODE NK_INTEGER", + /* 64 */ "cmd ::= DROP BNODE ON DNODE NK_INTEGER", + /* 65 */ "cmd ::= CREATE SNODE ON DNODE NK_INTEGER", + /* 66 */ "cmd ::= DROP SNODE ON DNODE NK_INTEGER", + /* 67 */ "cmd ::= CREATE MNODE ON DNODE NK_INTEGER", + /* 68 */ "cmd ::= DROP MNODE ON DNODE NK_INTEGER", + /* 69 */ "cmd ::= CREATE DATABASE not_exists_opt db_name db_options", + /* 70 */ "cmd ::= DROP DATABASE exists_opt db_name", + /* 71 */ "cmd ::= USE db_name", + /* 72 */ "cmd ::= ALTER DATABASE db_name alter_db_options", + /* 73 */ "cmd ::= FLUSH DATABASE db_name", + /* 74 */ "cmd ::= TRIM DATABASE db_name speed_opt", + /* 75 */ "cmd ::= COMPACT DATABASE db_name start_opt end_opt", + /* 76 */ "not_exists_opt ::= IF NOT EXISTS", + /* 77 */ "not_exists_opt ::=", + /* 78 */ "exists_opt ::= IF EXISTS", + /* 79 */ "exists_opt ::=", + /* 80 */ "db_options ::=", + /* 81 */ "db_options ::= db_options BUFFER NK_INTEGER", + /* 82 */ "db_options ::= db_options CACHEMODEL NK_STRING", + /* 83 */ "db_options ::= db_options CACHESIZE NK_INTEGER", + /* 84 */ "db_options ::= db_options COMP NK_INTEGER", + /* 85 */ "db_options ::= db_options DURATION NK_INTEGER", + /* 86 */ "db_options ::= db_options DURATION NK_VARIABLE", + /* 87 */ "db_options ::= db_options MAXROWS NK_INTEGER", + /* 88 */ "db_options ::= db_options MINROWS NK_INTEGER", + /* 89 */ "db_options ::= db_options KEEP integer_list", + /* 90 */ "db_options ::= db_options KEEP variable_list", + /* 91 */ "db_options ::= db_options PAGES NK_INTEGER", + /* 92 */ "db_options ::= db_options PAGESIZE NK_INTEGER", + /* 93 */ "db_options ::= db_options TSDB_PAGESIZE NK_INTEGER", + /* 94 */ "db_options ::= db_options PRECISION NK_STRING", + /* 95 */ "db_options ::= db_options REPLICA NK_INTEGER", + /* 96 */ "db_options ::= db_options VGROUPS NK_INTEGER", + /* 97 */ "db_options ::= db_options SINGLE_STABLE NK_INTEGER", + /* 98 */ "db_options ::= db_options RETENTIONS retention_list", + /* 99 */ "db_options ::= db_options SCHEMALESS NK_INTEGER", + /* 100 */ "db_options ::= db_options WAL_LEVEL NK_INTEGER", + /* 101 */ "db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER", + /* 102 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER", + /* 103 */ "db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 104 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER", + /* 105 */ "db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 106 */ "db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER", + /* 107 */ "db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER", + /* 108 */ "db_options ::= db_options STT_TRIGGER NK_INTEGER", + /* 109 */ "db_options ::= db_options TABLE_PREFIX NK_INTEGER", + /* 110 */ "db_options ::= db_options TABLE_SUFFIX NK_INTEGER", + /* 111 */ "alter_db_options ::= alter_db_option", + /* 112 */ "alter_db_options ::= alter_db_options alter_db_option", + /* 113 */ "alter_db_option ::= BUFFER NK_INTEGER", + /* 114 */ "alter_db_option ::= CACHEMODEL NK_STRING", + /* 115 */ "alter_db_option ::= CACHESIZE NK_INTEGER", + /* 116 */ "alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER", + /* 117 */ "alter_db_option ::= KEEP integer_list", + /* 118 */ "alter_db_option ::= KEEP variable_list", + /* 119 */ "alter_db_option ::= PAGES NK_INTEGER", + /* 120 */ "alter_db_option ::= REPLICA NK_INTEGER", + /* 121 */ "alter_db_option ::= WAL_LEVEL NK_INTEGER", + /* 122 */ "alter_db_option ::= STT_TRIGGER NK_INTEGER", + /* 123 */ "alter_db_option ::= MINROWS NK_INTEGER", + /* 124 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER", + /* 125 */ "alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER", + /* 126 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER", + /* 127 */ "alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER", + /* 128 */ "integer_list ::= NK_INTEGER", + /* 129 */ "integer_list ::= integer_list NK_COMMA NK_INTEGER", + /* 130 */ "variable_list ::= NK_VARIABLE", + /* 131 */ "variable_list ::= variable_list NK_COMMA NK_VARIABLE", + /* 132 */ "retention_list ::= retention", + /* 133 */ "retention_list ::= retention_list NK_COMMA retention", + /* 134 */ "retention ::= NK_VARIABLE NK_COLON NK_VARIABLE", + /* 135 */ "speed_opt ::=", + /* 136 */ "speed_opt ::= MAX_SPEED NK_INTEGER", + /* 137 */ "start_opt ::=", + /* 138 */ "start_opt ::= START WITH NK_INTEGER", + /* 139 */ "start_opt ::= START WITH NK_STRING", + /* 140 */ "start_opt ::= START WITH TIMESTAMP NK_STRING", + /* 141 */ "end_opt ::=", + /* 142 */ "end_opt ::= END WITH NK_INTEGER", + /* 143 */ "end_opt ::= END WITH NK_STRING", + /* 144 */ "end_opt ::= END WITH TIMESTAMP NK_STRING", + /* 145 */ "cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options", + /* 146 */ "cmd ::= CREATE TABLE multi_create_clause", + /* 147 */ "cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options", + /* 148 */ "cmd ::= DROP TABLE multi_drop_clause", + /* 149 */ "cmd ::= DROP STABLE exists_opt full_table_name", + /* 150 */ "cmd ::= ALTER TABLE alter_table_clause", + /* 151 */ "cmd ::= ALTER STABLE alter_table_clause", + /* 152 */ "alter_table_clause ::= full_table_name alter_table_options", + /* 153 */ "alter_table_clause ::= full_table_name ADD COLUMN column_name type_name", + /* 154 */ "alter_table_clause ::= full_table_name DROP COLUMN column_name", + /* 155 */ "alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name", + /* 156 */ "alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name", + /* 157 */ "alter_table_clause ::= full_table_name ADD TAG column_name type_name", + /* 158 */ "alter_table_clause ::= full_table_name DROP TAG column_name", + /* 159 */ "alter_table_clause ::= full_table_name MODIFY TAG column_name type_name", + /* 160 */ "alter_table_clause ::= full_table_name RENAME TAG column_name column_name", + /* 161 */ "alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal", + /* 162 */ "multi_create_clause ::= create_subtable_clause", + /* 163 */ "multi_create_clause ::= multi_create_clause create_subtable_clause", + /* 164 */ "create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options", + /* 165 */ "multi_drop_clause ::= drop_table_clause", + /* 166 */ "multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause", + /* 167 */ "drop_table_clause ::= exists_opt full_table_name", + /* 168 */ "specific_cols_opt ::=", + /* 169 */ "specific_cols_opt ::= NK_LP col_name_list NK_RP", + /* 170 */ "full_table_name ::= table_name", + /* 171 */ "full_table_name ::= db_name NK_DOT table_name", + /* 172 */ "column_def_list ::= column_def", + /* 173 */ "column_def_list ::= column_def_list NK_COMMA column_def", + /* 174 */ "column_def ::= column_name type_name", + /* 175 */ "type_name ::= BOOL", + /* 176 */ "type_name ::= TINYINT", + /* 177 */ "type_name ::= SMALLINT", + /* 178 */ "type_name ::= INT", + /* 179 */ "type_name ::= INTEGER", + /* 180 */ "type_name ::= BIGINT", + /* 181 */ "type_name ::= FLOAT", + /* 182 */ "type_name ::= DOUBLE", + /* 183 */ "type_name ::= BINARY NK_LP NK_INTEGER NK_RP", + /* 184 */ "type_name ::= TIMESTAMP", + /* 185 */ "type_name ::= NCHAR NK_LP NK_INTEGER NK_RP", + /* 186 */ "type_name ::= TINYINT UNSIGNED", + /* 187 */ "type_name ::= SMALLINT UNSIGNED", + /* 188 */ "type_name ::= INT UNSIGNED", + /* 189 */ "type_name ::= BIGINT UNSIGNED", + /* 190 */ "type_name ::= JSON", + /* 191 */ "type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP", + /* 192 */ "type_name ::= MEDIUMBLOB", + /* 193 */ "type_name ::= BLOB", + /* 194 */ "type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP", + /* 195 */ "type_name ::= DECIMAL", + /* 196 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP", + /* 197 */ "type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP", + /* 198 */ "tags_def_opt ::=", + /* 199 */ "tags_def_opt ::= tags_def", + /* 200 */ "tags_def ::= TAGS NK_LP column_def_list NK_RP", + /* 201 */ "table_options ::=", + /* 202 */ "table_options ::= table_options COMMENT NK_STRING", + /* 203 */ "table_options ::= table_options MAX_DELAY duration_list", + /* 204 */ "table_options ::= table_options WATERMARK duration_list", + /* 205 */ "table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP", + /* 206 */ "table_options ::= table_options TTL NK_INTEGER", + /* 207 */ "table_options ::= table_options SMA NK_LP col_name_list NK_RP", + /* 208 */ "table_options ::= table_options DELETE_MARK duration_list", + /* 209 */ "alter_table_options ::= alter_table_option", + /* 210 */ "alter_table_options ::= alter_table_options alter_table_option", + /* 211 */ "alter_table_option ::= COMMENT NK_STRING", + /* 212 */ "alter_table_option ::= TTL NK_INTEGER", + /* 213 */ "duration_list ::= duration_literal", + /* 214 */ "duration_list ::= duration_list NK_COMMA duration_literal", + /* 215 */ "rollup_func_list ::= rollup_func_name", + /* 216 */ "rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name", + /* 217 */ "rollup_func_name ::= function_name", + /* 218 */ "rollup_func_name ::= FIRST", + /* 219 */ "rollup_func_name ::= LAST", + /* 220 */ "col_name_list ::= col_name", + /* 221 */ "col_name_list ::= col_name_list NK_COMMA col_name", + /* 222 */ "col_name ::= column_name", + /* 223 */ "cmd ::= SHOW DNODES", + /* 224 */ "cmd ::= SHOW USERS", + /* 225 */ "cmd ::= SHOW USER PRIVILEGES", + /* 226 */ "cmd ::= SHOW DATABASES", + /* 227 */ "cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt", + /* 228 */ "cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt", + /* 229 */ "cmd ::= SHOW db_name_cond_opt VGROUPS", + /* 230 */ "cmd ::= SHOW MNODES", + /* 231 */ "cmd ::= SHOW QNODES", + /* 232 */ "cmd ::= SHOW FUNCTIONS", + /* 233 */ "cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt", + /* 234 */ "cmd ::= SHOW STREAMS", + /* 235 */ "cmd ::= SHOW ACCOUNTS", + /* 236 */ "cmd ::= SHOW APPS", + /* 237 */ "cmd ::= SHOW CONNECTIONS", + /* 238 */ "cmd ::= SHOW LICENCES", + /* 239 */ "cmd ::= SHOW GRANTS", + /* 240 */ "cmd ::= SHOW CREATE DATABASE db_name", + /* 241 */ "cmd ::= SHOW CREATE TABLE full_table_name", + /* 242 */ "cmd ::= SHOW CREATE STABLE full_table_name", + /* 243 */ "cmd ::= SHOW QUERIES", + /* 244 */ "cmd ::= SHOW SCORES", + /* 245 */ "cmd ::= SHOW TOPICS", + /* 246 */ "cmd ::= SHOW VARIABLES", + /* 247 */ "cmd ::= SHOW CLUSTER VARIABLES", + /* 248 */ "cmd ::= SHOW LOCAL VARIABLES", + /* 249 */ "cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt", + /* 250 */ "cmd ::= SHOW BNODES", + /* 251 */ "cmd ::= SHOW SNODES", + /* 252 */ "cmd ::= SHOW CLUSTER", + /* 253 */ "cmd ::= SHOW TRANSACTIONS", + /* 254 */ "cmd ::= SHOW TABLE DISTRIBUTED full_table_name", + /* 255 */ "cmd ::= SHOW CONSUMERS", + /* 256 */ "cmd ::= SHOW SUBSCRIPTIONS", + /* 257 */ "cmd ::= SHOW TAGS FROM table_name_cond from_db_opt", + /* 258 */ "cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt", + /* 259 */ "cmd ::= SHOW VNODES NK_INTEGER", + /* 260 */ "cmd ::= SHOW VNODES NK_STRING", + /* 261 */ "cmd ::= SHOW db_name_cond_opt ALIVE", + /* 262 */ "cmd ::= SHOW CLUSTER ALIVE", + /* 263 */ "db_name_cond_opt ::=", + /* 264 */ "db_name_cond_opt ::= db_name NK_DOT", + /* 265 */ "like_pattern_opt ::=", + /* 266 */ "like_pattern_opt ::= LIKE NK_STRING", + /* 267 */ "table_name_cond ::= table_name", + /* 268 */ "from_db_opt ::=", + /* 269 */ "from_db_opt ::= FROM db_name", + /* 270 */ "tag_list_opt ::=", + /* 271 */ "tag_list_opt ::= tag_item", + /* 272 */ "tag_list_opt ::= tag_list_opt NK_COMMA tag_item", + /* 273 */ "tag_item ::= TBNAME", + /* 274 */ "tag_item ::= QTAGS", + /* 275 */ "tag_item ::= column_name", + /* 276 */ "tag_item ::= column_name column_alias", + /* 277 */ "tag_item ::= column_name AS column_alias", + /* 278 */ "cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options", + /* 279 */ "cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP", + /* 280 */ "cmd ::= DROP INDEX exists_opt full_index_name", + /* 281 */ "full_index_name ::= index_name", + /* 282 */ "full_index_name ::= db_name NK_DOT index_name", + /* 283 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt", + /* 284 */ "index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt", + /* 285 */ "func_list ::= func", + /* 286 */ "func_list ::= func_list NK_COMMA func", + /* 287 */ "func ::= sma_func_name NK_LP expression_list NK_RP", + /* 288 */ "sma_func_name ::= function_name", + /* 289 */ "sma_func_name ::= COUNT", + /* 290 */ "sma_func_name ::= FIRST", + /* 291 */ "sma_func_name ::= LAST", + /* 292 */ "sma_func_name ::= LAST_ROW", + /* 293 */ "sma_stream_opt ::=", + /* 294 */ "sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal", + /* 295 */ "sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal", + /* 296 */ "sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal", + /* 297 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery", + /* 298 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name", + /* 299 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name", + /* 300 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name", + /* 301 */ "cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name", + /* 302 */ "cmd ::= DROP TOPIC exists_opt topic_name", + /* 303 */ "cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name", + /* 304 */ "cmd ::= DESC full_table_name", + /* 305 */ "cmd ::= DESCRIBE full_table_name", + /* 306 */ "cmd ::= RESET QUERY CACHE", + /* 307 */ "cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery", + /* 308 */ "cmd ::= EXPLAIN analyze_opt explain_options insert_query", + /* 309 */ "analyze_opt ::=", + /* 310 */ "analyze_opt ::= ANALYZE", + /* 311 */ "explain_options ::=", + /* 312 */ "explain_options ::= explain_options VERBOSE NK_BOOL", + /* 313 */ "explain_options ::= explain_options RATIO NK_FLOAT", + /* 314 */ "cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt", + /* 315 */ "cmd ::= DROP FUNCTION exists_opt function_name", + /* 316 */ "agg_func_opt ::=", + /* 317 */ "agg_func_opt ::= AGGREGATE", + /* 318 */ "bufsize_opt ::=", + /* 319 */ "bufsize_opt ::= BUFSIZE NK_INTEGER", + /* 320 */ "language_opt ::=", + /* 321 */ "language_opt ::= LANGUAGE NK_STRING", + /* 322 */ "or_replace_opt ::=", + /* 323 */ "or_replace_opt ::= OR REPLACE", + /* 324 */ "cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery", + /* 325 */ "cmd ::= DROP STREAM exists_opt stream_name", + /* 326 */ "col_list_opt ::=", + /* 327 */ "col_list_opt ::= NK_LP col_name_list NK_RP", + /* 328 */ "tag_def_or_ref_opt ::=", + /* 329 */ "tag_def_or_ref_opt ::= tags_def", + /* 330 */ "tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP", + /* 331 */ "stream_options ::=", + /* 332 */ "stream_options ::= stream_options TRIGGER AT_ONCE", + /* 333 */ "stream_options ::= stream_options TRIGGER WINDOW_CLOSE", + /* 334 */ "stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal", + /* 335 */ "stream_options ::= stream_options WATERMARK duration_literal", + /* 336 */ "stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER", + /* 337 */ "stream_options ::= stream_options FILL_HISTORY NK_INTEGER", + /* 338 */ "stream_options ::= stream_options DELETE_MARK duration_literal", + /* 339 */ "stream_options ::= stream_options IGNORE UPDATE NK_INTEGER", + /* 340 */ "subtable_opt ::=", + /* 341 */ "subtable_opt ::= SUBTABLE NK_LP expression NK_RP", + /* 342 */ "cmd ::= KILL CONNECTION NK_INTEGER", + /* 343 */ "cmd ::= KILL QUERY NK_STRING", + /* 344 */ "cmd ::= KILL TRANSACTION NK_INTEGER", + /* 345 */ "cmd ::= BALANCE VGROUP", + /* 346 */ "cmd ::= BALANCE VGROUP LEADER", + /* 347 */ "cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER", + /* 348 */ "cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list", + /* 349 */ "cmd ::= SPLIT VGROUP NK_INTEGER", + /* 350 */ "dnode_list ::= DNODE NK_INTEGER", + /* 351 */ "dnode_list ::= dnode_list DNODE NK_INTEGER", + /* 352 */ "cmd ::= DELETE FROM full_table_name where_clause_opt", + /* 353 */ "cmd ::= query_or_subquery", + /* 354 */ "cmd ::= insert_query", + /* 355 */ "insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery", + /* 356 */ "insert_query ::= INSERT INTO full_table_name query_or_subquery", + /* 357 */ "literal ::= NK_INTEGER", + /* 358 */ "literal ::= NK_FLOAT", + /* 359 */ "literal ::= NK_STRING", + /* 360 */ "literal ::= NK_BOOL", + /* 361 */ "literal ::= TIMESTAMP NK_STRING", + /* 362 */ "literal ::= duration_literal", + /* 363 */ "literal ::= NULL", + /* 364 */ "literal ::= NK_QUESTION", + /* 365 */ "duration_literal ::= NK_VARIABLE", + /* 366 */ "signed ::= NK_INTEGER", + /* 367 */ "signed ::= NK_PLUS NK_INTEGER", + /* 368 */ "signed ::= NK_MINUS NK_INTEGER", + /* 369 */ "signed ::= NK_FLOAT", + /* 370 */ "signed ::= NK_PLUS NK_FLOAT", + /* 371 */ "signed ::= NK_MINUS NK_FLOAT", + /* 372 */ "signed_literal ::= signed", + /* 373 */ "signed_literal ::= NK_STRING", + /* 374 */ "signed_literal ::= NK_BOOL", + /* 375 */ "signed_literal ::= TIMESTAMP NK_STRING", + /* 376 */ "signed_literal ::= duration_literal", + /* 377 */ "signed_literal ::= NULL", + /* 378 */ "signed_literal ::= literal_func", + /* 379 */ "signed_literal ::= NK_QUESTION", + /* 380 */ "literal_list ::= signed_literal", + /* 381 */ "literal_list ::= literal_list NK_COMMA signed_literal", + /* 382 */ "db_name ::= NK_ID", + /* 383 */ "table_name ::= NK_ID", + /* 384 */ "column_name ::= NK_ID", + /* 385 */ "function_name ::= NK_ID", + /* 386 */ "table_alias ::= NK_ID", + /* 387 */ "column_alias ::= NK_ID", + /* 388 */ "user_name ::= NK_ID", + /* 389 */ "topic_name ::= NK_ID", + /* 390 */ "stream_name ::= NK_ID", + /* 391 */ "cgroup_name ::= NK_ID", + /* 392 */ "index_name ::= NK_ID", + /* 393 */ "expr_or_subquery ::= expression", + /* 394 */ "expression ::= literal", + /* 395 */ "expression ::= pseudo_column", + /* 396 */ "expression ::= column_reference", + /* 397 */ "expression ::= function_expression", + /* 398 */ "expression ::= case_when_expression", + /* 399 */ "expression ::= NK_LP expression NK_RP", + /* 400 */ "expression ::= NK_PLUS expr_or_subquery", + /* 401 */ "expression ::= NK_MINUS expr_or_subquery", + /* 402 */ "expression ::= expr_or_subquery NK_PLUS expr_or_subquery", + /* 403 */ "expression ::= expr_or_subquery NK_MINUS expr_or_subquery", + /* 404 */ "expression ::= expr_or_subquery NK_STAR expr_or_subquery", + /* 405 */ "expression ::= expr_or_subquery NK_SLASH expr_or_subquery", + /* 406 */ "expression ::= expr_or_subquery NK_REM expr_or_subquery", + /* 407 */ "expression ::= column_reference NK_ARROW NK_STRING", + /* 408 */ "expression ::= expr_or_subquery NK_BITAND expr_or_subquery", + /* 409 */ "expression ::= expr_or_subquery NK_BITOR expr_or_subquery", + /* 410 */ "expression_list ::= expr_or_subquery", + /* 411 */ "expression_list ::= expression_list NK_COMMA expr_or_subquery", + /* 412 */ "column_reference ::= column_name", + /* 413 */ "column_reference ::= table_name NK_DOT column_name", + /* 414 */ "pseudo_column ::= ROWTS", + /* 415 */ "pseudo_column ::= TBNAME", + /* 416 */ "pseudo_column ::= table_name NK_DOT TBNAME", + /* 417 */ "pseudo_column ::= QSTART", + /* 418 */ "pseudo_column ::= QEND", + /* 419 */ "pseudo_column ::= QDURATION", + /* 420 */ "pseudo_column ::= WSTART", + /* 421 */ "pseudo_column ::= WEND", + /* 422 */ "pseudo_column ::= WDURATION", + /* 423 */ "pseudo_column ::= IROWTS", + /* 424 */ "pseudo_column ::= ISFILLED", + /* 425 */ "pseudo_column ::= QTAGS", + /* 426 */ "function_expression ::= function_name NK_LP expression_list NK_RP", + /* 427 */ "function_expression ::= star_func NK_LP star_func_para_list NK_RP", + /* 428 */ "function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP", + /* 429 */ "function_expression ::= literal_func", + /* 430 */ "literal_func ::= noarg_func NK_LP NK_RP", + /* 431 */ "literal_func ::= NOW", + /* 432 */ "noarg_func ::= NOW", + /* 433 */ "noarg_func ::= TODAY", + /* 434 */ "noarg_func ::= TIMEZONE", + /* 435 */ "noarg_func ::= DATABASE", + /* 436 */ "noarg_func ::= CLIENT_VERSION", + /* 437 */ "noarg_func ::= SERVER_VERSION", + /* 438 */ "noarg_func ::= SERVER_STATUS", + /* 439 */ "noarg_func ::= CURRENT_USER", + /* 440 */ "noarg_func ::= USER", + /* 441 */ "star_func ::= COUNT", + /* 442 */ "star_func ::= FIRST", + /* 443 */ "star_func ::= LAST", + /* 444 */ "star_func ::= LAST_ROW", + /* 445 */ "star_func_para_list ::= NK_STAR", + /* 446 */ "star_func_para_list ::= other_para_list", + /* 447 */ "other_para_list ::= star_func_para", + /* 448 */ "other_para_list ::= other_para_list NK_COMMA star_func_para", + /* 449 */ "star_func_para ::= expr_or_subquery", + /* 450 */ "star_func_para ::= table_name NK_DOT NK_STAR", + /* 451 */ "case_when_expression ::= CASE when_then_list case_when_else_opt END", + /* 452 */ "case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END", + /* 453 */ "when_then_list ::= when_then_expr", + /* 454 */ "when_then_list ::= when_then_list when_then_expr", + /* 455 */ "when_then_expr ::= WHEN common_expression THEN common_expression", + /* 456 */ "case_when_else_opt ::=", + /* 457 */ "case_when_else_opt ::= ELSE common_expression", + /* 458 */ "predicate ::= expr_or_subquery compare_op expr_or_subquery", + /* 459 */ "predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery", + /* 460 */ "predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery", + /* 461 */ "predicate ::= expr_or_subquery IS NULL", + /* 462 */ "predicate ::= expr_or_subquery IS NOT NULL", + /* 463 */ "predicate ::= expr_or_subquery in_op in_predicate_value", + /* 464 */ "compare_op ::= NK_LT", + /* 465 */ "compare_op ::= NK_GT", + /* 466 */ "compare_op ::= NK_LE", + /* 467 */ "compare_op ::= NK_GE", + /* 468 */ "compare_op ::= NK_NE", + /* 469 */ "compare_op ::= NK_EQ", + /* 470 */ "compare_op ::= LIKE", + /* 471 */ "compare_op ::= NOT LIKE", + /* 472 */ "compare_op ::= MATCH", + /* 473 */ "compare_op ::= NMATCH", + /* 474 */ "compare_op ::= CONTAINS", + /* 475 */ "in_op ::= IN", + /* 476 */ "in_op ::= NOT IN", + /* 477 */ "in_predicate_value ::= NK_LP literal_list NK_RP", + /* 478 */ "boolean_value_expression ::= boolean_primary", + /* 479 */ "boolean_value_expression ::= NOT boolean_primary", + /* 480 */ "boolean_value_expression ::= boolean_value_expression OR boolean_value_expression", + /* 481 */ "boolean_value_expression ::= boolean_value_expression AND boolean_value_expression", + /* 482 */ "boolean_primary ::= predicate", + /* 483 */ "boolean_primary ::= NK_LP boolean_value_expression NK_RP", + /* 484 */ "common_expression ::= expr_or_subquery", + /* 485 */ "common_expression ::= boolean_value_expression", + /* 486 */ "from_clause_opt ::=", + /* 487 */ "from_clause_opt ::= FROM table_reference_list", + /* 488 */ "table_reference_list ::= table_reference", + /* 489 */ "table_reference_list ::= table_reference_list NK_COMMA table_reference", + /* 490 */ "table_reference ::= table_primary", + /* 491 */ "table_reference ::= joined_table", + /* 492 */ "table_primary ::= table_name alias_opt", + /* 493 */ "table_primary ::= db_name NK_DOT table_name alias_opt", + /* 494 */ "table_primary ::= subquery alias_opt", + /* 495 */ "table_primary ::= parenthesized_joined_table", + /* 496 */ "alias_opt ::=", + /* 497 */ "alias_opt ::= table_alias", + /* 498 */ "alias_opt ::= AS table_alias", + /* 499 */ "parenthesized_joined_table ::= NK_LP joined_table NK_RP", + /* 500 */ "parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP", + /* 501 */ "joined_table ::= table_reference join_type JOIN table_reference ON search_condition", + /* 502 */ "join_type ::=", + /* 503 */ "join_type ::= INNER", + /* 504 */ "query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt", + /* 505 */ "set_quantifier_opt ::=", + /* 506 */ "set_quantifier_opt ::= DISTINCT", + /* 507 */ "set_quantifier_opt ::= ALL", + /* 508 */ "select_list ::= select_item", + /* 509 */ "select_list ::= select_list NK_COMMA select_item", + /* 510 */ "select_item ::= NK_STAR", + /* 511 */ "select_item ::= common_expression", + /* 512 */ "select_item ::= common_expression column_alias", + /* 513 */ "select_item ::= common_expression AS column_alias", + /* 514 */ "select_item ::= table_name NK_DOT NK_STAR", + /* 515 */ "where_clause_opt ::=", + /* 516 */ "where_clause_opt ::= WHERE search_condition", + /* 517 */ "partition_by_clause_opt ::=", + /* 518 */ "partition_by_clause_opt ::= PARTITION BY partition_list", + /* 519 */ "partition_list ::= partition_item", + /* 520 */ "partition_list ::= partition_list NK_COMMA partition_item", + /* 521 */ "partition_item ::= expr_or_subquery", + /* 522 */ "partition_item ::= expr_or_subquery column_alias", + /* 523 */ "partition_item ::= expr_or_subquery AS column_alias", + /* 524 */ "twindow_clause_opt ::=", + /* 525 */ "twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP", + /* 526 */ "twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP", + /* 527 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt", + /* 528 */ "twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt", + /* 529 */ "twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition", + /* 530 */ "sliding_opt ::=", + /* 531 */ "sliding_opt ::= SLIDING NK_LP duration_literal NK_RP", + /* 532 */ "fill_opt ::=", + /* 533 */ "fill_opt ::= FILL NK_LP fill_mode NK_RP", + /* 534 */ "fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP", + /* 535 */ "fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP", + /* 536 */ "fill_mode ::= NONE", + /* 537 */ "fill_mode ::= PREV", + /* 538 */ "fill_mode ::= NULL", + /* 539 */ "fill_mode ::= NULL_F", + /* 540 */ "fill_mode ::= LINEAR", + /* 541 */ "fill_mode ::= NEXT", + /* 542 */ "group_by_clause_opt ::=", + /* 543 */ "group_by_clause_opt ::= GROUP BY group_by_list", + /* 544 */ "group_by_list ::= expr_or_subquery", + /* 545 */ "group_by_list ::= group_by_list NK_COMMA expr_or_subquery", + /* 546 */ "having_clause_opt ::=", + /* 547 */ "having_clause_opt ::= HAVING search_condition", + /* 548 */ "range_opt ::=", + /* 549 */ "range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP", + /* 550 */ "every_opt ::=", + /* 551 */ "every_opt ::= EVERY NK_LP duration_literal NK_RP", + /* 552 */ "query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt", + /* 553 */ "query_simple ::= query_specification", + /* 554 */ "query_simple ::= union_query_expression", + /* 555 */ "union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery", + /* 556 */ "union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery", + /* 557 */ "query_simple_or_subquery ::= query_simple", + /* 558 */ "query_simple_or_subquery ::= subquery", + /* 559 */ "query_or_subquery ::= query_expression", + /* 560 */ "query_or_subquery ::= subquery", + /* 561 */ "order_by_clause_opt ::=", + /* 562 */ "order_by_clause_opt ::= ORDER BY sort_specification_list", + /* 563 */ "slimit_clause_opt ::=", + /* 564 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER", + /* 565 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER", + /* 566 */ "slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 567 */ "limit_clause_opt ::=", + /* 568 */ "limit_clause_opt ::= LIMIT NK_INTEGER", + /* 569 */ "limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER", + /* 570 */ "limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER", + /* 571 */ "subquery ::= NK_LP query_expression NK_RP", + /* 572 */ "subquery ::= NK_LP subquery NK_RP", + /* 573 */ "search_condition ::= common_expression", + /* 574 */ "sort_specification_list ::= sort_specification", + /* 575 */ "sort_specification_list ::= sort_specification_list NK_COMMA sort_specification", + /* 576 */ "sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt", + /* 577 */ "ordering_specification_opt ::=", + /* 578 */ "ordering_specification_opt ::= ASC", + /* 579 */ "ordering_specification_opt ::= DESC", + /* 580 */ "null_ordering_opt ::=", + /* 581 */ "null_ordering_opt ::= NULLS FIRST", + /* 582 */ "null_ordering_opt ::= NULLS LAST", }; #endif /* NDEBUG */ @@ -2664,108 +2655,108 @@ static void yy_destructor( /* Default NON-TERMINAL Destructor */ case 330: /* cmd */ case 333: /* literal */ - case 346: /* db_options */ - case 348: /* alter_db_options */ - case 350: /* start_opt */ - case 351: /* end_opt */ - case 356: /* retention */ - case 357: /* full_table_name */ - case 360: /* table_options */ - case 364: /* alter_table_clause */ - case 365: /* alter_table_options */ - case 368: /* signed_literal */ - case 369: /* create_subtable_clause */ - case 372: /* drop_table_clause */ - case 375: /* column_def */ - case 379: /* duration_literal */ - case 380: /* rollup_func_name */ - case 382: /* col_name */ - case 383: /* db_name_cond_opt */ - case 384: /* like_pattern_opt */ - case 385: /* table_name_cond */ - case 386: /* from_db_opt */ - case 388: /* tag_item */ - case 390: /* full_index_name */ - case 391: /* index_options */ - case 394: /* sliding_opt */ - case 395: /* sma_stream_opt */ - case 396: /* func */ - case 398: /* query_or_subquery */ - case 401: /* explain_options */ - case 402: /* insert_query */ - case 408: /* stream_options */ - case 411: /* subtable_opt */ - case 412: /* expression */ - case 414: /* where_clause_opt */ - case 415: /* signed */ - case 416: /* literal_func */ - case 419: /* expr_or_subquery */ - case 420: /* pseudo_column */ - case 421: /* column_reference */ - case 422: /* function_expression */ - case 423: /* case_when_expression */ - case 428: /* star_func_para */ - case 430: /* case_when_else_opt */ - case 431: /* common_expression */ - case 432: /* when_then_expr */ - case 433: /* predicate */ - case 436: /* in_predicate_value */ - case 437: /* boolean_value_expression */ - case 438: /* boolean_primary */ - case 439: /* from_clause_opt */ - case 440: /* table_reference_list */ - case 441: /* table_reference */ - case 442: /* table_primary */ - case 443: /* joined_table */ - case 445: /* subquery */ - case 446: /* parenthesized_joined_table */ - case 448: /* search_condition */ - case 449: /* query_specification */ - case 453: /* range_opt */ - case 454: /* every_opt */ - case 455: /* fill_opt */ - case 456: /* twindow_clause_opt */ - case 458: /* having_clause_opt */ - case 459: /* select_item */ - case 461: /* partition_item */ - case 464: /* query_expression */ - case 465: /* query_simple */ - case 467: /* slimit_clause_opt */ - case 468: /* limit_clause_opt */ - case 469: /* union_query_expression */ - case 470: /* query_simple_or_subquery */ - case 472: /* sort_specification */ + case 339: /* with_opt */ + case 345: /* search_condition */ + case 349: /* db_options */ + case 351: /* alter_db_options */ + case 353: /* start_opt */ + case 354: /* end_opt */ + case 359: /* retention */ + case 360: /* full_table_name */ + case 363: /* table_options */ + case 367: /* alter_table_clause */ + case 368: /* alter_table_options */ + case 371: /* signed_literal */ + case 372: /* create_subtable_clause */ + case 375: /* drop_table_clause */ + case 377: /* column_def */ + case 381: /* duration_literal */ + case 382: /* rollup_func_name */ + case 384: /* col_name */ + case 385: /* db_name_cond_opt */ + case 386: /* like_pattern_opt */ + case 387: /* table_name_cond */ + case 388: /* from_db_opt */ + case 390: /* tag_item */ + case 392: /* full_index_name */ + case 393: /* index_options */ + case 396: /* sliding_opt */ + case 397: /* sma_stream_opt */ + case 398: /* func */ + case 400: /* query_or_subquery */ + case 403: /* explain_options */ + case 404: /* insert_query */ + case 410: /* stream_options */ + case 413: /* subtable_opt */ + case 414: /* expression */ + case 416: /* where_clause_opt */ + case 417: /* signed */ + case 418: /* literal_func */ + case 421: /* expr_or_subquery */ + case 422: /* pseudo_column */ + case 423: /* column_reference */ + case 424: /* function_expression */ + case 425: /* case_when_expression */ + case 430: /* star_func_para */ + case 432: /* case_when_else_opt */ + case 433: /* common_expression */ + case 434: /* when_then_expr */ + case 435: /* predicate */ + case 438: /* in_predicate_value */ + case 439: /* boolean_value_expression */ + case 440: /* boolean_primary */ + case 441: /* from_clause_opt */ + case 442: /* table_reference_list */ + case 443: /* table_reference */ + case 444: /* table_primary */ + case 445: /* joined_table */ + case 447: /* subquery */ + case 448: /* parenthesized_joined_table */ + case 450: /* query_specification */ + case 454: /* range_opt */ + case 455: /* every_opt */ + case 456: /* fill_opt */ + case 457: /* twindow_clause_opt */ + case 459: /* having_clause_opt */ + case 460: /* select_item */ + case 462: /* partition_item */ + case 465: /* query_expression */ + case 466: /* query_simple */ + case 468: /* slimit_clause_opt */ + case 469: /* limit_clause_opt */ + case 470: /* union_query_expression */ + case 471: /* query_simple_or_subquery */ + case 473: /* sort_specification */ { - nodesDestroyNode((yypminor->yy712)); + nodesDestroyNode((yypminor->yy448)); } break; case 331: /* account_options */ case 332: /* alter_account_options */ case 334: /* alter_account_option */ - case 349: /* speed_opt */ - case 405: /* bufsize_opt */ + case 352: /* speed_opt */ + case 407: /* bufsize_opt */ { } break; case 335: /* user_name */ - case 338: /* priv_level */ - case 341: /* db_name */ - case 342: /* topic_name */ - case 343: /* dnode_endpoint */ - case 366: /* column_name */ - case 374: /* table_name */ - case 381: /* function_name */ - case 389: /* column_alias */ - case 392: /* index_name */ - case 397: /* sma_func_name */ - case 399: /* cgroup_name */ - case 406: /* language_opt */ - case 407: /* stream_name */ - case 418: /* table_alias */ - case 424: /* star_func */ - case 426: /* noarg_func */ - case 444: /* alias_opt */ + case 342: /* db_name */ + case 343: /* table_name */ + case 344: /* topic_name */ + case 346: /* dnode_endpoint */ + case 369: /* column_name */ + case 383: /* function_name */ + case 391: /* column_alias */ + case 394: /* index_name */ + case 399: /* sma_func_name */ + case 401: /* cgroup_name */ + case 408: /* language_opt */ + case 409: /* stream_name */ + case 420: /* table_alias */ + case 426: /* star_func */ + case 428: /* noarg_func */ + case 446: /* alias_opt */ { } @@ -2776,89 +2767,94 @@ static void yy_destructor( } break; case 337: /* privileges */ - case 339: /* priv_type_list */ - case 340: /* priv_type */ + case 340: /* priv_type_list */ + case 341: /* priv_type */ +{ + +} + break; + case 338: /* priv_level */ { } break; - case 344: /* force_opt */ - case 345: /* not_exists_opt */ - case 347: /* exists_opt */ - case 400: /* analyze_opt */ - case 403: /* or_replace_opt */ - case 404: /* agg_func_opt */ - case 450: /* set_quantifier_opt */ + case 347: /* force_opt */ + case 348: /* not_exists_opt */ + case 350: /* exists_opt */ + case 402: /* analyze_opt */ + case 405: /* or_replace_opt */ + case 406: /* agg_func_opt */ + case 451: /* set_quantifier_opt */ { } break; - case 352: /* integer_list */ - case 353: /* variable_list */ - case 354: /* retention_list */ - case 358: /* column_def_list */ - case 359: /* tags_def_opt */ - case 361: /* multi_create_clause */ - case 362: /* tags_def */ - case 363: /* multi_drop_clause */ - case 370: /* specific_cols_opt */ - case 371: /* expression_list */ - case 373: /* col_name_list */ - case 376: /* duration_list */ - case 377: /* rollup_func_list */ - case 387: /* tag_list_opt */ - case 393: /* func_list */ - case 409: /* col_list_opt */ - case 410: /* tag_def_or_ref_opt */ - case 413: /* dnode_list */ - case 417: /* literal_list */ - case 425: /* star_func_para_list */ - case 427: /* other_para_list */ - case 429: /* when_then_list */ - case 451: /* select_list */ - case 452: /* partition_by_clause_opt */ - case 457: /* group_by_clause_opt */ - case 460: /* partition_list */ - case 463: /* group_by_list */ - case 466: /* order_by_clause_opt */ - case 471: /* sort_specification_list */ + case 355: /* integer_list */ + case 356: /* variable_list */ + case 357: /* retention_list */ + case 361: /* column_def_list */ + case 362: /* tags_def_opt */ + case 364: /* multi_create_clause */ + case 365: /* tags_def */ + case 366: /* multi_drop_clause */ + case 373: /* specific_cols_opt */ + case 374: /* expression_list */ + case 376: /* col_name_list */ + case 378: /* duration_list */ + case 379: /* rollup_func_list */ + case 389: /* tag_list_opt */ + case 395: /* func_list */ + case 411: /* col_list_opt */ + case 412: /* tag_def_or_ref_opt */ + case 415: /* dnode_list */ + case 419: /* literal_list */ + case 427: /* star_func_para_list */ + case 429: /* other_para_list */ + case 431: /* when_then_list */ + case 452: /* select_list */ + case 453: /* partition_by_clause_opt */ + case 458: /* group_by_clause_opt */ + case 461: /* partition_list */ + case 464: /* group_by_list */ + case 467: /* order_by_clause_opt */ + case 472: /* sort_specification_list */ { - nodesDestroyList((yypminor->yy274)); + nodesDestroyList((yypminor->yy432)); } break; - case 355: /* alter_db_option */ - case 378: /* alter_table_option */ + case 358: /* alter_db_option */ + case 380: /* alter_table_option */ { } break; - case 367: /* type_name */ + case 370: /* type_name */ { } break; - case 434: /* compare_op */ - case 435: /* in_op */ + case 436: /* compare_op */ + case 437: /* in_op */ { } break; - case 447: /* join_type */ + case 449: /* join_type */ { } break; - case 462: /* fill_mode */ + case 463: /* fill_mode */ { } break; - case 473: /* ordering_specification_opt */ + case 474: /* ordering_specification_opt */ { } break; - case 474: /* null_ordering_opt */ + case 475: /* null_ordering_opt */ { } @@ -3188,556 +3184,558 @@ static const struct { { 330, -3 }, /* (28) cmd ::= DROP USER user_name */ { 336, 0 }, /* (29) sysinfo_opt ::= */ { 336, -2 }, /* (30) sysinfo_opt ::= SYSINFO NK_INTEGER */ - { 330, -6 }, /* (31) cmd ::= GRANT privileges ON priv_level TO user_name */ - { 330, -6 }, /* (32) cmd ::= REVOKE privileges ON priv_level FROM user_name */ + { 330, -7 }, /* (31) cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ + { 330, -7 }, /* (32) cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ { 337, -1 }, /* (33) privileges ::= ALL */ { 337, -1 }, /* (34) privileges ::= priv_type_list */ { 337, -1 }, /* (35) privileges ::= SUBSCRIBE */ - { 339, -1 }, /* (36) priv_type_list ::= priv_type */ - { 339, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ - { 340, -1 }, /* (38) priv_type ::= READ */ - { 340, -1 }, /* (39) priv_type ::= WRITE */ + { 340, -1 }, /* (36) priv_type_list ::= priv_type */ + { 340, -3 }, /* (37) priv_type_list ::= priv_type_list NK_COMMA priv_type */ + { 341, -1 }, /* (38) priv_type ::= READ */ + { 341, -1 }, /* (39) priv_type ::= WRITE */ { 338, -3 }, /* (40) priv_level ::= NK_STAR NK_DOT NK_STAR */ { 338, -3 }, /* (41) priv_level ::= db_name NK_DOT NK_STAR */ - { 338, -1 }, /* (42) priv_level ::= topic_name */ - { 330, -3 }, /* (43) cmd ::= CREATE DNODE dnode_endpoint */ - { 330, -5 }, /* (44) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ - { 330, -4 }, /* (45) cmd ::= DROP DNODE NK_INTEGER force_opt */ - { 330, -4 }, /* (46) cmd ::= DROP DNODE dnode_endpoint force_opt */ - { 330, -4 }, /* (47) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ - { 330, -5 }, /* (48) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ - { 330, -4 }, /* (49) cmd ::= ALTER ALL DNODES NK_STRING */ - { 330, -5 }, /* (50) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ - { 343, -1 }, /* (51) dnode_endpoint ::= NK_STRING */ - { 343, -1 }, /* (52) dnode_endpoint ::= NK_ID */ - { 343, -1 }, /* (53) dnode_endpoint ::= NK_IPTOKEN */ - { 344, 0 }, /* (54) force_opt ::= */ - { 344, -1 }, /* (55) force_opt ::= FORCE */ - { 330, -3 }, /* (56) cmd ::= ALTER LOCAL NK_STRING */ - { 330, -4 }, /* (57) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ - { 330, -5 }, /* (58) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (59) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (60) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (61) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (62) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (63) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (64) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (65) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ - { 330, -5 }, /* (66) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ - { 330, -4 }, /* (67) cmd ::= DROP DATABASE exists_opt db_name */ - { 330, -2 }, /* (68) cmd ::= USE db_name */ - { 330, -4 }, /* (69) cmd ::= ALTER DATABASE db_name alter_db_options */ - { 330, -3 }, /* (70) cmd ::= FLUSH DATABASE db_name */ - { 330, -4 }, /* (71) cmd ::= TRIM DATABASE db_name speed_opt */ - { 330, -5 }, /* (72) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ - { 345, -3 }, /* (73) not_exists_opt ::= IF NOT EXISTS */ - { 345, 0 }, /* (74) not_exists_opt ::= */ - { 347, -2 }, /* (75) exists_opt ::= IF EXISTS */ - { 347, 0 }, /* (76) exists_opt ::= */ - { 346, 0 }, /* (77) db_options ::= */ - { 346, -3 }, /* (78) db_options ::= db_options BUFFER NK_INTEGER */ - { 346, -3 }, /* (79) db_options ::= db_options CACHEMODEL NK_STRING */ - { 346, -3 }, /* (80) db_options ::= db_options CACHESIZE NK_INTEGER */ - { 346, -3 }, /* (81) db_options ::= db_options COMP NK_INTEGER */ - { 346, -3 }, /* (82) db_options ::= db_options DURATION NK_INTEGER */ - { 346, -3 }, /* (83) db_options ::= db_options DURATION NK_VARIABLE */ - { 346, -3 }, /* (84) db_options ::= db_options MAXROWS NK_INTEGER */ - { 346, -3 }, /* (85) db_options ::= db_options MINROWS NK_INTEGER */ - { 346, -3 }, /* (86) db_options ::= db_options KEEP integer_list */ - { 346, -3 }, /* (87) db_options ::= db_options KEEP variable_list */ - { 346, -3 }, /* (88) db_options ::= db_options PAGES NK_INTEGER */ - { 346, -3 }, /* (89) db_options ::= db_options PAGESIZE NK_INTEGER */ - { 346, -3 }, /* (90) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ - { 346, -3 }, /* (91) db_options ::= db_options PRECISION NK_STRING */ - { 346, -3 }, /* (92) db_options ::= db_options REPLICA NK_INTEGER */ - { 346, -3 }, /* (93) db_options ::= db_options VGROUPS NK_INTEGER */ - { 346, -3 }, /* (94) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ - { 346, -3 }, /* (95) db_options ::= db_options RETENTIONS retention_list */ - { 346, -3 }, /* (96) db_options ::= db_options SCHEMALESS NK_INTEGER */ - { 346, -3 }, /* (97) db_options ::= db_options WAL_LEVEL NK_INTEGER */ - { 346, -3 }, /* (98) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ - { 346, -3 }, /* (99) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ - { 346, -4 }, /* (100) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 346, -3 }, /* (101) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ - { 346, -4 }, /* (102) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 346, -3 }, /* (103) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ - { 346, -3 }, /* (104) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ - { 346, -3 }, /* (105) db_options ::= db_options STT_TRIGGER NK_INTEGER */ - { 346, -3 }, /* (106) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ - { 346, -3 }, /* (107) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ - { 348, -1 }, /* (108) alter_db_options ::= alter_db_option */ - { 348, -2 }, /* (109) alter_db_options ::= alter_db_options alter_db_option */ - { 355, -2 }, /* (110) alter_db_option ::= BUFFER NK_INTEGER */ - { 355, -2 }, /* (111) alter_db_option ::= CACHEMODEL NK_STRING */ - { 355, -2 }, /* (112) alter_db_option ::= CACHESIZE NK_INTEGER */ - { 355, -2 }, /* (113) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ - { 355, -2 }, /* (114) alter_db_option ::= KEEP integer_list */ - { 355, -2 }, /* (115) alter_db_option ::= KEEP variable_list */ - { 355, -2 }, /* (116) alter_db_option ::= PAGES NK_INTEGER */ - { 355, -2 }, /* (117) alter_db_option ::= REPLICA NK_INTEGER */ - { 355, -2 }, /* (118) alter_db_option ::= WAL_LEVEL NK_INTEGER */ - { 355, -2 }, /* (119) alter_db_option ::= STT_TRIGGER NK_INTEGER */ - { 355, -2 }, /* (120) alter_db_option ::= MINROWS NK_INTEGER */ - { 355, -2 }, /* (121) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ - { 355, -3 }, /* (122) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ - { 355, -2 }, /* (123) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ - { 355, -3 }, /* (124) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ - { 352, -1 }, /* (125) integer_list ::= NK_INTEGER */ - { 352, -3 }, /* (126) integer_list ::= integer_list NK_COMMA NK_INTEGER */ - { 353, -1 }, /* (127) variable_list ::= NK_VARIABLE */ - { 353, -3 }, /* (128) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ - { 354, -1 }, /* (129) retention_list ::= retention */ - { 354, -3 }, /* (130) retention_list ::= retention_list NK_COMMA retention */ - { 356, -3 }, /* (131) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ - { 349, 0 }, /* (132) speed_opt ::= */ - { 349, -2 }, /* (133) speed_opt ::= MAX_SPEED NK_INTEGER */ - { 350, 0 }, /* (134) start_opt ::= */ - { 350, -3 }, /* (135) start_opt ::= START WITH NK_INTEGER */ - { 350, -3 }, /* (136) start_opt ::= START WITH NK_STRING */ - { 350, -4 }, /* (137) start_opt ::= START WITH TIMESTAMP NK_STRING */ - { 351, 0 }, /* (138) end_opt ::= */ - { 351, -3 }, /* (139) end_opt ::= END WITH NK_INTEGER */ - { 351, -3 }, /* (140) end_opt ::= END WITH NK_STRING */ - { 351, -4 }, /* (141) end_opt ::= END WITH TIMESTAMP NK_STRING */ - { 330, -9 }, /* (142) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - { 330, -3 }, /* (143) cmd ::= CREATE TABLE multi_create_clause */ - { 330, -9 }, /* (144) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ - { 330, -3 }, /* (145) cmd ::= DROP TABLE multi_drop_clause */ - { 330, -4 }, /* (146) cmd ::= DROP STABLE exists_opt full_table_name */ - { 330, -3 }, /* (147) cmd ::= ALTER TABLE alter_table_clause */ - { 330, -3 }, /* (148) cmd ::= ALTER STABLE alter_table_clause */ - { 364, -2 }, /* (149) alter_table_clause ::= full_table_name alter_table_options */ - { 364, -5 }, /* (150) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ - { 364, -4 }, /* (151) alter_table_clause ::= full_table_name DROP COLUMN column_name */ - { 364, -5 }, /* (152) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ - { 364, -5 }, /* (153) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ - { 364, -5 }, /* (154) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ - { 364, -4 }, /* (155) alter_table_clause ::= full_table_name DROP TAG column_name */ - { 364, -5 }, /* (156) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ - { 364, -5 }, /* (157) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ - { 364, -6 }, /* (158) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ - { 361, -1 }, /* (159) multi_create_clause ::= create_subtable_clause */ - { 361, -2 }, /* (160) multi_create_clause ::= multi_create_clause create_subtable_clause */ - { 369, -10 }, /* (161) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ - { 363, -1 }, /* (162) multi_drop_clause ::= drop_table_clause */ - { 363, -3 }, /* (163) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ - { 372, -2 }, /* (164) drop_table_clause ::= exists_opt full_table_name */ - { 370, 0 }, /* (165) specific_cols_opt ::= */ - { 370, -3 }, /* (166) specific_cols_opt ::= NK_LP col_name_list NK_RP */ - { 357, -1 }, /* (167) full_table_name ::= table_name */ - { 357, -3 }, /* (168) full_table_name ::= db_name NK_DOT table_name */ - { 358, -1 }, /* (169) column_def_list ::= column_def */ - { 358, -3 }, /* (170) column_def_list ::= column_def_list NK_COMMA column_def */ - { 375, -2 }, /* (171) column_def ::= column_name type_name */ - { 375, -4 }, /* (172) column_def ::= column_name type_name COMMENT NK_STRING */ - { 367, -1 }, /* (173) type_name ::= BOOL */ - { 367, -1 }, /* (174) type_name ::= TINYINT */ - { 367, -1 }, /* (175) type_name ::= SMALLINT */ - { 367, -1 }, /* (176) type_name ::= INT */ - { 367, -1 }, /* (177) type_name ::= INTEGER */ - { 367, -1 }, /* (178) type_name ::= BIGINT */ - { 367, -1 }, /* (179) type_name ::= FLOAT */ - { 367, -1 }, /* (180) type_name ::= DOUBLE */ - { 367, -4 }, /* (181) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ - { 367, -1 }, /* (182) type_name ::= TIMESTAMP */ - { 367, -4 }, /* (183) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ - { 367, -2 }, /* (184) type_name ::= TINYINT UNSIGNED */ - { 367, -2 }, /* (185) type_name ::= SMALLINT UNSIGNED */ - { 367, -2 }, /* (186) type_name ::= INT UNSIGNED */ - { 367, -2 }, /* (187) type_name ::= BIGINT UNSIGNED */ - { 367, -1 }, /* (188) type_name ::= JSON */ - { 367, -4 }, /* (189) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ - { 367, -1 }, /* (190) type_name ::= MEDIUMBLOB */ - { 367, -1 }, /* (191) type_name ::= BLOB */ - { 367, -4 }, /* (192) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ - { 367, -1 }, /* (193) type_name ::= DECIMAL */ - { 367, -4 }, /* (194) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ - { 367, -6 }, /* (195) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ - { 359, 0 }, /* (196) tags_def_opt ::= */ - { 359, -1 }, /* (197) tags_def_opt ::= tags_def */ - { 362, -4 }, /* (198) tags_def ::= TAGS NK_LP column_def_list NK_RP */ - { 360, 0 }, /* (199) table_options ::= */ - { 360, -3 }, /* (200) table_options ::= table_options COMMENT NK_STRING */ - { 360, -3 }, /* (201) table_options ::= table_options MAX_DELAY duration_list */ - { 360, -3 }, /* (202) table_options ::= table_options WATERMARK duration_list */ - { 360, -5 }, /* (203) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ - { 360, -3 }, /* (204) table_options ::= table_options TTL NK_INTEGER */ - { 360, -5 }, /* (205) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ - { 360, -3 }, /* (206) table_options ::= table_options DELETE_MARK duration_list */ - { 365, -1 }, /* (207) alter_table_options ::= alter_table_option */ - { 365, -2 }, /* (208) alter_table_options ::= alter_table_options alter_table_option */ - { 378, -2 }, /* (209) alter_table_option ::= COMMENT NK_STRING */ - { 378, -2 }, /* (210) alter_table_option ::= TTL NK_INTEGER */ - { 376, -1 }, /* (211) duration_list ::= duration_literal */ - { 376, -3 }, /* (212) duration_list ::= duration_list NK_COMMA duration_literal */ - { 377, -1 }, /* (213) rollup_func_list ::= rollup_func_name */ - { 377, -3 }, /* (214) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ - { 380, -1 }, /* (215) rollup_func_name ::= function_name */ - { 380, -1 }, /* (216) rollup_func_name ::= FIRST */ - { 380, -1 }, /* (217) rollup_func_name ::= LAST */ - { 373, -1 }, /* (218) col_name_list ::= col_name */ - { 373, -3 }, /* (219) col_name_list ::= col_name_list NK_COMMA col_name */ - { 382, -1 }, /* (220) col_name ::= column_name */ - { 330, -2 }, /* (221) cmd ::= SHOW DNODES */ - { 330, -2 }, /* (222) cmd ::= SHOW USERS */ - { 330, -3 }, /* (223) cmd ::= SHOW USER PRIVILEGES */ - { 330, -2 }, /* (224) cmd ::= SHOW DATABASES */ - { 330, -4 }, /* (225) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ - { 330, -4 }, /* (226) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ - { 330, -3 }, /* (227) cmd ::= SHOW db_name_cond_opt VGROUPS */ - { 330, -2 }, /* (228) cmd ::= SHOW MNODES */ - { 330, -2 }, /* (229) cmd ::= SHOW QNODES */ - { 330, -2 }, /* (230) cmd ::= SHOW FUNCTIONS */ - { 330, -5 }, /* (231) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ - { 330, -2 }, /* (232) cmd ::= SHOW STREAMS */ - { 330, -2 }, /* (233) cmd ::= SHOW ACCOUNTS */ - { 330, -2 }, /* (234) cmd ::= SHOW APPS */ - { 330, -2 }, /* (235) cmd ::= SHOW CONNECTIONS */ - { 330, -2 }, /* (236) cmd ::= SHOW LICENCES */ - { 330, -2 }, /* (237) cmd ::= SHOW GRANTS */ - { 330, -4 }, /* (238) cmd ::= SHOW CREATE DATABASE db_name */ - { 330, -4 }, /* (239) cmd ::= SHOW CREATE TABLE full_table_name */ - { 330, -4 }, /* (240) cmd ::= SHOW CREATE STABLE full_table_name */ - { 330, -2 }, /* (241) cmd ::= SHOW QUERIES */ - { 330, -2 }, /* (242) cmd ::= SHOW SCORES */ - { 330, -2 }, /* (243) cmd ::= SHOW TOPICS */ - { 330, -2 }, /* (244) cmd ::= SHOW VARIABLES */ - { 330, -3 }, /* (245) cmd ::= SHOW CLUSTER VARIABLES */ - { 330, -3 }, /* (246) cmd ::= SHOW LOCAL VARIABLES */ - { 330, -5 }, /* (247) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ - { 330, -2 }, /* (248) cmd ::= SHOW BNODES */ - { 330, -2 }, /* (249) cmd ::= SHOW SNODES */ - { 330, -2 }, /* (250) cmd ::= SHOW CLUSTER */ - { 330, -2 }, /* (251) cmd ::= SHOW TRANSACTIONS */ - { 330, -4 }, /* (252) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ - { 330, -2 }, /* (253) cmd ::= SHOW CONSUMERS */ - { 330, -2 }, /* (254) cmd ::= SHOW SUBSCRIPTIONS */ - { 330, -5 }, /* (255) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ - { 330, -7 }, /* (256) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ - { 330, -3 }, /* (257) cmd ::= SHOW VNODES NK_INTEGER */ - { 330, -3 }, /* (258) cmd ::= SHOW VNODES NK_STRING */ - { 330, -3 }, /* (259) cmd ::= SHOW db_name_cond_opt ALIVE */ - { 330, -3 }, /* (260) cmd ::= SHOW CLUSTER ALIVE */ - { 383, 0 }, /* (261) db_name_cond_opt ::= */ - { 383, -2 }, /* (262) db_name_cond_opt ::= db_name NK_DOT */ - { 384, 0 }, /* (263) like_pattern_opt ::= */ - { 384, -2 }, /* (264) like_pattern_opt ::= LIKE NK_STRING */ - { 385, -1 }, /* (265) table_name_cond ::= table_name */ - { 386, 0 }, /* (266) from_db_opt ::= */ - { 386, -2 }, /* (267) from_db_opt ::= FROM db_name */ - { 387, 0 }, /* (268) tag_list_opt ::= */ - { 387, -1 }, /* (269) tag_list_opt ::= tag_item */ - { 387, -3 }, /* (270) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ - { 388, -1 }, /* (271) tag_item ::= TBNAME */ - { 388, -1 }, /* (272) tag_item ::= QTAGS */ - { 388, -1 }, /* (273) tag_item ::= column_name */ - { 388, -2 }, /* (274) tag_item ::= column_name column_alias */ - { 388, -3 }, /* (275) tag_item ::= column_name AS column_alias */ - { 330, -8 }, /* (276) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ - { 330, -9 }, /* (277) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ - { 330, -4 }, /* (278) cmd ::= DROP INDEX exists_opt full_index_name */ - { 390, -1 }, /* (279) full_index_name ::= index_name */ - { 390, -3 }, /* (280) full_index_name ::= db_name NK_DOT index_name */ - { 391, -10 }, /* (281) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ - { 391, -12 }, /* (282) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ - { 393, -1 }, /* (283) func_list ::= func */ - { 393, -3 }, /* (284) func_list ::= func_list NK_COMMA func */ - { 396, -4 }, /* (285) func ::= sma_func_name NK_LP expression_list NK_RP */ - { 397, -1 }, /* (286) sma_func_name ::= function_name */ - { 397, -1 }, /* (287) sma_func_name ::= COUNT */ - { 397, -1 }, /* (288) sma_func_name ::= FIRST */ - { 397, -1 }, /* (289) sma_func_name ::= LAST */ - { 397, -1 }, /* (290) sma_func_name ::= LAST_ROW */ - { 395, 0 }, /* (291) sma_stream_opt ::= */ - { 395, -3 }, /* (292) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ - { 395, -3 }, /* (293) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ - { 395, -3 }, /* (294) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ - { 330, -6 }, /* (295) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ - { 330, -7 }, /* (296) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ - { 330, -9 }, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ - { 330, -7 }, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ - { 330, -9 }, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ - { 330, -4 }, /* (300) cmd ::= DROP TOPIC exists_opt topic_name */ - { 330, -7 }, /* (301) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ - { 330, -2 }, /* (302) cmd ::= DESC full_table_name */ - { 330, -2 }, /* (303) cmd ::= DESCRIBE full_table_name */ - { 330, -3 }, /* (304) cmd ::= RESET QUERY CACHE */ - { 330, -4 }, /* (305) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - { 330, -4 }, /* (306) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ - { 400, 0 }, /* (307) analyze_opt ::= */ - { 400, -1 }, /* (308) analyze_opt ::= ANALYZE */ - { 401, 0 }, /* (309) explain_options ::= */ - { 401, -3 }, /* (310) explain_options ::= explain_options VERBOSE NK_BOOL */ - { 401, -3 }, /* (311) explain_options ::= explain_options RATIO NK_FLOAT */ - { 330, -12 }, /* (312) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ - { 330, -4 }, /* (313) cmd ::= DROP FUNCTION exists_opt function_name */ - { 404, 0 }, /* (314) agg_func_opt ::= */ - { 404, -1 }, /* (315) agg_func_opt ::= AGGREGATE */ - { 405, 0 }, /* (316) bufsize_opt ::= */ - { 405, -2 }, /* (317) bufsize_opt ::= BUFSIZE NK_INTEGER */ - { 406, 0 }, /* (318) language_opt ::= */ - { 406, -2 }, /* (319) language_opt ::= LANGUAGE NK_STRING */ - { 403, 0 }, /* (320) or_replace_opt ::= */ - { 403, -2 }, /* (321) or_replace_opt ::= OR REPLACE */ - { 330, -12 }, /* (322) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ - { 330, -4 }, /* (323) cmd ::= DROP STREAM exists_opt stream_name */ - { 409, 0 }, /* (324) col_list_opt ::= */ - { 409, -3 }, /* (325) col_list_opt ::= NK_LP col_name_list NK_RP */ - { 410, 0 }, /* (326) tag_def_or_ref_opt ::= */ - { 410, -1 }, /* (327) tag_def_or_ref_opt ::= tags_def */ - { 410, -4 }, /* (328) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ - { 408, 0 }, /* (329) stream_options ::= */ - { 408, -3 }, /* (330) stream_options ::= stream_options TRIGGER AT_ONCE */ - { 408, -3 }, /* (331) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ - { 408, -4 }, /* (332) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ - { 408, -3 }, /* (333) stream_options ::= stream_options WATERMARK duration_literal */ - { 408, -4 }, /* (334) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ - { 408, -3 }, /* (335) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ - { 408, -3 }, /* (336) stream_options ::= stream_options DELETE_MARK duration_literal */ - { 408, -4 }, /* (337) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ - { 411, 0 }, /* (338) subtable_opt ::= */ - { 411, -4 }, /* (339) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - { 330, -3 }, /* (340) cmd ::= KILL CONNECTION NK_INTEGER */ - { 330, -3 }, /* (341) cmd ::= KILL QUERY NK_STRING */ - { 330, -3 }, /* (342) cmd ::= KILL TRANSACTION NK_INTEGER */ - { 330, -2 }, /* (343) cmd ::= BALANCE VGROUP */ - { 330, -3 }, /* (344) cmd ::= BALANCE VGROUP LEADER */ - { 330, -4 }, /* (345) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ - { 330, -4 }, /* (346) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ - { 330, -3 }, /* (347) cmd ::= SPLIT VGROUP NK_INTEGER */ - { 413, -2 }, /* (348) dnode_list ::= DNODE NK_INTEGER */ - { 413, -3 }, /* (349) dnode_list ::= dnode_list DNODE NK_INTEGER */ - { 330, -4 }, /* (350) cmd ::= DELETE FROM full_table_name where_clause_opt */ - { 330, -1 }, /* (351) cmd ::= query_or_subquery */ - { 330, -1 }, /* (352) cmd ::= insert_query */ - { 402, -7 }, /* (353) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ - { 402, -4 }, /* (354) insert_query ::= INSERT INTO full_table_name query_or_subquery */ - { 333, -1 }, /* (355) literal ::= NK_INTEGER */ - { 333, -1 }, /* (356) literal ::= NK_FLOAT */ - { 333, -1 }, /* (357) literal ::= NK_STRING */ - { 333, -1 }, /* (358) literal ::= NK_BOOL */ - { 333, -2 }, /* (359) literal ::= TIMESTAMP NK_STRING */ - { 333, -1 }, /* (360) literal ::= duration_literal */ - { 333, -1 }, /* (361) literal ::= NULL */ - { 333, -1 }, /* (362) literal ::= NK_QUESTION */ - { 379, -1 }, /* (363) duration_literal ::= NK_VARIABLE */ - { 415, -1 }, /* (364) signed ::= NK_INTEGER */ - { 415, -2 }, /* (365) signed ::= NK_PLUS NK_INTEGER */ - { 415, -2 }, /* (366) signed ::= NK_MINUS NK_INTEGER */ - { 415, -1 }, /* (367) signed ::= NK_FLOAT */ - { 415, -2 }, /* (368) signed ::= NK_PLUS NK_FLOAT */ - { 415, -2 }, /* (369) signed ::= NK_MINUS NK_FLOAT */ - { 368, -1 }, /* (370) signed_literal ::= signed */ - { 368, -1 }, /* (371) signed_literal ::= NK_STRING */ - { 368, -1 }, /* (372) signed_literal ::= NK_BOOL */ - { 368, -2 }, /* (373) signed_literal ::= TIMESTAMP NK_STRING */ - { 368, -1 }, /* (374) signed_literal ::= duration_literal */ - { 368, -1 }, /* (375) signed_literal ::= NULL */ - { 368, -1 }, /* (376) signed_literal ::= literal_func */ - { 368, -1 }, /* (377) signed_literal ::= NK_QUESTION */ - { 417, -1 }, /* (378) literal_list ::= signed_literal */ - { 417, -3 }, /* (379) literal_list ::= literal_list NK_COMMA signed_literal */ - { 341, -1 }, /* (380) db_name ::= NK_ID */ - { 374, -1 }, /* (381) table_name ::= NK_ID */ - { 366, -1 }, /* (382) column_name ::= NK_ID */ - { 381, -1 }, /* (383) function_name ::= NK_ID */ - { 418, -1 }, /* (384) table_alias ::= NK_ID */ - { 389, -1 }, /* (385) column_alias ::= NK_ID */ - { 335, -1 }, /* (386) user_name ::= NK_ID */ - { 342, -1 }, /* (387) topic_name ::= NK_ID */ - { 407, -1 }, /* (388) stream_name ::= NK_ID */ - { 399, -1 }, /* (389) cgroup_name ::= NK_ID */ - { 392, -1 }, /* (390) index_name ::= NK_ID */ - { 419, -1 }, /* (391) expr_or_subquery ::= expression */ - { 412, -1 }, /* (392) expression ::= literal */ - { 412, -1 }, /* (393) expression ::= pseudo_column */ - { 412, -1 }, /* (394) expression ::= column_reference */ - { 412, -1 }, /* (395) expression ::= function_expression */ - { 412, -1 }, /* (396) expression ::= case_when_expression */ - { 412, -3 }, /* (397) expression ::= NK_LP expression NK_RP */ - { 412, -2 }, /* (398) expression ::= NK_PLUS expr_or_subquery */ - { 412, -2 }, /* (399) expression ::= NK_MINUS expr_or_subquery */ - { 412, -3 }, /* (400) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ - { 412, -3 }, /* (401) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ - { 412, -3 }, /* (402) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ - { 412, -3 }, /* (403) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ - { 412, -3 }, /* (404) expression ::= expr_or_subquery NK_REM expr_or_subquery */ - { 412, -3 }, /* (405) expression ::= column_reference NK_ARROW NK_STRING */ - { 412, -3 }, /* (406) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ - { 412, -3 }, /* (407) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ - { 371, -1 }, /* (408) expression_list ::= expr_or_subquery */ - { 371, -3 }, /* (409) expression_list ::= expression_list NK_COMMA expr_or_subquery */ - { 421, -1 }, /* (410) column_reference ::= column_name */ - { 421, -3 }, /* (411) column_reference ::= table_name NK_DOT column_name */ - { 420, -1 }, /* (412) pseudo_column ::= ROWTS */ - { 420, -1 }, /* (413) pseudo_column ::= TBNAME */ - { 420, -3 }, /* (414) pseudo_column ::= table_name NK_DOT TBNAME */ - { 420, -1 }, /* (415) pseudo_column ::= QSTART */ - { 420, -1 }, /* (416) pseudo_column ::= QEND */ - { 420, -1 }, /* (417) pseudo_column ::= QDURATION */ - { 420, -1 }, /* (418) pseudo_column ::= WSTART */ - { 420, -1 }, /* (419) pseudo_column ::= WEND */ - { 420, -1 }, /* (420) pseudo_column ::= WDURATION */ - { 420, -1 }, /* (421) pseudo_column ::= IROWTS */ - { 420, -1 }, /* (422) pseudo_column ::= ISFILLED */ - { 420, -1 }, /* (423) pseudo_column ::= QTAGS */ - { 422, -4 }, /* (424) function_expression ::= function_name NK_LP expression_list NK_RP */ - { 422, -4 }, /* (425) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ - { 422, -6 }, /* (426) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ - { 422, -1 }, /* (427) function_expression ::= literal_func */ - { 416, -3 }, /* (428) literal_func ::= noarg_func NK_LP NK_RP */ - { 416, -1 }, /* (429) literal_func ::= NOW */ - { 426, -1 }, /* (430) noarg_func ::= NOW */ - { 426, -1 }, /* (431) noarg_func ::= TODAY */ - { 426, -1 }, /* (432) noarg_func ::= TIMEZONE */ - { 426, -1 }, /* (433) noarg_func ::= DATABASE */ - { 426, -1 }, /* (434) noarg_func ::= CLIENT_VERSION */ - { 426, -1 }, /* (435) noarg_func ::= SERVER_VERSION */ - { 426, -1 }, /* (436) noarg_func ::= SERVER_STATUS */ - { 426, -1 }, /* (437) noarg_func ::= CURRENT_USER */ - { 426, -1 }, /* (438) noarg_func ::= USER */ - { 424, -1 }, /* (439) star_func ::= COUNT */ - { 424, -1 }, /* (440) star_func ::= FIRST */ - { 424, -1 }, /* (441) star_func ::= LAST */ - { 424, -1 }, /* (442) star_func ::= LAST_ROW */ - { 425, -1 }, /* (443) star_func_para_list ::= NK_STAR */ - { 425, -1 }, /* (444) star_func_para_list ::= other_para_list */ - { 427, -1 }, /* (445) other_para_list ::= star_func_para */ - { 427, -3 }, /* (446) other_para_list ::= other_para_list NK_COMMA star_func_para */ - { 428, -1 }, /* (447) star_func_para ::= expr_or_subquery */ - { 428, -3 }, /* (448) star_func_para ::= table_name NK_DOT NK_STAR */ - { 423, -4 }, /* (449) case_when_expression ::= CASE when_then_list case_when_else_opt END */ - { 423, -5 }, /* (450) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ - { 429, -1 }, /* (451) when_then_list ::= when_then_expr */ - { 429, -2 }, /* (452) when_then_list ::= when_then_list when_then_expr */ - { 432, -4 }, /* (453) when_then_expr ::= WHEN common_expression THEN common_expression */ - { 430, 0 }, /* (454) case_when_else_opt ::= */ - { 430, -2 }, /* (455) case_when_else_opt ::= ELSE common_expression */ - { 433, -3 }, /* (456) predicate ::= expr_or_subquery compare_op expr_or_subquery */ - { 433, -5 }, /* (457) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ - { 433, -6 }, /* (458) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ - { 433, -3 }, /* (459) predicate ::= expr_or_subquery IS NULL */ - { 433, -4 }, /* (460) predicate ::= expr_or_subquery IS NOT NULL */ - { 433, -3 }, /* (461) predicate ::= expr_or_subquery in_op in_predicate_value */ - { 434, -1 }, /* (462) compare_op ::= NK_LT */ - { 434, -1 }, /* (463) compare_op ::= NK_GT */ - { 434, -1 }, /* (464) compare_op ::= NK_LE */ - { 434, -1 }, /* (465) compare_op ::= NK_GE */ - { 434, -1 }, /* (466) compare_op ::= NK_NE */ - { 434, -1 }, /* (467) compare_op ::= NK_EQ */ - { 434, -1 }, /* (468) compare_op ::= LIKE */ - { 434, -2 }, /* (469) compare_op ::= NOT LIKE */ - { 434, -1 }, /* (470) compare_op ::= MATCH */ - { 434, -1 }, /* (471) compare_op ::= NMATCH */ - { 434, -1 }, /* (472) compare_op ::= CONTAINS */ - { 435, -1 }, /* (473) in_op ::= IN */ - { 435, -2 }, /* (474) in_op ::= NOT IN */ - { 436, -3 }, /* (475) in_predicate_value ::= NK_LP literal_list NK_RP */ - { 437, -1 }, /* (476) boolean_value_expression ::= boolean_primary */ - { 437, -2 }, /* (477) boolean_value_expression ::= NOT boolean_primary */ - { 437, -3 }, /* (478) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ - { 437, -3 }, /* (479) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ - { 438, -1 }, /* (480) boolean_primary ::= predicate */ - { 438, -3 }, /* (481) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ - { 431, -1 }, /* (482) common_expression ::= expr_or_subquery */ - { 431, -1 }, /* (483) common_expression ::= boolean_value_expression */ - { 439, 0 }, /* (484) from_clause_opt ::= */ - { 439, -2 }, /* (485) from_clause_opt ::= FROM table_reference_list */ - { 440, -1 }, /* (486) table_reference_list ::= table_reference */ - { 440, -3 }, /* (487) table_reference_list ::= table_reference_list NK_COMMA table_reference */ - { 441, -1 }, /* (488) table_reference ::= table_primary */ - { 441, -1 }, /* (489) table_reference ::= joined_table */ - { 442, -2 }, /* (490) table_primary ::= table_name alias_opt */ - { 442, -4 }, /* (491) table_primary ::= db_name NK_DOT table_name alias_opt */ - { 442, -2 }, /* (492) table_primary ::= subquery alias_opt */ - { 442, -1 }, /* (493) table_primary ::= parenthesized_joined_table */ - { 444, 0 }, /* (494) alias_opt ::= */ - { 444, -1 }, /* (495) alias_opt ::= table_alias */ - { 444, -2 }, /* (496) alias_opt ::= AS table_alias */ - { 446, -3 }, /* (497) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - { 446, -3 }, /* (498) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ - { 443, -6 }, /* (499) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ - { 447, 0 }, /* (500) join_type ::= */ - { 447, -1 }, /* (501) join_type ::= INNER */ - { 449, -12 }, /* (502) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ - { 450, 0 }, /* (503) set_quantifier_opt ::= */ - { 450, -1 }, /* (504) set_quantifier_opt ::= DISTINCT */ - { 450, -1 }, /* (505) set_quantifier_opt ::= ALL */ - { 451, -1 }, /* (506) select_list ::= select_item */ - { 451, -3 }, /* (507) select_list ::= select_list NK_COMMA select_item */ - { 459, -1 }, /* (508) select_item ::= NK_STAR */ - { 459, -1 }, /* (509) select_item ::= common_expression */ - { 459, -2 }, /* (510) select_item ::= common_expression column_alias */ - { 459, -3 }, /* (511) select_item ::= common_expression AS column_alias */ - { 459, -3 }, /* (512) select_item ::= table_name NK_DOT NK_STAR */ - { 414, 0 }, /* (513) where_clause_opt ::= */ - { 414, -2 }, /* (514) where_clause_opt ::= WHERE search_condition */ - { 452, 0 }, /* (515) partition_by_clause_opt ::= */ - { 452, -3 }, /* (516) partition_by_clause_opt ::= PARTITION BY partition_list */ - { 460, -1 }, /* (517) partition_list ::= partition_item */ - { 460, -3 }, /* (518) partition_list ::= partition_list NK_COMMA partition_item */ - { 461, -1 }, /* (519) partition_item ::= expr_or_subquery */ - { 461, -2 }, /* (520) partition_item ::= expr_or_subquery column_alias */ - { 461, -3 }, /* (521) partition_item ::= expr_or_subquery AS column_alias */ - { 456, 0 }, /* (522) twindow_clause_opt ::= */ - { 456, -6 }, /* (523) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ - { 456, -4 }, /* (524) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ - { 456, -6 }, /* (525) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ - { 456, -8 }, /* (526) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ - { 456, -7 }, /* (527) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ - { 394, 0 }, /* (528) sliding_opt ::= */ - { 394, -4 }, /* (529) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ - { 455, 0 }, /* (530) fill_opt ::= */ - { 455, -4 }, /* (531) fill_opt ::= FILL NK_LP fill_mode NK_RP */ - { 455, -6 }, /* (532) fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ - { 455, -6 }, /* (533) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ - { 462, -1 }, /* (534) fill_mode ::= NONE */ - { 462, -1 }, /* (535) fill_mode ::= PREV */ - { 462, -1 }, /* (536) fill_mode ::= NULL */ - { 462, -1 }, /* (537) fill_mode ::= NULL_F */ - { 462, -1 }, /* (538) fill_mode ::= LINEAR */ - { 462, -1 }, /* (539) fill_mode ::= NEXT */ - { 457, 0 }, /* (540) group_by_clause_opt ::= */ - { 457, -3 }, /* (541) group_by_clause_opt ::= GROUP BY group_by_list */ - { 463, -1 }, /* (542) group_by_list ::= expr_or_subquery */ - { 463, -3 }, /* (543) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ - { 458, 0 }, /* (544) having_clause_opt ::= */ - { 458, -2 }, /* (545) having_clause_opt ::= HAVING search_condition */ - { 453, 0 }, /* (546) range_opt ::= */ - { 453, -6 }, /* (547) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ - { 454, 0 }, /* (548) every_opt ::= */ - { 454, -4 }, /* (549) every_opt ::= EVERY NK_LP duration_literal NK_RP */ - { 464, -4 }, /* (550) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ - { 465, -1 }, /* (551) query_simple ::= query_specification */ - { 465, -1 }, /* (552) query_simple ::= union_query_expression */ - { 469, -4 }, /* (553) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ - { 469, -3 }, /* (554) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ - { 470, -1 }, /* (555) query_simple_or_subquery ::= query_simple */ - { 470, -1 }, /* (556) query_simple_or_subquery ::= subquery */ - { 398, -1 }, /* (557) query_or_subquery ::= query_expression */ - { 398, -1 }, /* (558) query_or_subquery ::= subquery */ - { 466, 0 }, /* (559) order_by_clause_opt ::= */ - { 466, -3 }, /* (560) order_by_clause_opt ::= ORDER BY sort_specification_list */ - { 467, 0 }, /* (561) slimit_clause_opt ::= */ - { 467, -2 }, /* (562) slimit_clause_opt ::= SLIMIT NK_INTEGER */ - { 467, -4 }, /* (563) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - { 467, -4 }, /* (564) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 468, 0 }, /* (565) limit_clause_opt ::= */ - { 468, -2 }, /* (566) limit_clause_opt ::= LIMIT NK_INTEGER */ - { 468, -4 }, /* (567) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ - { 468, -4 }, /* (568) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - { 445, -3 }, /* (569) subquery ::= NK_LP query_expression NK_RP */ - { 445, -3 }, /* (570) subquery ::= NK_LP subquery NK_RP */ - { 448, -1 }, /* (571) search_condition ::= common_expression */ - { 471, -1 }, /* (572) sort_specification_list ::= sort_specification */ - { 471, -3 }, /* (573) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ - { 472, -3 }, /* (574) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ - { 473, 0 }, /* (575) ordering_specification_opt ::= */ - { 473, -1 }, /* (576) ordering_specification_opt ::= ASC */ - { 473, -1 }, /* (577) ordering_specification_opt ::= DESC */ - { 474, 0 }, /* (578) null_ordering_opt ::= */ - { 474, -2 }, /* (579) null_ordering_opt ::= NULLS FIRST */ - { 474, -2 }, /* (580) null_ordering_opt ::= NULLS LAST */ + { 338, -3 }, /* (42) priv_level ::= db_name NK_DOT table_name */ + { 338, -1 }, /* (43) priv_level ::= topic_name */ + { 339, 0 }, /* (44) with_opt ::= */ + { 339, -2 }, /* (45) with_opt ::= WITH search_condition */ + { 330, -3 }, /* (46) cmd ::= CREATE DNODE dnode_endpoint */ + { 330, -5 }, /* (47) cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ + { 330, -4 }, /* (48) cmd ::= DROP DNODE NK_INTEGER force_opt */ + { 330, -4 }, /* (49) cmd ::= DROP DNODE dnode_endpoint force_opt */ + { 330, -4 }, /* (50) cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ + { 330, -5 }, /* (51) cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + { 330, -4 }, /* (52) cmd ::= ALTER ALL DNODES NK_STRING */ + { 330, -5 }, /* (53) cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + { 346, -1 }, /* (54) dnode_endpoint ::= NK_STRING */ + { 346, -1 }, /* (55) dnode_endpoint ::= NK_ID */ + { 346, -1 }, /* (56) dnode_endpoint ::= NK_IPTOKEN */ + { 347, 0 }, /* (57) force_opt ::= */ + { 347, -1 }, /* (58) force_opt ::= FORCE */ + { 330, -3 }, /* (59) cmd ::= ALTER LOCAL NK_STRING */ + { 330, -4 }, /* (60) cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + { 330, -5 }, /* (61) cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (62) cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (63) cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (64) cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (65) cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (66) cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (67) cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (68) cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + { 330, -5 }, /* (69) cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ + { 330, -4 }, /* (70) cmd ::= DROP DATABASE exists_opt db_name */ + { 330, -2 }, /* (71) cmd ::= USE db_name */ + { 330, -4 }, /* (72) cmd ::= ALTER DATABASE db_name alter_db_options */ + { 330, -3 }, /* (73) cmd ::= FLUSH DATABASE db_name */ + { 330, -4 }, /* (74) cmd ::= TRIM DATABASE db_name speed_opt */ + { 330, -5 }, /* (75) cmd ::= COMPACT DATABASE db_name start_opt end_opt */ + { 348, -3 }, /* (76) not_exists_opt ::= IF NOT EXISTS */ + { 348, 0 }, /* (77) not_exists_opt ::= */ + { 350, -2 }, /* (78) exists_opt ::= IF EXISTS */ + { 350, 0 }, /* (79) exists_opt ::= */ + { 349, 0 }, /* (80) db_options ::= */ + { 349, -3 }, /* (81) db_options ::= db_options BUFFER NK_INTEGER */ + { 349, -3 }, /* (82) db_options ::= db_options CACHEMODEL NK_STRING */ + { 349, -3 }, /* (83) db_options ::= db_options CACHESIZE NK_INTEGER */ + { 349, -3 }, /* (84) db_options ::= db_options COMP NK_INTEGER */ + { 349, -3 }, /* (85) db_options ::= db_options DURATION NK_INTEGER */ + { 349, -3 }, /* (86) db_options ::= db_options DURATION NK_VARIABLE */ + { 349, -3 }, /* (87) db_options ::= db_options MAXROWS NK_INTEGER */ + { 349, -3 }, /* (88) db_options ::= db_options MINROWS NK_INTEGER */ + { 349, -3 }, /* (89) db_options ::= db_options KEEP integer_list */ + { 349, -3 }, /* (90) db_options ::= db_options KEEP variable_list */ + { 349, -3 }, /* (91) db_options ::= db_options PAGES NK_INTEGER */ + { 349, -3 }, /* (92) db_options ::= db_options PAGESIZE NK_INTEGER */ + { 349, -3 }, /* (93) db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ + { 349, -3 }, /* (94) db_options ::= db_options PRECISION NK_STRING */ + { 349, -3 }, /* (95) db_options ::= db_options REPLICA NK_INTEGER */ + { 349, -3 }, /* (96) db_options ::= db_options VGROUPS NK_INTEGER */ + { 349, -3 }, /* (97) db_options ::= db_options SINGLE_STABLE NK_INTEGER */ + { 349, -3 }, /* (98) db_options ::= db_options RETENTIONS retention_list */ + { 349, -3 }, /* (99) db_options ::= db_options SCHEMALESS NK_INTEGER */ + { 349, -3 }, /* (100) db_options ::= db_options WAL_LEVEL NK_INTEGER */ + { 349, -3 }, /* (101) db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ + { 349, -3 }, /* (102) db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ + { 349, -4 }, /* (103) db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 349, -3 }, /* (104) db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ + { 349, -4 }, /* (105) db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 349, -3 }, /* (106) db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ + { 349, -3 }, /* (107) db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ + { 349, -3 }, /* (108) db_options ::= db_options STT_TRIGGER NK_INTEGER */ + { 349, -3 }, /* (109) db_options ::= db_options TABLE_PREFIX NK_INTEGER */ + { 349, -3 }, /* (110) db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ + { 351, -1 }, /* (111) alter_db_options ::= alter_db_option */ + { 351, -2 }, /* (112) alter_db_options ::= alter_db_options alter_db_option */ + { 358, -2 }, /* (113) alter_db_option ::= BUFFER NK_INTEGER */ + { 358, -2 }, /* (114) alter_db_option ::= CACHEMODEL NK_STRING */ + { 358, -2 }, /* (115) alter_db_option ::= CACHESIZE NK_INTEGER */ + { 358, -2 }, /* (116) alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ + { 358, -2 }, /* (117) alter_db_option ::= KEEP integer_list */ + { 358, -2 }, /* (118) alter_db_option ::= KEEP variable_list */ + { 358, -2 }, /* (119) alter_db_option ::= PAGES NK_INTEGER */ + { 358, -2 }, /* (120) alter_db_option ::= REPLICA NK_INTEGER */ + { 358, -2 }, /* (121) alter_db_option ::= WAL_LEVEL NK_INTEGER */ + { 358, -2 }, /* (122) alter_db_option ::= STT_TRIGGER NK_INTEGER */ + { 358, -2 }, /* (123) alter_db_option ::= MINROWS NK_INTEGER */ + { 358, -2 }, /* (124) alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ + { 358, -3 }, /* (125) alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + { 358, -2 }, /* (126) alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ + { 358, -3 }, /* (127) alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + { 355, -1 }, /* (128) integer_list ::= NK_INTEGER */ + { 355, -3 }, /* (129) integer_list ::= integer_list NK_COMMA NK_INTEGER */ + { 356, -1 }, /* (130) variable_list ::= NK_VARIABLE */ + { 356, -3 }, /* (131) variable_list ::= variable_list NK_COMMA NK_VARIABLE */ + { 357, -1 }, /* (132) retention_list ::= retention */ + { 357, -3 }, /* (133) retention_list ::= retention_list NK_COMMA retention */ + { 359, -3 }, /* (134) retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ + { 352, 0 }, /* (135) speed_opt ::= */ + { 352, -2 }, /* (136) speed_opt ::= MAX_SPEED NK_INTEGER */ + { 353, 0 }, /* (137) start_opt ::= */ + { 353, -3 }, /* (138) start_opt ::= START WITH NK_INTEGER */ + { 353, -3 }, /* (139) start_opt ::= START WITH NK_STRING */ + { 353, -4 }, /* (140) start_opt ::= START WITH TIMESTAMP NK_STRING */ + { 354, 0 }, /* (141) end_opt ::= */ + { 354, -3 }, /* (142) end_opt ::= END WITH NK_INTEGER */ + { 354, -3 }, /* (143) end_opt ::= END WITH NK_STRING */ + { 354, -4 }, /* (144) end_opt ::= END WITH TIMESTAMP NK_STRING */ + { 330, -9 }, /* (145) cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + { 330, -3 }, /* (146) cmd ::= CREATE TABLE multi_create_clause */ + { 330, -9 }, /* (147) cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ + { 330, -3 }, /* (148) cmd ::= DROP TABLE multi_drop_clause */ + { 330, -4 }, /* (149) cmd ::= DROP STABLE exists_opt full_table_name */ + { 330, -3 }, /* (150) cmd ::= ALTER TABLE alter_table_clause */ + { 330, -3 }, /* (151) cmd ::= ALTER STABLE alter_table_clause */ + { 367, -2 }, /* (152) alter_table_clause ::= full_table_name alter_table_options */ + { 367, -5 }, /* (153) alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ + { 367, -4 }, /* (154) alter_table_clause ::= full_table_name DROP COLUMN column_name */ + { 367, -5 }, /* (155) alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ + { 367, -5 }, /* (156) alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ + { 367, -5 }, /* (157) alter_table_clause ::= full_table_name ADD TAG column_name type_name */ + { 367, -4 }, /* (158) alter_table_clause ::= full_table_name DROP TAG column_name */ + { 367, -5 }, /* (159) alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ + { 367, -5 }, /* (160) alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ + { 367, -6 }, /* (161) alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ + { 364, -1 }, /* (162) multi_create_clause ::= create_subtable_clause */ + { 364, -2 }, /* (163) multi_create_clause ::= multi_create_clause create_subtable_clause */ + { 372, -10 }, /* (164) create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ + { 366, -1 }, /* (165) multi_drop_clause ::= drop_table_clause */ + { 366, -3 }, /* (166) multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ + { 375, -2 }, /* (167) drop_table_clause ::= exists_opt full_table_name */ + { 373, 0 }, /* (168) specific_cols_opt ::= */ + { 373, -3 }, /* (169) specific_cols_opt ::= NK_LP col_name_list NK_RP */ + { 360, -1 }, /* (170) full_table_name ::= table_name */ + { 360, -3 }, /* (171) full_table_name ::= db_name NK_DOT table_name */ + { 361, -1 }, /* (172) column_def_list ::= column_def */ + { 361, -3 }, /* (173) column_def_list ::= column_def_list NK_COMMA column_def */ + { 377, -2 }, /* (174) column_def ::= column_name type_name */ + { 370, -1 }, /* (175) type_name ::= BOOL */ + { 370, -1 }, /* (176) type_name ::= TINYINT */ + { 370, -1 }, /* (177) type_name ::= SMALLINT */ + { 370, -1 }, /* (178) type_name ::= INT */ + { 370, -1 }, /* (179) type_name ::= INTEGER */ + { 370, -1 }, /* (180) type_name ::= BIGINT */ + { 370, -1 }, /* (181) type_name ::= FLOAT */ + { 370, -1 }, /* (182) type_name ::= DOUBLE */ + { 370, -4 }, /* (183) type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ + { 370, -1 }, /* (184) type_name ::= TIMESTAMP */ + { 370, -4 }, /* (185) type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ + { 370, -2 }, /* (186) type_name ::= TINYINT UNSIGNED */ + { 370, -2 }, /* (187) type_name ::= SMALLINT UNSIGNED */ + { 370, -2 }, /* (188) type_name ::= INT UNSIGNED */ + { 370, -2 }, /* (189) type_name ::= BIGINT UNSIGNED */ + { 370, -1 }, /* (190) type_name ::= JSON */ + { 370, -4 }, /* (191) type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ + { 370, -1 }, /* (192) type_name ::= MEDIUMBLOB */ + { 370, -1 }, /* (193) type_name ::= BLOB */ + { 370, -4 }, /* (194) type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ + { 370, -1 }, /* (195) type_name ::= DECIMAL */ + { 370, -4 }, /* (196) type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ + { 370, -6 }, /* (197) type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ + { 362, 0 }, /* (198) tags_def_opt ::= */ + { 362, -1 }, /* (199) tags_def_opt ::= tags_def */ + { 365, -4 }, /* (200) tags_def ::= TAGS NK_LP column_def_list NK_RP */ + { 363, 0 }, /* (201) table_options ::= */ + { 363, -3 }, /* (202) table_options ::= table_options COMMENT NK_STRING */ + { 363, -3 }, /* (203) table_options ::= table_options MAX_DELAY duration_list */ + { 363, -3 }, /* (204) table_options ::= table_options WATERMARK duration_list */ + { 363, -5 }, /* (205) table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ + { 363, -3 }, /* (206) table_options ::= table_options TTL NK_INTEGER */ + { 363, -5 }, /* (207) table_options ::= table_options SMA NK_LP col_name_list NK_RP */ + { 363, -3 }, /* (208) table_options ::= table_options DELETE_MARK duration_list */ + { 368, -1 }, /* (209) alter_table_options ::= alter_table_option */ + { 368, -2 }, /* (210) alter_table_options ::= alter_table_options alter_table_option */ + { 380, -2 }, /* (211) alter_table_option ::= COMMENT NK_STRING */ + { 380, -2 }, /* (212) alter_table_option ::= TTL NK_INTEGER */ + { 378, -1 }, /* (213) duration_list ::= duration_literal */ + { 378, -3 }, /* (214) duration_list ::= duration_list NK_COMMA duration_literal */ + { 379, -1 }, /* (215) rollup_func_list ::= rollup_func_name */ + { 379, -3 }, /* (216) rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ + { 382, -1 }, /* (217) rollup_func_name ::= function_name */ + { 382, -1 }, /* (218) rollup_func_name ::= FIRST */ + { 382, -1 }, /* (219) rollup_func_name ::= LAST */ + { 376, -1 }, /* (220) col_name_list ::= col_name */ + { 376, -3 }, /* (221) col_name_list ::= col_name_list NK_COMMA col_name */ + { 384, -1 }, /* (222) col_name ::= column_name */ + { 330, -2 }, /* (223) cmd ::= SHOW DNODES */ + { 330, -2 }, /* (224) cmd ::= SHOW USERS */ + { 330, -3 }, /* (225) cmd ::= SHOW USER PRIVILEGES */ + { 330, -2 }, /* (226) cmd ::= SHOW DATABASES */ + { 330, -4 }, /* (227) cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ + { 330, -4 }, /* (228) cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ + { 330, -3 }, /* (229) cmd ::= SHOW db_name_cond_opt VGROUPS */ + { 330, -2 }, /* (230) cmd ::= SHOW MNODES */ + { 330, -2 }, /* (231) cmd ::= SHOW QNODES */ + { 330, -2 }, /* (232) cmd ::= SHOW FUNCTIONS */ + { 330, -5 }, /* (233) cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ + { 330, -2 }, /* (234) cmd ::= SHOW STREAMS */ + { 330, -2 }, /* (235) cmd ::= SHOW ACCOUNTS */ + { 330, -2 }, /* (236) cmd ::= SHOW APPS */ + { 330, -2 }, /* (237) cmd ::= SHOW CONNECTIONS */ + { 330, -2 }, /* (238) cmd ::= SHOW LICENCES */ + { 330, -2 }, /* (239) cmd ::= SHOW GRANTS */ + { 330, -4 }, /* (240) cmd ::= SHOW CREATE DATABASE db_name */ + { 330, -4 }, /* (241) cmd ::= SHOW CREATE TABLE full_table_name */ + { 330, -4 }, /* (242) cmd ::= SHOW CREATE STABLE full_table_name */ + { 330, -2 }, /* (243) cmd ::= SHOW QUERIES */ + { 330, -2 }, /* (244) cmd ::= SHOW SCORES */ + { 330, -2 }, /* (245) cmd ::= SHOW TOPICS */ + { 330, -2 }, /* (246) cmd ::= SHOW VARIABLES */ + { 330, -3 }, /* (247) cmd ::= SHOW CLUSTER VARIABLES */ + { 330, -3 }, /* (248) cmd ::= SHOW LOCAL VARIABLES */ + { 330, -5 }, /* (249) cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ + { 330, -2 }, /* (250) cmd ::= SHOW BNODES */ + { 330, -2 }, /* (251) cmd ::= SHOW SNODES */ + { 330, -2 }, /* (252) cmd ::= SHOW CLUSTER */ + { 330, -2 }, /* (253) cmd ::= SHOW TRANSACTIONS */ + { 330, -4 }, /* (254) cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ + { 330, -2 }, /* (255) cmd ::= SHOW CONSUMERS */ + { 330, -2 }, /* (256) cmd ::= SHOW SUBSCRIPTIONS */ + { 330, -5 }, /* (257) cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ + { 330, -7 }, /* (258) cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ + { 330, -3 }, /* (259) cmd ::= SHOW VNODES NK_INTEGER */ + { 330, -3 }, /* (260) cmd ::= SHOW VNODES NK_STRING */ + { 330, -3 }, /* (261) cmd ::= SHOW db_name_cond_opt ALIVE */ + { 330, -3 }, /* (262) cmd ::= SHOW CLUSTER ALIVE */ + { 385, 0 }, /* (263) db_name_cond_opt ::= */ + { 385, -2 }, /* (264) db_name_cond_opt ::= db_name NK_DOT */ + { 386, 0 }, /* (265) like_pattern_opt ::= */ + { 386, -2 }, /* (266) like_pattern_opt ::= LIKE NK_STRING */ + { 387, -1 }, /* (267) table_name_cond ::= table_name */ + { 388, 0 }, /* (268) from_db_opt ::= */ + { 388, -2 }, /* (269) from_db_opt ::= FROM db_name */ + { 389, 0 }, /* (270) tag_list_opt ::= */ + { 389, -1 }, /* (271) tag_list_opt ::= tag_item */ + { 389, -3 }, /* (272) tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ + { 390, -1 }, /* (273) tag_item ::= TBNAME */ + { 390, -1 }, /* (274) tag_item ::= QTAGS */ + { 390, -1 }, /* (275) tag_item ::= column_name */ + { 390, -2 }, /* (276) tag_item ::= column_name column_alias */ + { 390, -3 }, /* (277) tag_item ::= column_name AS column_alias */ + { 330, -8 }, /* (278) cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ + { 330, -9 }, /* (279) cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ + { 330, -4 }, /* (280) cmd ::= DROP INDEX exists_opt full_index_name */ + { 392, -1 }, /* (281) full_index_name ::= index_name */ + { 392, -3 }, /* (282) full_index_name ::= db_name NK_DOT index_name */ + { 393, -10 }, /* (283) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ + { 393, -12 }, /* (284) index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ + { 395, -1 }, /* (285) func_list ::= func */ + { 395, -3 }, /* (286) func_list ::= func_list NK_COMMA func */ + { 398, -4 }, /* (287) func ::= sma_func_name NK_LP expression_list NK_RP */ + { 399, -1 }, /* (288) sma_func_name ::= function_name */ + { 399, -1 }, /* (289) sma_func_name ::= COUNT */ + { 399, -1 }, /* (290) sma_func_name ::= FIRST */ + { 399, -1 }, /* (291) sma_func_name ::= LAST */ + { 399, -1 }, /* (292) sma_func_name ::= LAST_ROW */ + { 397, 0 }, /* (293) sma_stream_opt ::= */ + { 397, -3 }, /* (294) sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ + { 397, -3 }, /* (295) sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ + { 397, -3 }, /* (296) sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ + { 330, -6 }, /* (297) cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ + { 330, -7 }, /* (298) cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ + { 330, -9 }, /* (299) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ + { 330, -7 }, /* (300) cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ + { 330, -9 }, /* (301) cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ + { 330, -4 }, /* (302) cmd ::= DROP TOPIC exists_opt topic_name */ + { 330, -7 }, /* (303) cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ + { 330, -2 }, /* (304) cmd ::= DESC full_table_name */ + { 330, -2 }, /* (305) cmd ::= DESCRIBE full_table_name */ + { 330, -3 }, /* (306) cmd ::= RESET QUERY CACHE */ + { 330, -4 }, /* (307) cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + { 330, -4 }, /* (308) cmd ::= EXPLAIN analyze_opt explain_options insert_query */ + { 402, 0 }, /* (309) analyze_opt ::= */ + { 402, -1 }, /* (310) analyze_opt ::= ANALYZE */ + { 403, 0 }, /* (311) explain_options ::= */ + { 403, -3 }, /* (312) explain_options ::= explain_options VERBOSE NK_BOOL */ + { 403, -3 }, /* (313) explain_options ::= explain_options RATIO NK_FLOAT */ + { 330, -12 }, /* (314) cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ + { 330, -4 }, /* (315) cmd ::= DROP FUNCTION exists_opt function_name */ + { 406, 0 }, /* (316) agg_func_opt ::= */ + { 406, -1 }, /* (317) agg_func_opt ::= AGGREGATE */ + { 407, 0 }, /* (318) bufsize_opt ::= */ + { 407, -2 }, /* (319) bufsize_opt ::= BUFSIZE NK_INTEGER */ + { 408, 0 }, /* (320) language_opt ::= */ + { 408, -2 }, /* (321) language_opt ::= LANGUAGE NK_STRING */ + { 405, 0 }, /* (322) or_replace_opt ::= */ + { 405, -2 }, /* (323) or_replace_opt ::= OR REPLACE */ + { 330, -12 }, /* (324) cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ + { 330, -4 }, /* (325) cmd ::= DROP STREAM exists_opt stream_name */ + { 411, 0 }, /* (326) col_list_opt ::= */ + { 411, -3 }, /* (327) col_list_opt ::= NK_LP col_name_list NK_RP */ + { 412, 0 }, /* (328) tag_def_or_ref_opt ::= */ + { 412, -1 }, /* (329) tag_def_or_ref_opt ::= tags_def */ + { 412, -4 }, /* (330) tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ + { 410, 0 }, /* (331) stream_options ::= */ + { 410, -3 }, /* (332) stream_options ::= stream_options TRIGGER AT_ONCE */ + { 410, -3 }, /* (333) stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ + { 410, -4 }, /* (334) stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ + { 410, -3 }, /* (335) stream_options ::= stream_options WATERMARK duration_literal */ + { 410, -4 }, /* (336) stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ + { 410, -3 }, /* (337) stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ + { 410, -3 }, /* (338) stream_options ::= stream_options DELETE_MARK duration_literal */ + { 410, -4 }, /* (339) stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ + { 413, 0 }, /* (340) subtable_opt ::= */ + { 413, -4 }, /* (341) subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + { 330, -3 }, /* (342) cmd ::= KILL CONNECTION NK_INTEGER */ + { 330, -3 }, /* (343) cmd ::= KILL QUERY NK_STRING */ + { 330, -3 }, /* (344) cmd ::= KILL TRANSACTION NK_INTEGER */ + { 330, -2 }, /* (345) cmd ::= BALANCE VGROUP */ + { 330, -3 }, /* (346) cmd ::= BALANCE VGROUP LEADER */ + { 330, -4 }, /* (347) cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + { 330, -4 }, /* (348) cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ + { 330, -3 }, /* (349) cmd ::= SPLIT VGROUP NK_INTEGER */ + { 415, -2 }, /* (350) dnode_list ::= DNODE NK_INTEGER */ + { 415, -3 }, /* (351) dnode_list ::= dnode_list DNODE NK_INTEGER */ + { 330, -4 }, /* (352) cmd ::= DELETE FROM full_table_name where_clause_opt */ + { 330, -1 }, /* (353) cmd ::= query_or_subquery */ + { 330, -1 }, /* (354) cmd ::= insert_query */ + { 404, -7 }, /* (355) insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ + { 404, -4 }, /* (356) insert_query ::= INSERT INTO full_table_name query_or_subquery */ + { 333, -1 }, /* (357) literal ::= NK_INTEGER */ + { 333, -1 }, /* (358) literal ::= NK_FLOAT */ + { 333, -1 }, /* (359) literal ::= NK_STRING */ + { 333, -1 }, /* (360) literal ::= NK_BOOL */ + { 333, -2 }, /* (361) literal ::= TIMESTAMP NK_STRING */ + { 333, -1 }, /* (362) literal ::= duration_literal */ + { 333, -1 }, /* (363) literal ::= NULL */ + { 333, -1 }, /* (364) literal ::= NK_QUESTION */ + { 381, -1 }, /* (365) duration_literal ::= NK_VARIABLE */ + { 417, -1 }, /* (366) signed ::= NK_INTEGER */ + { 417, -2 }, /* (367) signed ::= NK_PLUS NK_INTEGER */ + { 417, -2 }, /* (368) signed ::= NK_MINUS NK_INTEGER */ + { 417, -1 }, /* (369) signed ::= NK_FLOAT */ + { 417, -2 }, /* (370) signed ::= NK_PLUS NK_FLOAT */ + { 417, -2 }, /* (371) signed ::= NK_MINUS NK_FLOAT */ + { 371, -1 }, /* (372) signed_literal ::= signed */ + { 371, -1 }, /* (373) signed_literal ::= NK_STRING */ + { 371, -1 }, /* (374) signed_literal ::= NK_BOOL */ + { 371, -2 }, /* (375) signed_literal ::= TIMESTAMP NK_STRING */ + { 371, -1 }, /* (376) signed_literal ::= duration_literal */ + { 371, -1 }, /* (377) signed_literal ::= NULL */ + { 371, -1 }, /* (378) signed_literal ::= literal_func */ + { 371, -1 }, /* (379) signed_literal ::= NK_QUESTION */ + { 419, -1 }, /* (380) literal_list ::= signed_literal */ + { 419, -3 }, /* (381) literal_list ::= literal_list NK_COMMA signed_literal */ + { 342, -1 }, /* (382) db_name ::= NK_ID */ + { 343, -1 }, /* (383) table_name ::= NK_ID */ + { 369, -1 }, /* (384) column_name ::= NK_ID */ + { 383, -1 }, /* (385) function_name ::= NK_ID */ + { 420, -1 }, /* (386) table_alias ::= NK_ID */ + { 391, -1 }, /* (387) column_alias ::= NK_ID */ + { 335, -1 }, /* (388) user_name ::= NK_ID */ + { 344, -1 }, /* (389) topic_name ::= NK_ID */ + { 409, -1 }, /* (390) stream_name ::= NK_ID */ + { 401, -1 }, /* (391) cgroup_name ::= NK_ID */ + { 394, -1 }, /* (392) index_name ::= NK_ID */ + { 421, -1 }, /* (393) expr_or_subquery ::= expression */ + { 414, -1 }, /* (394) expression ::= literal */ + { 414, -1 }, /* (395) expression ::= pseudo_column */ + { 414, -1 }, /* (396) expression ::= column_reference */ + { 414, -1 }, /* (397) expression ::= function_expression */ + { 414, -1 }, /* (398) expression ::= case_when_expression */ + { 414, -3 }, /* (399) expression ::= NK_LP expression NK_RP */ + { 414, -2 }, /* (400) expression ::= NK_PLUS expr_or_subquery */ + { 414, -2 }, /* (401) expression ::= NK_MINUS expr_or_subquery */ + { 414, -3 }, /* (402) expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + { 414, -3 }, /* (403) expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + { 414, -3 }, /* (404) expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + { 414, -3 }, /* (405) expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + { 414, -3 }, /* (406) expression ::= expr_or_subquery NK_REM expr_or_subquery */ + { 414, -3 }, /* (407) expression ::= column_reference NK_ARROW NK_STRING */ + { 414, -3 }, /* (408) expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + { 414, -3 }, /* (409) expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + { 374, -1 }, /* (410) expression_list ::= expr_or_subquery */ + { 374, -3 }, /* (411) expression_list ::= expression_list NK_COMMA expr_or_subquery */ + { 423, -1 }, /* (412) column_reference ::= column_name */ + { 423, -3 }, /* (413) column_reference ::= table_name NK_DOT column_name */ + { 422, -1 }, /* (414) pseudo_column ::= ROWTS */ + { 422, -1 }, /* (415) pseudo_column ::= TBNAME */ + { 422, -3 }, /* (416) pseudo_column ::= table_name NK_DOT TBNAME */ + { 422, -1 }, /* (417) pseudo_column ::= QSTART */ + { 422, -1 }, /* (418) pseudo_column ::= QEND */ + { 422, -1 }, /* (419) pseudo_column ::= QDURATION */ + { 422, -1 }, /* (420) pseudo_column ::= WSTART */ + { 422, -1 }, /* (421) pseudo_column ::= WEND */ + { 422, -1 }, /* (422) pseudo_column ::= WDURATION */ + { 422, -1 }, /* (423) pseudo_column ::= IROWTS */ + { 422, -1 }, /* (424) pseudo_column ::= ISFILLED */ + { 422, -1 }, /* (425) pseudo_column ::= QTAGS */ + { 424, -4 }, /* (426) function_expression ::= function_name NK_LP expression_list NK_RP */ + { 424, -4 }, /* (427) function_expression ::= star_func NK_LP star_func_para_list NK_RP */ + { 424, -6 }, /* (428) function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ + { 424, -1 }, /* (429) function_expression ::= literal_func */ + { 418, -3 }, /* (430) literal_func ::= noarg_func NK_LP NK_RP */ + { 418, -1 }, /* (431) literal_func ::= NOW */ + { 428, -1 }, /* (432) noarg_func ::= NOW */ + { 428, -1 }, /* (433) noarg_func ::= TODAY */ + { 428, -1 }, /* (434) noarg_func ::= TIMEZONE */ + { 428, -1 }, /* (435) noarg_func ::= DATABASE */ + { 428, -1 }, /* (436) noarg_func ::= CLIENT_VERSION */ + { 428, -1 }, /* (437) noarg_func ::= SERVER_VERSION */ + { 428, -1 }, /* (438) noarg_func ::= SERVER_STATUS */ + { 428, -1 }, /* (439) noarg_func ::= CURRENT_USER */ + { 428, -1 }, /* (440) noarg_func ::= USER */ + { 426, -1 }, /* (441) star_func ::= COUNT */ + { 426, -1 }, /* (442) star_func ::= FIRST */ + { 426, -1 }, /* (443) star_func ::= LAST */ + { 426, -1 }, /* (444) star_func ::= LAST_ROW */ + { 427, -1 }, /* (445) star_func_para_list ::= NK_STAR */ + { 427, -1 }, /* (446) star_func_para_list ::= other_para_list */ + { 429, -1 }, /* (447) other_para_list ::= star_func_para */ + { 429, -3 }, /* (448) other_para_list ::= other_para_list NK_COMMA star_func_para */ + { 430, -1 }, /* (449) star_func_para ::= expr_or_subquery */ + { 430, -3 }, /* (450) star_func_para ::= table_name NK_DOT NK_STAR */ + { 425, -4 }, /* (451) case_when_expression ::= CASE when_then_list case_when_else_opt END */ + { 425, -5 }, /* (452) case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ + { 431, -1 }, /* (453) when_then_list ::= when_then_expr */ + { 431, -2 }, /* (454) when_then_list ::= when_then_list when_then_expr */ + { 434, -4 }, /* (455) when_then_expr ::= WHEN common_expression THEN common_expression */ + { 432, 0 }, /* (456) case_when_else_opt ::= */ + { 432, -2 }, /* (457) case_when_else_opt ::= ELSE common_expression */ + { 435, -3 }, /* (458) predicate ::= expr_or_subquery compare_op expr_or_subquery */ + { 435, -5 }, /* (459) predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + { 435, -6 }, /* (460) predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + { 435, -3 }, /* (461) predicate ::= expr_or_subquery IS NULL */ + { 435, -4 }, /* (462) predicate ::= expr_or_subquery IS NOT NULL */ + { 435, -3 }, /* (463) predicate ::= expr_or_subquery in_op in_predicate_value */ + { 436, -1 }, /* (464) compare_op ::= NK_LT */ + { 436, -1 }, /* (465) compare_op ::= NK_GT */ + { 436, -1 }, /* (466) compare_op ::= NK_LE */ + { 436, -1 }, /* (467) compare_op ::= NK_GE */ + { 436, -1 }, /* (468) compare_op ::= NK_NE */ + { 436, -1 }, /* (469) compare_op ::= NK_EQ */ + { 436, -1 }, /* (470) compare_op ::= LIKE */ + { 436, -2 }, /* (471) compare_op ::= NOT LIKE */ + { 436, -1 }, /* (472) compare_op ::= MATCH */ + { 436, -1 }, /* (473) compare_op ::= NMATCH */ + { 436, -1 }, /* (474) compare_op ::= CONTAINS */ + { 437, -1 }, /* (475) in_op ::= IN */ + { 437, -2 }, /* (476) in_op ::= NOT IN */ + { 438, -3 }, /* (477) in_predicate_value ::= NK_LP literal_list NK_RP */ + { 439, -1 }, /* (478) boolean_value_expression ::= boolean_primary */ + { 439, -2 }, /* (479) boolean_value_expression ::= NOT boolean_primary */ + { 439, -3 }, /* (480) boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + { 439, -3 }, /* (481) boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + { 440, -1 }, /* (482) boolean_primary ::= predicate */ + { 440, -3 }, /* (483) boolean_primary ::= NK_LP boolean_value_expression NK_RP */ + { 433, -1 }, /* (484) common_expression ::= expr_or_subquery */ + { 433, -1 }, /* (485) common_expression ::= boolean_value_expression */ + { 441, 0 }, /* (486) from_clause_opt ::= */ + { 441, -2 }, /* (487) from_clause_opt ::= FROM table_reference_list */ + { 442, -1 }, /* (488) table_reference_list ::= table_reference */ + { 442, -3 }, /* (489) table_reference_list ::= table_reference_list NK_COMMA table_reference */ + { 443, -1 }, /* (490) table_reference ::= table_primary */ + { 443, -1 }, /* (491) table_reference ::= joined_table */ + { 444, -2 }, /* (492) table_primary ::= table_name alias_opt */ + { 444, -4 }, /* (493) table_primary ::= db_name NK_DOT table_name alias_opt */ + { 444, -2 }, /* (494) table_primary ::= subquery alias_opt */ + { 444, -1 }, /* (495) table_primary ::= parenthesized_joined_table */ + { 446, 0 }, /* (496) alias_opt ::= */ + { 446, -1 }, /* (497) alias_opt ::= table_alias */ + { 446, -2 }, /* (498) alias_opt ::= AS table_alias */ + { 448, -3 }, /* (499) parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + { 448, -3 }, /* (500) parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ + { 445, -6 }, /* (501) joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ + { 449, 0 }, /* (502) join_type ::= */ + { 449, -1 }, /* (503) join_type ::= INNER */ + { 450, -12 }, /* (504) query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + { 451, 0 }, /* (505) set_quantifier_opt ::= */ + { 451, -1 }, /* (506) set_quantifier_opt ::= DISTINCT */ + { 451, -1 }, /* (507) set_quantifier_opt ::= ALL */ + { 452, -1 }, /* (508) select_list ::= select_item */ + { 452, -3 }, /* (509) select_list ::= select_list NK_COMMA select_item */ + { 460, -1 }, /* (510) select_item ::= NK_STAR */ + { 460, -1 }, /* (511) select_item ::= common_expression */ + { 460, -2 }, /* (512) select_item ::= common_expression column_alias */ + { 460, -3 }, /* (513) select_item ::= common_expression AS column_alias */ + { 460, -3 }, /* (514) select_item ::= table_name NK_DOT NK_STAR */ + { 416, 0 }, /* (515) where_clause_opt ::= */ + { 416, -2 }, /* (516) where_clause_opt ::= WHERE search_condition */ + { 453, 0 }, /* (517) partition_by_clause_opt ::= */ + { 453, -3 }, /* (518) partition_by_clause_opt ::= PARTITION BY partition_list */ + { 461, -1 }, /* (519) partition_list ::= partition_item */ + { 461, -3 }, /* (520) partition_list ::= partition_list NK_COMMA partition_item */ + { 462, -1 }, /* (521) partition_item ::= expr_or_subquery */ + { 462, -2 }, /* (522) partition_item ::= expr_or_subquery column_alias */ + { 462, -3 }, /* (523) partition_item ::= expr_or_subquery AS column_alias */ + { 457, 0 }, /* (524) twindow_clause_opt ::= */ + { 457, -6 }, /* (525) twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ + { 457, -4 }, /* (526) twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ + { 457, -6 }, /* (527) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ + { 457, -8 }, /* (528) twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ + { 457, -7 }, /* (529) twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ + { 396, 0 }, /* (530) sliding_opt ::= */ + { 396, -4 }, /* (531) sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ + { 456, 0 }, /* (532) fill_opt ::= */ + { 456, -4 }, /* (533) fill_opt ::= FILL NK_LP fill_mode NK_RP */ + { 456, -6 }, /* (534) fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ + { 456, -6 }, /* (535) fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ + { 463, -1 }, /* (536) fill_mode ::= NONE */ + { 463, -1 }, /* (537) fill_mode ::= PREV */ + { 463, -1 }, /* (538) fill_mode ::= NULL */ + { 463, -1 }, /* (539) fill_mode ::= NULL_F */ + { 463, -1 }, /* (540) fill_mode ::= LINEAR */ + { 463, -1 }, /* (541) fill_mode ::= NEXT */ + { 458, 0 }, /* (542) group_by_clause_opt ::= */ + { 458, -3 }, /* (543) group_by_clause_opt ::= GROUP BY group_by_list */ + { 464, -1 }, /* (544) group_by_list ::= expr_or_subquery */ + { 464, -3 }, /* (545) group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ + { 459, 0 }, /* (546) having_clause_opt ::= */ + { 459, -2 }, /* (547) having_clause_opt ::= HAVING search_condition */ + { 454, 0 }, /* (548) range_opt ::= */ + { 454, -6 }, /* (549) range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ + { 455, 0 }, /* (550) every_opt ::= */ + { 455, -4 }, /* (551) every_opt ::= EVERY NK_LP duration_literal NK_RP */ + { 465, -4 }, /* (552) query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + { 466, -1 }, /* (553) query_simple ::= query_specification */ + { 466, -1 }, /* (554) query_simple ::= union_query_expression */ + { 470, -4 }, /* (555) union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ + { 470, -3 }, /* (556) union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ + { 471, -1 }, /* (557) query_simple_or_subquery ::= query_simple */ + { 471, -1 }, /* (558) query_simple_or_subquery ::= subquery */ + { 400, -1 }, /* (559) query_or_subquery ::= query_expression */ + { 400, -1 }, /* (560) query_or_subquery ::= subquery */ + { 467, 0 }, /* (561) order_by_clause_opt ::= */ + { 467, -3 }, /* (562) order_by_clause_opt ::= ORDER BY sort_specification_list */ + { 468, 0 }, /* (563) slimit_clause_opt ::= */ + { 468, -2 }, /* (564) slimit_clause_opt ::= SLIMIT NK_INTEGER */ + { 468, -4 }, /* (565) slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + { 468, -4 }, /* (566) slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 469, 0 }, /* (567) limit_clause_opt ::= */ + { 469, -2 }, /* (568) limit_clause_opt ::= LIMIT NK_INTEGER */ + { 469, -4 }, /* (569) limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ + { 469, -4 }, /* (570) limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + { 447, -3 }, /* (571) subquery ::= NK_LP query_expression NK_RP */ + { 447, -3 }, /* (572) subquery ::= NK_LP subquery NK_RP */ + { 345, -1 }, /* (573) search_condition ::= common_expression */ + { 472, -1 }, /* (574) sort_specification_list ::= sort_specification */ + { 472, -3 }, /* (575) sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ + { 473, -3 }, /* (576) sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ + { 474, 0 }, /* (577) ordering_specification_opt ::= */ + { 474, -1 }, /* (578) ordering_specification_opt ::= ASC */ + { 474, -1 }, /* (579) ordering_specification_opt ::= DESC */ + { 475, 0 }, /* (580) null_ordering_opt ::= */ + { 475, -2 }, /* (581) null_ordering_opt ::= NULLS FIRST */ + { 475, -2 }, /* (582) null_ordering_opt ::= NULLS LAST */ }; static void yy_accept(yyParser*); /* Forward Declaration */ @@ -3874,1592 +3872,1597 @@ static YYACTIONTYPE yy_reduce( yy_destructor(yypParser,333,&yymsp[0].minor); break; case 24: /* cmd ::= CREATE USER user_name PASS NK_STRING sysinfo_opt */ -{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy129, &yymsp[-1].minor.yy0, yymsp[0].minor.yy353); } +{ pCxt->pRootNode = createCreateUserStmt(pCxt, &yymsp[-3].minor.yy113, &yymsp[-1].minor.yy0, yymsp[0].minor.yy551); } break; case 25: /* cmd ::= ALTER USER user_name PASS NK_STRING */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy129, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_PASSWD, &yymsp[0].minor.yy0); } break; case 26: /* cmd ::= ALTER USER user_name ENABLE NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy129, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_ENABLE, &yymsp[0].minor.yy0); } break; case 27: /* cmd ::= ALTER USER user_name SYSINFO NK_INTEGER */ -{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy129, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } +{ pCxt->pRootNode = createAlterUserStmt(pCxt, &yymsp[-2].minor.yy113, TSDB_ALTER_USER_SYSINFO, &yymsp[0].minor.yy0); } break; case 28: /* cmd ::= DROP USER user_name */ -{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy129); } +{ pCxt->pRootNode = createDropUserStmt(pCxt, &yymsp[0].minor.yy113); } break; case 29: /* sysinfo_opt ::= */ -{ yymsp[1].minor.yy353 = 1; } +{ yymsp[1].minor.yy551 = 1; } break; case 30: /* sysinfo_opt ::= SYSINFO NK_INTEGER */ -{ yymsp[-1].minor.yy353 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } +{ yymsp[-1].minor.yy551 = taosStr2Int8(yymsp[0].minor.yy0.z, NULL, 10); } break; - case 31: /* cmd ::= GRANT privileges ON priv_level TO user_name */ -{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-4].minor.yy359, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129); } + case 31: /* cmd ::= GRANT privileges ON priv_level with_opt TO user_name */ +{ pCxt->pRootNode = createGrantStmt(pCxt, yymsp[-5].minor.yy837, &yymsp[-3].minor.yy777, &yymsp[0].minor.yy113, yymsp[-2].minor.yy448); } break; - case 32: /* cmd ::= REVOKE privileges ON priv_level FROM user_name */ -{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-4].minor.yy359, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129); } + case 32: /* cmd ::= REVOKE privileges ON priv_level with_opt FROM user_name */ +{ pCxt->pRootNode = createRevokeStmt(pCxt, yymsp[-5].minor.yy837, &yymsp[-3].minor.yy777, &yymsp[0].minor.yy113, yymsp[-2].minor.yy448); } break; case 33: /* privileges ::= ALL */ -{ yymsp[0].minor.yy359 = PRIVILEGE_TYPE_ALL; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_ALL; } break; case 34: /* privileges ::= priv_type_list */ case 36: /* priv_type_list ::= priv_type */ yytestcase(yyruleno==36); -{ yylhsminor.yy359 = yymsp[0].minor.yy359; } - yymsp[0].minor.yy359 = yylhsminor.yy359; +{ yylhsminor.yy837 = yymsp[0].minor.yy837; } + yymsp[0].minor.yy837 = yylhsminor.yy837; break; case 35: /* privileges ::= SUBSCRIBE */ -{ yymsp[0].minor.yy359 = PRIVILEGE_TYPE_SUBSCRIBE; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_SUBSCRIBE; } break; case 37: /* priv_type_list ::= priv_type_list NK_COMMA priv_type */ -{ yylhsminor.yy359 = yymsp[-2].minor.yy359 | yymsp[0].minor.yy359; } - yymsp[-2].minor.yy359 = yylhsminor.yy359; +{ yylhsminor.yy837 = yymsp[-2].minor.yy837 | yymsp[0].minor.yy837; } + yymsp[-2].minor.yy837 = yylhsminor.yy837; break; case 38: /* priv_type ::= READ */ -{ yymsp[0].minor.yy359 = PRIVILEGE_TYPE_READ; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_READ; } break; case 39: /* priv_type ::= WRITE */ -{ yymsp[0].minor.yy359 = PRIVILEGE_TYPE_WRITE; } +{ yymsp[0].minor.yy837 = PRIVILEGE_TYPE_WRITE; } break; case 40: /* priv_level ::= NK_STAR NK_DOT NK_STAR */ -{ yylhsminor.yy129 = yymsp[-2].minor.yy0; } - yymsp[-2].minor.yy129 = yylhsminor.yy129; +{ yylhsminor.yy777.first = yymsp[-2].minor.yy0; yylhsminor.yy777.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy777 = yylhsminor.yy777; break; case 41: /* priv_level ::= db_name NK_DOT NK_STAR */ -{ yylhsminor.yy129 = yymsp[-2].minor.yy129; } - yymsp[-2].minor.yy129 = yylhsminor.yy129; - break; - case 42: /* priv_level ::= topic_name */ - case 286: /* sma_func_name ::= function_name */ yytestcase(yyruleno==286); - case 495: /* alias_opt ::= table_alias */ yytestcase(yyruleno==495); -{ yylhsminor.yy129 = yymsp[0].minor.yy129; } - yymsp[0].minor.yy129 = yylhsminor.yy129; - break; - case 43: /* cmd ::= CREATE DNODE dnode_endpoint */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy129, NULL); } - break; - case 44: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ -{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0); } - break; - case 45: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy337); } - break; - case 46: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ -{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy129, yymsp[0].minor.yy337); } - break; - case 47: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ +{ yylhsminor.yy777.first = yymsp[-2].minor.yy113; yylhsminor.yy777.second = yymsp[0].minor.yy0; } + yymsp[-2].minor.yy777 = yylhsminor.yy777; + break; + case 42: /* priv_level ::= db_name NK_DOT table_name */ +{ yylhsminor.yy777.first = yymsp[-2].minor.yy113; yylhsminor.yy777.second = yymsp[0].minor.yy113; } + yymsp[-2].minor.yy777 = yylhsminor.yy777; + break; + case 43: /* priv_level ::= topic_name */ +{ yylhsminor.yy777.first = yymsp[0].minor.yy113; yylhsminor.yy777.second = nil_token; } + yymsp[0].minor.yy777 = yylhsminor.yy777; + break; + case 44: /* with_opt ::= */ + case 137: /* start_opt ::= */ yytestcase(yyruleno==137); + case 141: /* end_opt ::= */ yytestcase(yyruleno==141); + case 265: /* like_pattern_opt ::= */ yytestcase(yyruleno==265); + case 340: /* subtable_opt ::= */ yytestcase(yyruleno==340); + case 456: /* case_when_else_opt ::= */ yytestcase(yyruleno==456); + case 486: /* from_clause_opt ::= */ yytestcase(yyruleno==486); + case 515: /* where_clause_opt ::= */ yytestcase(yyruleno==515); + case 524: /* twindow_clause_opt ::= */ yytestcase(yyruleno==524); + case 530: /* sliding_opt ::= */ yytestcase(yyruleno==530); + case 532: /* fill_opt ::= */ yytestcase(yyruleno==532); + case 546: /* having_clause_opt ::= */ yytestcase(yyruleno==546); + case 548: /* range_opt ::= */ yytestcase(yyruleno==548); + case 550: /* every_opt ::= */ yytestcase(yyruleno==550); + case 563: /* slimit_clause_opt ::= */ yytestcase(yyruleno==563); + case 567: /* limit_clause_opt ::= */ yytestcase(yyruleno==567); +{ yymsp[1].minor.yy448 = NULL; } + break; + case 45: /* with_opt ::= WITH search_condition */ + case 487: /* from_clause_opt ::= FROM table_reference_list */ yytestcase(yyruleno==487); + case 516: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==516); + case 547: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==547); +{ yymsp[-1].minor.yy448 = yymsp[0].minor.yy448; } + break; + case 46: /* cmd ::= CREATE DNODE dnode_endpoint */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[0].minor.yy113, NULL); } + break; + case 47: /* cmd ::= CREATE DNODE dnode_endpoint PORT NK_INTEGER */ +{ pCxt->pRootNode = createCreateDnodeStmt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } + break; + case 48: /* cmd ::= DROP DNODE NK_INTEGER force_opt */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy369); } + break; + case 49: /* cmd ::= DROP DNODE dnode_endpoint force_opt */ +{ pCxt->pRootNode = createDropDnodeStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy369); } + break; + case 50: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, NULL); } break; - case 48: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ + case 51: /* cmd ::= ALTER DNODE NK_INTEGER NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, &yymsp[-2].minor.yy0, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 49: /* cmd ::= ALTER ALL DNODES NK_STRING */ + case 52: /* cmd ::= ALTER ALL DNODES NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[0].minor.yy0, NULL); } break; - case 50: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ + case 53: /* cmd ::= ALTER ALL DNODES NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterDnodeStmt(pCxt, NULL, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 51: /* dnode_endpoint ::= NK_STRING */ - case 52: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==52); - case 53: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==53); - case 287: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==287); - case 288: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==288); - case 289: /* sma_func_name ::= LAST */ yytestcase(yyruleno==289); - case 290: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==290); - case 380: /* db_name ::= NK_ID */ yytestcase(yyruleno==380); - case 381: /* table_name ::= NK_ID */ yytestcase(yyruleno==381); - case 382: /* column_name ::= NK_ID */ yytestcase(yyruleno==382); - case 383: /* function_name ::= NK_ID */ yytestcase(yyruleno==383); - case 384: /* table_alias ::= NK_ID */ yytestcase(yyruleno==384); - case 385: /* column_alias ::= NK_ID */ yytestcase(yyruleno==385); - case 386: /* user_name ::= NK_ID */ yytestcase(yyruleno==386); - case 387: /* topic_name ::= NK_ID */ yytestcase(yyruleno==387); - case 388: /* stream_name ::= NK_ID */ yytestcase(yyruleno==388); - case 389: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==389); - case 390: /* index_name ::= NK_ID */ yytestcase(yyruleno==390); - case 430: /* noarg_func ::= NOW */ yytestcase(yyruleno==430); - case 431: /* noarg_func ::= TODAY */ yytestcase(yyruleno==431); - case 432: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==432); - case 433: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==433); - case 434: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==434); - case 435: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==435); - case 436: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==436); - case 437: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==437); - case 438: /* noarg_func ::= USER */ yytestcase(yyruleno==438); - case 439: /* star_func ::= COUNT */ yytestcase(yyruleno==439); - case 440: /* star_func ::= FIRST */ yytestcase(yyruleno==440); - case 441: /* star_func ::= LAST */ yytestcase(yyruleno==441); - case 442: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==442); -{ yylhsminor.yy129 = yymsp[0].minor.yy0; } - yymsp[0].minor.yy129 = yylhsminor.yy129; - break; - case 54: /* force_opt ::= */ - case 74: /* not_exists_opt ::= */ yytestcase(yyruleno==74); - case 76: /* exists_opt ::= */ yytestcase(yyruleno==76); - case 307: /* analyze_opt ::= */ yytestcase(yyruleno==307); - case 314: /* agg_func_opt ::= */ yytestcase(yyruleno==314); - case 320: /* or_replace_opt ::= */ yytestcase(yyruleno==320); - case 503: /* set_quantifier_opt ::= */ yytestcase(yyruleno==503); -{ yymsp[1].minor.yy337 = false; } - break; - case 55: /* force_opt ::= FORCE */ - case 308: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==308); - case 315: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==315); - case 504: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==504); -{ yymsp[0].minor.yy337 = true; } - break; - case 56: /* cmd ::= ALTER LOCAL NK_STRING */ + case 54: /* dnode_endpoint ::= NK_STRING */ + case 55: /* dnode_endpoint ::= NK_ID */ yytestcase(yyruleno==55); + case 56: /* dnode_endpoint ::= NK_IPTOKEN */ yytestcase(yyruleno==56); + case 289: /* sma_func_name ::= COUNT */ yytestcase(yyruleno==289); + case 290: /* sma_func_name ::= FIRST */ yytestcase(yyruleno==290); + case 291: /* sma_func_name ::= LAST */ yytestcase(yyruleno==291); + case 292: /* sma_func_name ::= LAST_ROW */ yytestcase(yyruleno==292); + case 382: /* db_name ::= NK_ID */ yytestcase(yyruleno==382); + case 383: /* table_name ::= NK_ID */ yytestcase(yyruleno==383); + case 384: /* column_name ::= NK_ID */ yytestcase(yyruleno==384); + case 385: /* function_name ::= NK_ID */ yytestcase(yyruleno==385); + case 386: /* table_alias ::= NK_ID */ yytestcase(yyruleno==386); + case 387: /* column_alias ::= NK_ID */ yytestcase(yyruleno==387); + case 388: /* user_name ::= NK_ID */ yytestcase(yyruleno==388); + case 389: /* topic_name ::= NK_ID */ yytestcase(yyruleno==389); + case 390: /* stream_name ::= NK_ID */ yytestcase(yyruleno==390); + case 391: /* cgroup_name ::= NK_ID */ yytestcase(yyruleno==391); + case 392: /* index_name ::= NK_ID */ yytestcase(yyruleno==392); + case 432: /* noarg_func ::= NOW */ yytestcase(yyruleno==432); + case 433: /* noarg_func ::= TODAY */ yytestcase(yyruleno==433); + case 434: /* noarg_func ::= TIMEZONE */ yytestcase(yyruleno==434); + case 435: /* noarg_func ::= DATABASE */ yytestcase(yyruleno==435); + case 436: /* noarg_func ::= CLIENT_VERSION */ yytestcase(yyruleno==436); + case 437: /* noarg_func ::= SERVER_VERSION */ yytestcase(yyruleno==437); + case 438: /* noarg_func ::= SERVER_STATUS */ yytestcase(yyruleno==438); + case 439: /* noarg_func ::= CURRENT_USER */ yytestcase(yyruleno==439); + case 440: /* noarg_func ::= USER */ yytestcase(yyruleno==440); + case 441: /* star_func ::= COUNT */ yytestcase(yyruleno==441); + case 442: /* star_func ::= FIRST */ yytestcase(yyruleno==442); + case 443: /* star_func ::= LAST */ yytestcase(yyruleno==443); + case 444: /* star_func ::= LAST_ROW */ yytestcase(yyruleno==444); +{ yylhsminor.yy113 = yymsp[0].minor.yy0; } + yymsp[0].minor.yy113 = yylhsminor.yy113; + break; + case 57: /* force_opt ::= */ + case 77: /* not_exists_opt ::= */ yytestcase(yyruleno==77); + case 79: /* exists_opt ::= */ yytestcase(yyruleno==79); + case 309: /* analyze_opt ::= */ yytestcase(yyruleno==309); + case 316: /* agg_func_opt ::= */ yytestcase(yyruleno==316); + case 322: /* or_replace_opt ::= */ yytestcase(yyruleno==322); + case 505: /* set_quantifier_opt ::= */ yytestcase(yyruleno==505); +{ yymsp[1].minor.yy369 = false; } + break; + case 58: /* force_opt ::= FORCE */ + case 310: /* analyze_opt ::= ANALYZE */ yytestcase(yyruleno==310); + case 317: /* agg_func_opt ::= AGGREGATE */ yytestcase(yyruleno==317); + case 506: /* set_quantifier_opt ::= DISTINCT */ yytestcase(yyruleno==506); +{ yymsp[0].minor.yy369 = true; } + break; + case 59: /* cmd ::= ALTER LOCAL NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 57: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ + case 60: /* cmd ::= ALTER LOCAL NK_STRING NK_STRING */ { pCxt->pRootNode = createAlterLocalStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 58: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ + case 61: /* cmd ::= CREATE QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_QNODE_STMT, &yymsp[0].minor.yy0); } break; - case 59: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ + case 62: /* cmd ::= DROP QNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_QNODE_STMT, &yymsp[0].minor.yy0); } break; - case 60: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ + case 63: /* cmd ::= CREATE BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_BNODE_STMT, &yymsp[0].minor.yy0); } break; - case 61: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ + case 64: /* cmd ::= DROP BNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_BNODE_STMT, &yymsp[0].minor.yy0); } break; - case 62: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ + case 65: /* cmd ::= CREATE SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_SNODE_STMT, &yymsp[0].minor.yy0); } break; - case 63: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ + case 66: /* cmd ::= DROP SNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_SNODE_STMT, &yymsp[0].minor.yy0); } break; - case 64: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ + case 67: /* cmd ::= CREATE MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createCreateComponentNodeStmt(pCxt, QUERY_NODE_CREATE_MNODE_STMT, &yymsp[0].minor.yy0); } break; - case 65: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ + case 68: /* cmd ::= DROP MNODE ON DNODE NK_INTEGER */ { pCxt->pRootNode = createDropComponentNodeStmt(pCxt, QUERY_NODE_DROP_MNODE_STMT, &yymsp[0].minor.yy0); } break; - case 66: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ -{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy337, &yymsp[-1].minor.yy129, yymsp[0].minor.yy712); } + case 69: /* cmd ::= CREATE DATABASE not_exists_opt db_name db_options */ +{ pCxt->pRootNode = createCreateDatabaseStmt(pCxt, yymsp[-2].minor.yy369, &yymsp[-1].minor.yy113, yymsp[0].minor.yy448); } break; - case 67: /* cmd ::= DROP DATABASE exists_opt db_name */ -{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy337, &yymsp[0].minor.yy129); } + case 70: /* cmd ::= DROP DATABASE exists_opt db_name */ +{ pCxt->pRootNode = createDropDatabaseStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 68: /* cmd ::= USE db_name */ -{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy129); } + case 71: /* cmd ::= USE db_name */ +{ pCxt->pRootNode = createUseDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } break; - case 69: /* cmd ::= ALTER DATABASE db_name alter_db_options */ -{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy129, yymsp[0].minor.yy712); } + case 72: /* cmd ::= ALTER DATABASE db_name alter_db_options */ +{ pCxt->pRootNode = createAlterDatabaseStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy448); } break; - case 70: /* cmd ::= FLUSH DATABASE db_name */ -{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy129); } + case 73: /* cmd ::= FLUSH DATABASE db_name */ +{ pCxt->pRootNode = createFlushDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } break; - case 71: /* cmd ::= TRIM DATABASE db_name speed_opt */ -{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy129, yymsp[0].minor.yy310); } + case 74: /* cmd ::= TRIM DATABASE db_name speed_opt */ +{ pCxt->pRootNode = createTrimDatabaseStmt(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy788); } break; - case 72: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ -{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy129, yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } + case 75: /* cmd ::= COMPACT DATABASE db_name start_opt end_opt */ +{ pCxt->pRootNode = createCompactStmt(pCxt, &yymsp[-2].minor.yy113, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 73: /* not_exists_opt ::= IF NOT EXISTS */ -{ yymsp[-2].minor.yy337 = true; } + case 76: /* not_exists_opt ::= IF NOT EXISTS */ +{ yymsp[-2].minor.yy369 = true; } break; - case 75: /* exists_opt ::= IF EXISTS */ - case 321: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==321); -{ yymsp[-1].minor.yy337 = true; } + case 78: /* exists_opt ::= IF EXISTS */ + case 323: /* or_replace_opt ::= OR REPLACE */ yytestcase(yyruleno==323); +{ yymsp[-1].minor.yy369 = true; } break; - case 77: /* db_options ::= */ -{ yymsp[1].minor.yy712 = createDefaultDatabaseOptions(pCxt); } + case 80: /* db_options ::= */ +{ yymsp[1].minor.yy448 = createDefaultDatabaseOptions(pCxt); } break; - case 78: /* db_options ::= db_options BUFFER NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 81: /* db_options ::= db_options BUFFER NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_BUFFER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 79: /* db_options ::= db_options CACHEMODEL NK_STRING */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 82: /* db_options ::= db_options CACHEMODEL NK_STRING */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_CACHEMODEL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 80: /* db_options ::= db_options CACHESIZE NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 83: /* db_options ::= db_options CACHESIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_CACHESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 81: /* db_options ::= db_options COMP NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_COMP, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 84: /* db_options ::= db_options COMP NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_COMP, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 82: /* db_options ::= db_options DURATION NK_INTEGER */ - case 83: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==83); -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 85: /* db_options ::= db_options DURATION NK_INTEGER */ + case 86: /* db_options ::= db_options DURATION NK_VARIABLE */ yytestcase(yyruleno==86); +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_DAYS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 84: /* db_options ::= db_options MAXROWS NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 87: /* db_options ::= db_options MAXROWS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_MAXROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 85: /* db_options ::= db_options MINROWS NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 88: /* db_options ::= db_options MINROWS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_MINROWS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 86: /* db_options ::= db_options KEEP integer_list */ - case 87: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==87); -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_KEEP, yymsp[0].minor.yy274); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 89: /* db_options ::= db_options KEEP integer_list */ + case 90: /* db_options ::= db_options KEEP variable_list */ yytestcase(yyruleno==90); +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_KEEP, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 88: /* db_options ::= db_options PAGES NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 91: /* db_options ::= db_options PAGES NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_PAGES, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 89: /* db_options ::= db_options PAGESIZE NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 92: /* db_options ::= db_options PAGESIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 90: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 93: /* db_options ::= db_options TSDB_PAGESIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TSDB_PAGESIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 91: /* db_options ::= db_options PRECISION NK_STRING */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 94: /* db_options ::= db_options PRECISION NK_STRING */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_PRECISION, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 92: /* db_options ::= db_options REPLICA NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 95: /* db_options ::= db_options REPLICA NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_REPLICA, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 93: /* db_options ::= db_options VGROUPS NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 96: /* db_options ::= db_options VGROUPS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_VGROUPS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 94: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 97: /* db_options ::= db_options SINGLE_STABLE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_SINGLE_STABLE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 95: /* db_options ::= db_options RETENTIONS retention_list */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_RETENTIONS, yymsp[0].minor.yy274); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 98: /* db_options ::= db_options RETENTIONS retention_list */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_RETENTIONS, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 96: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 99: /* db_options ::= db_options SCHEMALESS NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_SCHEMALESS, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 97: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_WAL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 100: /* db_options ::= db_options WAL_LEVEL NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 98: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 101: /* db_options ::= db_options WAL_FSYNC_PERIOD NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_FSYNC, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 99: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 102: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_RETENTION_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 100: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + case 103: /* db_options ::= db_options WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-3].minor.yy712, DB_OPTION_WAL_RETENTION_PERIOD, &t); + yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-3].minor.yy448, DB_OPTION_WAL_RETENTION_PERIOD, &t); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 101: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 104: /* db_options ::= db_options WAL_RETENTION_SIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_RETENTION_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 102: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + case 105: /* db_options ::= db_options WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-3].minor.yy712, DB_OPTION_WAL_RETENTION_SIZE, &t); + yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-3].minor.yy448, DB_OPTION_WAL_RETENTION_SIZE, &t); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 103: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 106: /* db_options ::= db_options WAL_ROLL_PERIOD NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_ROLL_PERIOD, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 104: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 107: /* db_options ::= db_options WAL_SEGMENT_SIZE NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_WAL_SEGMENT_SIZE, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 105: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 108: /* db_options ::= db_options STT_TRIGGER NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_STT_TRIGGER, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 106: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 109: /* db_options ::= db_options TABLE_PREFIX NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_PREFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 107: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ -{ yylhsminor.yy712 = setDatabaseOption(pCxt, yymsp[-2].minor.yy712, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 110: /* db_options ::= db_options TABLE_SUFFIX NK_INTEGER */ +{ yylhsminor.yy448 = setDatabaseOption(pCxt, yymsp[-2].minor.yy448, DB_OPTION_TABLE_SUFFIX, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 108: /* alter_db_options ::= alter_db_option */ -{ yylhsminor.yy712 = createAlterDatabaseOptions(pCxt); yylhsminor.yy712 = setAlterDatabaseOption(pCxt, yylhsminor.yy712, &yymsp[0].minor.yy595); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 111: /* alter_db_options ::= alter_db_option */ +{ yylhsminor.yy448 = createAlterDatabaseOptions(pCxt); yylhsminor.yy448 = setAlterDatabaseOption(pCxt, yylhsminor.yy448, &yymsp[0].minor.yy53); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 109: /* alter_db_options ::= alter_db_options alter_db_option */ -{ yylhsminor.yy712 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy712, &yymsp[0].minor.yy595); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 112: /* alter_db_options ::= alter_db_options alter_db_option */ +{ yylhsminor.yy448 = setAlterDatabaseOption(pCxt, yymsp[-1].minor.yy448, &yymsp[0].minor.yy53); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 110: /* alter_db_option ::= BUFFER NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 113: /* alter_db_option ::= BUFFER NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_BUFFER; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 111: /* alter_db_option ::= CACHEMODEL NK_STRING */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 114: /* alter_db_option ::= CACHEMODEL NK_STRING */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_CACHEMODEL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 112: /* alter_db_option ::= CACHESIZE NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 115: /* alter_db_option ::= CACHESIZE NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_CACHESIZE; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 113: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 116: /* alter_db_option ::= WAL_FSYNC_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_FSYNC; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 114: /* alter_db_option ::= KEEP integer_list */ - case 115: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==115); -{ yymsp[-1].minor.yy595.type = DB_OPTION_KEEP; yymsp[-1].minor.yy595.pList = yymsp[0].minor.yy274; } + case 117: /* alter_db_option ::= KEEP integer_list */ + case 118: /* alter_db_option ::= KEEP variable_list */ yytestcase(yyruleno==118); +{ yymsp[-1].minor.yy53.type = DB_OPTION_KEEP; yymsp[-1].minor.yy53.pList = yymsp[0].minor.yy432; } break; - case 116: /* alter_db_option ::= PAGES NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_PAGES; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 119: /* alter_db_option ::= PAGES NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_PAGES; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 117: /* alter_db_option ::= REPLICA NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 120: /* alter_db_option ::= REPLICA NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_REPLICA; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 118: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_WAL; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 121: /* alter_db_option ::= WAL_LEVEL NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 119: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 122: /* alter_db_option ::= STT_TRIGGER NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_STT_TRIGGER; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 120: /* alter_db_option ::= MINROWS NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 123: /* alter_db_option ::= MINROWS NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_MINROWS; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 121: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 124: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 122: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ + case 125: /* alter_db_option ::= WAL_RETENTION_PERIOD NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy595.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy595.val = t; + yymsp[-2].minor.yy53.type = DB_OPTION_WAL_RETENTION_PERIOD; yymsp[-2].minor.yy53.val = t; } break; - case 123: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 126: /* alter_db_option ::= WAL_RETENTION_SIZE NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 124: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ + case 127: /* alter_db_option ::= WAL_RETENTION_SIZE NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yymsp[-2].minor.yy595.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy595.val = t; + yymsp[-2].minor.yy53.type = DB_OPTION_WAL_RETENTION_SIZE; yymsp[-2].minor.yy53.val = t; } break; - case 125: /* integer_list ::= NK_INTEGER */ -{ yylhsminor.yy274 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy274 = yylhsminor.yy274; - break; - case 126: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ - case 349: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==349); -{ yylhsminor.yy274 = addNodeToList(pCxt, yymsp[-2].minor.yy274, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy274 = yylhsminor.yy274; - break; - case 127: /* variable_list ::= NK_VARIABLE */ -{ yylhsminor.yy274 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy274 = yylhsminor.yy274; - break; - case 128: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ -{ yylhsminor.yy274 = addNodeToList(pCxt, yymsp[-2].minor.yy274, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy274 = yylhsminor.yy274; - break; - case 129: /* retention_list ::= retention */ - case 159: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==159); - case 162: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==162); - case 169: /* column_def_list ::= column_def */ yytestcase(yyruleno==169); - case 213: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==213); - case 218: /* col_name_list ::= col_name */ yytestcase(yyruleno==218); - case 269: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==269); - case 283: /* func_list ::= func */ yytestcase(yyruleno==283); - case 378: /* literal_list ::= signed_literal */ yytestcase(yyruleno==378); - case 445: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==445); - case 451: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==451); - case 506: /* select_list ::= select_item */ yytestcase(yyruleno==506); - case 517: /* partition_list ::= partition_item */ yytestcase(yyruleno==517); - case 572: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==572); -{ yylhsminor.yy274 = createNodeList(pCxt, yymsp[0].minor.yy712); } - yymsp[0].minor.yy274 = yylhsminor.yy274; - break; - case 130: /* retention_list ::= retention_list NK_COMMA retention */ - case 163: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==163); - case 170: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==170); - case 214: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==214); - case 219: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==219); - case 270: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==270); - case 284: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==284); - case 379: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==379); - case 446: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==446); - case 507: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==507); - case 518: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==518); - case 573: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==573); -{ yylhsminor.yy274 = addNodeToList(pCxt, yymsp[-2].minor.yy274, yymsp[0].minor.yy712); } - yymsp[-2].minor.yy274 = yylhsminor.yy274; - break; - case 131: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ -{ yylhsminor.yy712 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 132: /* speed_opt ::= */ - case 316: /* bufsize_opt ::= */ yytestcase(yyruleno==316); -{ yymsp[1].minor.yy310 = 0; } - break; - case 133: /* speed_opt ::= MAX_SPEED NK_INTEGER */ - case 317: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==317); -{ yymsp[-1].minor.yy310 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } - break; - case 134: /* start_opt ::= */ - case 138: /* end_opt ::= */ yytestcase(yyruleno==138); - case 263: /* like_pattern_opt ::= */ yytestcase(yyruleno==263); - case 338: /* subtable_opt ::= */ yytestcase(yyruleno==338); - case 454: /* case_when_else_opt ::= */ yytestcase(yyruleno==454); - case 484: /* from_clause_opt ::= */ yytestcase(yyruleno==484); - case 513: /* where_clause_opt ::= */ yytestcase(yyruleno==513); - case 522: /* twindow_clause_opt ::= */ yytestcase(yyruleno==522); - case 528: /* sliding_opt ::= */ yytestcase(yyruleno==528); - case 530: /* fill_opt ::= */ yytestcase(yyruleno==530); - case 544: /* having_clause_opt ::= */ yytestcase(yyruleno==544); - case 546: /* range_opt ::= */ yytestcase(yyruleno==546); - case 548: /* every_opt ::= */ yytestcase(yyruleno==548); - case 561: /* slimit_clause_opt ::= */ yytestcase(yyruleno==561); - case 565: /* limit_clause_opt ::= */ yytestcase(yyruleno==565); -{ yymsp[1].minor.yy712 = NULL; } - break; - case 135: /* start_opt ::= START WITH NK_INTEGER */ - case 139: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==139); -{ yymsp[-2].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } - break; - case 136: /* start_opt ::= START WITH NK_STRING */ - case 140: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==140); -{ yymsp[-2].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 137: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ - case 141: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==141); -{ yymsp[-3].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 142: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ - case 144: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==144); -{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy337, yymsp[-5].minor.yy712, yymsp[-3].minor.yy274, yymsp[-1].minor.yy274, yymsp[0].minor.yy712); } - break; - case 143: /* cmd ::= CREATE TABLE multi_create_clause */ -{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy274); } - break; - case 145: /* cmd ::= DROP TABLE multi_drop_clause */ -{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy274); } - break; - case 146: /* cmd ::= DROP STABLE exists_opt full_table_name */ -{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy337, yymsp[0].minor.yy712); } - break; - case 147: /* cmd ::= ALTER TABLE alter_table_clause */ - case 351: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==351); - case 352: /* cmd ::= insert_query */ yytestcase(yyruleno==352); -{ pCxt->pRootNode = yymsp[0].minor.yy712; } - break; - case 148: /* cmd ::= ALTER STABLE alter_table_clause */ -{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy712); } - break; - case 149: /* alter_table_clause ::= full_table_name alter_table_options */ -{ yylhsminor.yy712 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; - break; - case 150: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ -{ yylhsminor.yy712 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy712, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy129, yymsp[0].minor.yy184); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 151: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ -{ yylhsminor.yy712 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy712, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy129); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; - break; - case 152: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ -{ yylhsminor.yy712 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy712, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy129, yymsp[0].minor.yy184); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 153: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ -{ yylhsminor.yy712 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy712, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 154: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ -{ yylhsminor.yy712 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy712, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy129, yymsp[0].minor.yy184); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 155: /* alter_table_clause ::= full_table_name DROP TAG column_name */ -{ yylhsminor.yy712 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy712, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy129); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; - break; - case 156: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ -{ yylhsminor.yy712 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy712, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy129, yymsp[0].minor.yy184); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 157: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ -{ yylhsminor.yy712 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy712, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 158: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ -{ yylhsminor.yy712 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy712, &yymsp[-2].minor.yy129, yymsp[0].minor.yy712); } - yymsp[-5].minor.yy712 = yylhsminor.yy712; + case 128: /* integer_list ::= NK_INTEGER */ +{ yylhsminor.yy432 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 129: /* integer_list ::= integer_list NK_COMMA NK_INTEGER */ + case 351: /* dnode_list ::= dnode_list DNODE NK_INTEGER */ yytestcase(yyruleno==351); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; + break; + case 130: /* variable_list ::= NK_VARIABLE */ +{ yylhsminor.yy432 = createNodeList(pCxt, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 131: /* variable_list ::= variable_list NK_COMMA NK_VARIABLE */ +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; + break; + case 132: /* retention_list ::= retention */ + case 162: /* multi_create_clause ::= create_subtable_clause */ yytestcase(yyruleno==162); + case 165: /* multi_drop_clause ::= drop_table_clause */ yytestcase(yyruleno==165); + case 172: /* column_def_list ::= column_def */ yytestcase(yyruleno==172); + case 215: /* rollup_func_list ::= rollup_func_name */ yytestcase(yyruleno==215); + case 220: /* col_name_list ::= col_name */ yytestcase(yyruleno==220); + case 271: /* tag_list_opt ::= tag_item */ yytestcase(yyruleno==271); + case 285: /* func_list ::= func */ yytestcase(yyruleno==285); + case 380: /* literal_list ::= signed_literal */ yytestcase(yyruleno==380); + case 447: /* other_para_list ::= star_func_para */ yytestcase(yyruleno==447); + case 453: /* when_then_list ::= when_then_expr */ yytestcase(yyruleno==453); + case 508: /* select_list ::= select_item */ yytestcase(yyruleno==508); + case 519: /* partition_list ::= partition_item */ yytestcase(yyruleno==519); + case 574: /* sort_specification_list ::= sort_specification */ yytestcase(yyruleno==574); +{ yylhsminor.yy432 = createNodeList(pCxt, yymsp[0].minor.yy448); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 133: /* retention_list ::= retention_list NK_COMMA retention */ + case 166: /* multi_drop_clause ::= multi_drop_clause NK_COMMA drop_table_clause */ yytestcase(yyruleno==166); + case 173: /* column_def_list ::= column_def_list NK_COMMA column_def */ yytestcase(yyruleno==173); + case 216: /* rollup_func_list ::= rollup_func_list NK_COMMA rollup_func_name */ yytestcase(yyruleno==216); + case 221: /* col_name_list ::= col_name_list NK_COMMA col_name */ yytestcase(yyruleno==221); + case 272: /* tag_list_opt ::= tag_list_opt NK_COMMA tag_item */ yytestcase(yyruleno==272); + case 286: /* func_list ::= func_list NK_COMMA func */ yytestcase(yyruleno==286); + case 381: /* literal_list ::= literal_list NK_COMMA signed_literal */ yytestcase(yyruleno==381); + case 448: /* other_para_list ::= other_para_list NK_COMMA star_func_para */ yytestcase(yyruleno==448); + case 509: /* select_list ::= select_list NK_COMMA select_item */ yytestcase(yyruleno==509); + case 520: /* partition_list ::= partition_list NK_COMMA partition_item */ yytestcase(yyruleno==520); + case 575: /* sort_specification_list ::= sort_specification_list NK_COMMA sort_specification */ yytestcase(yyruleno==575); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, yymsp[0].minor.yy448); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; + break; + case 134: /* retention ::= NK_VARIABLE NK_COLON NK_VARIABLE */ +{ yylhsminor.yy448 = createNodeListNodeEx(pCxt, createDurationValueNode(pCxt, &yymsp[-2].minor.yy0), createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 135: /* speed_opt ::= */ + case 318: /* bufsize_opt ::= */ yytestcase(yyruleno==318); +{ yymsp[1].minor.yy788 = 0; } + break; + case 136: /* speed_opt ::= MAX_SPEED NK_INTEGER */ + case 319: /* bufsize_opt ::= BUFSIZE NK_INTEGER */ yytestcase(yyruleno==319); +{ yymsp[-1].minor.yy788 = taosStr2Int32(yymsp[0].minor.yy0.z, NULL, 10); } + break; + case 138: /* start_opt ::= START WITH NK_INTEGER */ + case 142: /* end_opt ::= END WITH NK_INTEGER */ yytestcase(yyruleno==142); +{ yymsp[-2].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0); } + break; + case 139: /* start_opt ::= START WITH NK_STRING */ + case 143: /* end_opt ::= END WITH NK_STRING */ yytestcase(yyruleno==143); +{ yymsp[-2].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 140: /* start_opt ::= START WITH TIMESTAMP NK_STRING */ + case 144: /* end_opt ::= END WITH TIMESTAMP NK_STRING */ yytestcase(yyruleno==144); +{ yymsp[-3].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 145: /* cmd ::= CREATE TABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def_opt table_options */ + case 147: /* cmd ::= CREATE STABLE not_exists_opt full_table_name NK_LP column_def_list NK_RP tags_def table_options */ yytestcase(yyruleno==147); +{ pCxt->pRootNode = createCreateTableStmt(pCxt, yymsp[-6].minor.yy369, yymsp[-5].minor.yy448, yymsp[-3].minor.yy432, yymsp[-1].minor.yy432, yymsp[0].minor.yy448); } + break; + case 146: /* cmd ::= CREATE TABLE multi_create_clause */ +{ pCxt->pRootNode = createCreateMultiTableStmt(pCxt, yymsp[0].minor.yy432); } + break; + case 148: /* cmd ::= DROP TABLE multi_drop_clause */ +{ pCxt->pRootNode = createDropTableStmt(pCxt, yymsp[0].minor.yy432); } + break; + case 149: /* cmd ::= DROP STABLE exists_opt full_table_name */ +{ pCxt->pRootNode = createDropSuperTableStmt(pCxt, yymsp[-1].minor.yy369, yymsp[0].minor.yy448); } + break; + case 150: /* cmd ::= ALTER TABLE alter_table_clause */ + case 353: /* cmd ::= query_or_subquery */ yytestcase(yyruleno==353); + case 354: /* cmd ::= insert_query */ yytestcase(yyruleno==354); +{ pCxt->pRootNode = yymsp[0].minor.yy448; } + break; + case 151: /* cmd ::= ALTER STABLE alter_table_clause */ +{ pCxt->pRootNode = setAlterSuperTableType(yymsp[0].minor.yy448); } + break; + case 152: /* alter_table_clause ::= full_table_name alter_table_options */ +{ yylhsminor.yy448 = createAlterTableModifyOptions(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; + break; + case 153: /* alter_table_clause ::= full_table_name ADD COLUMN column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_ADD_COLUMN, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 154: /* alter_table_clause ::= full_table_name DROP COLUMN column_name */ +{ yylhsminor.yy448 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy448, TSDB_ALTER_TABLE_DROP_COLUMN, &yymsp[0].minor.yy113); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 155: /* alter_table_clause ::= full_table_name MODIFY COLUMN column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_COLUMN_BYTES, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 156: /* alter_table_clause ::= full_table_name RENAME COLUMN column_name column_name */ +{ yylhsminor.yy448 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_COLUMN_NAME, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 157: /* alter_table_clause ::= full_table_name ADD TAG column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_ADD_TAG, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 158: /* alter_table_clause ::= full_table_name DROP TAG column_name */ +{ yylhsminor.yy448 = createAlterTableDropCol(pCxt, yymsp[-3].minor.yy448, TSDB_ALTER_TABLE_DROP_TAG, &yymsp[0].minor.yy113); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 159: /* alter_table_clause ::= full_table_name MODIFY TAG column_name type_name */ +{ yylhsminor.yy448 = createAlterTableAddModifyCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_TAG_BYTES, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 160: /* alter_table_clause ::= full_table_name RENAME TAG column_name column_name */ +{ yylhsminor.yy448 = createAlterTableRenameCol(pCxt, yymsp[-4].minor.yy448, TSDB_ALTER_TABLE_UPDATE_TAG_NAME, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 160: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ - case 452: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==452); -{ yylhsminor.yy274 = addNodeToList(pCxt, yymsp[-1].minor.yy274, yymsp[0].minor.yy712); } - yymsp[-1].minor.yy274 = yylhsminor.yy274; + case 161: /* alter_table_clause ::= full_table_name SET TAG column_name NK_EQ signed_literal */ +{ yylhsminor.yy448 = createAlterTableSetTag(pCxt, yymsp[-5].minor.yy448, &yymsp[-2].minor.yy113, yymsp[0].minor.yy448); } + yymsp[-5].minor.yy448 = yylhsminor.yy448; break; - case 161: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ -{ yylhsminor.yy712 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy337, yymsp[-8].minor.yy712, yymsp[-6].minor.yy712, yymsp[-5].minor.yy274, yymsp[-2].minor.yy274, yymsp[0].minor.yy712); } - yymsp[-9].minor.yy712 = yylhsminor.yy712; + case 163: /* multi_create_clause ::= multi_create_clause create_subtable_clause */ + case 454: /* when_then_list ::= when_then_list when_then_expr */ yytestcase(yyruleno==454); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-1].minor.yy432, yymsp[0].minor.yy448); } + yymsp[-1].minor.yy432 = yylhsminor.yy432; break; - case 164: /* drop_table_clause ::= exists_opt full_table_name */ -{ yylhsminor.yy712 = createDropTableClause(pCxt, yymsp[-1].minor.yy337, yymsp[0].minor.yy712); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 164: /* create_subtable_clause ::= not_exists_opt full_table_name USING full_table_name specific_cols_opt TAGS NK_LP expression_list NK_RP table_options */ +{ yylhsminor.yy448 = createCreateSubTableClause(pCxt, yymsp[-9].minor.yy369, yymsp[-8].minor.yy448, yymsp[-6].minor.yy448, yymsp[-5].minor.yy432, yymsp[-2].minor.yy432, yymsp[0].minor.yy448); } + yymsp[-9].minor.yy448 = yylhsminor.yy448; break; - case 165: /* specific_cols_opt ::= */ - case 196: /* tags_def_opt ::= */ yytestcase(yyruleno==196); - case 268: /* tag_list_opt ::= */ yytestcase(yyruleno==268); - case 324: /* col_list_opt ::= */ yytestcase(yyruleno==324); - case 326: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==326); - case 515: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==515); - case 540: /* group_by_clause_opt ::= */ yytestcase(yyruleno==540); - case 559: /* order_by_clause_opt ::= */ yytestcase(yyruleno==559); -{ yymsp[1].minor.yy274 = NULL; } + case 167: /* drop_table_clause ::= exists_opt full_table_name */ +{ yylhsminor.yy448 = createDropTableClause(pCxt, yymsp[-1].minor.yy369, yymsp[0].minor.yy448); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 166: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ - case 325: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==325); -{ yymsp[-2].minor.yy274 = yymsp[-1].minor.yy274; } + case 168: /* specific_cols_opt ::= */ + case 198: /* tags_def_opt ::= */ yytestcase(yyruleno==198); + case 270: /* tag_list_opt ::= */ yytestcase(yyruleno==270); + case 326: /* col_list_opt ::= */ yytestcase(yyruleno==326); + case 328: /* tag_def_or_ref_opt ::= */ yytestcase(yyruleno==328); + case 517: /* partition_by_clause_opt ::= */ yytestcase(yyruleno==517); + case 542: /* group_by_clause_opt ::= */ yytestcase(yyruleno==542); + case 561: /* order_by_clause_opt ::= */ yytestcase(yyruleno==561); +{ yymsp[1].minor.yy432 = NULL; } break; - case 167: /* full_table_name ::= table_name */ -{ yylhsminor.yy712 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy129, NULL); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 169: /* specific_cols_opt ::= NK_LP col_name_list NK_RP */ + case 327: /* col_list_opt ::= NK_LP col_name_list NK_RP */ yytestcase(yyruleno==327); +{ yymsp[-2].minor.yy432 = yymsp[-1].minor.yy432; } break; - case 168: /* full_table_name ::= db_name NK_DOT table_name */ -{ yylhsminor.yy712 = createRealTableNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129, NULL); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 170: /* full_table_name ::= table_name */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, NULL, &yymsp[0].minor.yy113, NULL); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 171: /* column_def ::= column_name type_name */ -{ yylhsminor.yy712 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy129, yymsp[0].minor.yy184, NULL); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 171: /* full_table_name ::= db_name NK_DOT table_name */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 172: /* column_def ::= column_name type_name COMMENT NK_STRING */ -{ yylhsminor.yy712 = createColumnDefNode(pCxt, &yymsp[-3].minor.yy129, yymsp[-2].minor.yy184, &yymsp[0].minor.yy0); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 174: /* column_def ::= column_name type_name */ +{ yylhsminor.yy448 = createColumnDefNode(pCxt, &yymsp[-1].minor.yy113, yymsp[0].minor.yy728, NULL); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 173: /* type_name ::= BOOL */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_BOOL); } + case 175: /* type_name ::= BOOL */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_BOOL); } break; - case 174: /* type_name ::= TINYINT */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_TINYINT); } + case 176: /* type_name ::= TINYINT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_TINYINT); } break; - case 175: /* type_name ::= SMALLINT */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_SMALLINT); } + case 177: /* type_name ::= SMALLINT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_SMALLINT); } break; - case 176: /* type_name ::= INT */ - case 177: /* type_name ::= INTEGER */ yytestcase(yyruleno==177); -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_INT); } + case 178: /* type_name ::= INT */ + case 179: /* type_name ::= INTEGER */ yytestcase(yyruleno==179); +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_INT); } break; - case 178: /* type_name ::= BIGINT */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_BIGINT); } + case 180: /* type_name ::= BIGINT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_BIGINT); } break; - case 179: /* type_name ::= FLOAT */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_FLOAT); } + case 181: /* type_name ::= FLOAT */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_FLOAT); } break; - case 180: /* type_name ::= DOUBLE */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_DOUBLE); } + case 182: /* type_name ::= DOUBLE */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_DOUBLE); } break; - case 181: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy184 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } + case 183: /* type_name ::= BINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_BINARY, &yymsp[-1].minor.yy0); } break; - case 182: /* type_name ::= TIMESTAMP */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } + case 184: /* type_name ::= TIMESTAMP */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_TIMESTAMP); } break; - case 183: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy184 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } + case 185: /* type_name ::= NCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_NCHAR, &yymsp[-1].minor.yy0); } break; - case 184: /* type_name ::= TINYINT UNSIGNED */ -{ yymsp[-1].minor.yy184 = createDataType(TSDB_DATA_TYPE_UTINYINT); } + case 186: /* type_name ::= TINYINT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_UTINYINT); } break; - case 185: /* type_name ::= SMALLINT UNSIGNED */ -{ yymsp[-1].minor.yy184 = createDataType(TSDB_DATA_TYPE_USMALLINT); } + case 187: /* type_name ::= SMALLINT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_USMALLINT); } break; - case 186: /* type_name ::= INT UNSIGNED */ -{ yymsp[-1].minor.yy184 = createDataType(TSDB_DATA_TYPE_UINT); } + case 188: /* type_name ::= INT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_UINT); } break; - case 187: /* type_name ::= BIGINT UNSIGNED */ -{ yymsp[-1].minor.yy184 = createDataType(TSDB_DATA_TYPE_UBIGINT); } + case 189: /* type_name ::= BIGINT UNSIGNED */ +{ yymsp[-1].minor.yy728 = createDataType(TSDB_DATA_TYPE_UBIGINT); } break; - case 188: /* type_name ::= JSON */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_JSON); } + case 190: /* type_name ::= JSON */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_JSON); } break; - case 189: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy184 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } + case 191: /* type_name ::= VARCHAR NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_VARCHAR, &yymsp[-1].minor.yy0); } break; - case 190: /* type_name ::= MEDIUMBLOB */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } + case 192: /* type_name ::= MEDIUMBLOB */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_MEDIUMBLOB); } break; - case 191: /* type_name ::= BLOB */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_BLOB); } + case 193: /* type_name ::= BLOB */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_BLOB); } break; - case 192: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy184 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } + case 194: /* type_name ::= VARBINARY NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createVarLenDataType(TSDB_DATA_TYPE_VARBINARY, &yymsp[-1].minor.yy0); } break; - case 193: /* type_name ::= DECIMAL */ -{ yymsp[0].minor.yy184 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 195: /* type_name ::= DECIMAL */ +{ yymsp[0].minor.yy728 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 194: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ -{ yymsp[-3].minor.yy184 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 196: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_RP */ +{ yymsp[-3].minor.yy728 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 195: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ -{ yymsp[-5].minor.yy184 = createDataType(TSDB_DATA_TYPE_DECIMAL); } + case 197: /* type_name ::= DECIMAL NK_LP NK_INTEGER NK_COMMA NK_INTEGER NK_RP */ +{ yymsp[-5].minor.yy728 = createDataType(TSDB_DATA_TYPE_DECIMAL); } break; - case 197: /* tags_def_opt ::= tags_def */ - case 327: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==327); - case 444: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==444); -{ yylhsminor.yy274 = yymsp[0].minor.yy274; } - yymsp[0].minor.yy274 = yylhsminor.yy274; + case 199: /* tags_def_opt ::= tags_def */ + case 329: /* tag_def_or_ref_opt ::= tags_def */ yytestcase(yyruleno==329); + case 446: /* star_func_para_list ::= other_para_list */ yytestcase(yyruleno==446); +{ yylhsminor.yy432 = yymsp[0].minor.yy432; } + yymsp[0].minor.yy432 = yylhsminor.yy432; break; - case 198: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ - case 328: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==328); -{ yymsp[-3].minor.yy274 = yymsp[-1].minor.yy274; } + case 200: /* tags_def ::= TAGS NK_LP column_def_list NK_RP */ + case 330: /* tag_def_or_ref_opt ::= TAGS NK_LP col_name_list NK_RP */ yytestcase(yyruleno==330); +{ yymsp[-3].minor.yy432 = yymsp[-1].minor.yy432; } break; - case 199: /* table_options ::= */ -{ yymsp[1].minor.yy712 = createDefaultTableOptions(pCxt); } + case 201: /* table_options ::= */ +{ yymsp[1].minor.yy448 = createDefaultTableOptions(pCxt); } break; - case 200: /* table_options ::= table_options COMMENT NK_STRING */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-2].minor.yy712, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 202: /* table_options ::= table_options COMMENT NK_STRING */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_COMMENT, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 201: /* table_options ::= table_options MAX_DELAY duration_list */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-2].minor.yy712, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy274); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 203: /* table_options ::= table_options MAX_DELAY duration_list */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_MAXDELAY, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 202: /* table_options ::= table_options WATERMARK duration_list */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-2].minor.yy712, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy274); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 204: /* table_options ::= table_options WATERMARK duration_list */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_WATERMARK, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 203: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-4].minor.yy712, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy274); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; + case 205: /* table_options ::= table_options ROLLUP NK_LP rollup_func_list NK_RP */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-4].minor.yy448, TABLE_OPTION_ROLLUP, yymsp[-1].minor.yy432); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 204: /* table_options ::= table_options TTL NK_INTEGER */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-2].minor.yy712, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 206: /* table_options ::= table_options TTL NK_INTEGER */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_TTL, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 205: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-4].minor.yy712, TABLE_OPTION_SMA, yymsp[-1].minor.yy274); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; + case 207: /* table_options ::= table_options SMA NK_LP col_name_list NK_RP */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-4].minor.yy448, TABLE_OPTION_SMA, yymsp[-1].minor.yy432); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 206: /* table_options ::= table_options DELETE_MARK duration_list */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-2].minor.yy712, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy274); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 208: /* table_options ::= table_options DELETE_MARK duration_list */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-2].minor.yy448, TABLE_OPTION_DELETE_MARK, yymsp[0].minor.yy432); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 207: /* alter_table_options ::= alter_table_option */ -{ yylhsminor.yy712 = createAlterTableOptions(pCxt); yylhsminor.yy712 = setTableOption(pCxt, yylhsminor.yy712, yymsp[0].minor.yy595.type, &yymsp[0].minor.yy595.val); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 209: /* alter_table_options ::= alter_table_option */ +{ yylhsminor.yy448 = createAlterTableOptions(pCxt); yylhsminor.yy448 = setTableOption(pCxt, yylhsminor.yy448, yymsp[0].minor.yy53.type, &yymsp[0].minor.yy53.val); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 208: /* alter_table_options ::= alter_table_options alter_table_option */ -{ yylhsminor.yy712 = setTableOption(pCxt, yymsp[-1].minor.yy712, yymsp[0].minor.yy595.type, &yymsp[0].minor.yy595.val); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 210: /* alter_table_options ::= alter_table_options alter_table_option */ +{ yylhsminor.yy448 = setTableOption(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy53.type, &yymsp[0].minor.yy53.val); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 209: /* alter_table_option ::= COMMENT NK_STRING */ -{ yymsp[-1].minor.yy595.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 211: /* alter_table_option ::= COMMENT NK_STRING */ +{ yymsp[-1].minor.yy53.type = TABLE_OPTION_COMMENT; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 210: /* alter_table_option ::= TTL NK_INTEGER */ -{ yymsp[-1].minor.yy595.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy595.val = yymsp[0].minor.yy0; } + case 212: /* alter_table_option ::= TTL NK_INTEGER */ +{ yymsp[-1].minor.yy53.type = TABLE_OPTION_TTL; yymsp[-1].minor.yy53.val = yymsp[0].minor.yy0; } break; - case 211: /* duration_list ::= duration_literal */ - case 408: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==408); -{ yylhsminor.yy274 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); } - yymsp[0].minor.yy274 = yylhsminor.yy274; + case 213: /* duration_list ::= duration_literal */ + case 410: /* expression_list ::= expr_or_subquery */ yytestcase(yyruleno==410); +{ yylhsminor.yy432 = createNodeList(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; break; - case 212: /* duration_list ::= duration_list NK_COMMA duration_literal */ - case 409: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==409); -{ yylhsminor.yy274 = addNodeToList(pCxt, yymsp[-2].minor.yy274, releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); } - yymsp[-2].minor.yy274 = yylhsminor.yy274; + case 214: /* duration_list ::= duration_list NK_COMMA duration_literal */ + case 411: /* expression_list ::= expression_list NK_COMMA expr_or_subquery */ yytestcase(yyruleno==411); +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; break; - case 215: /* rollup_func_name ::= function_name */ -{ yylhsminor.yy712 = createFunctionNode(pCxt, &yymsp[0].minor.yy129, NULL); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 217: /* rollup_func_name ::= function_name */ +{ yylhsminor.yy448 = createFunctionNode(pCxt, &yymsp[0].minor.yy113, NULL); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 216: /* rollup_func_name ::= FIRST */ - case 217: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==217); - case 272: /* tag_item ::= QTAGS */ yytestcase(yyruleno==272); -{ yylhsminor.yy712 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 218: /* rollup_func_name ::= FIRST */ + case 219: /* rollup_func_name ::= LAST */ yytestcase(yyruleno==219); + case 274: /* tag_item ::= QTAGS */ yytestcase(yyruleno==274); +{ yylhsminor.yy448 = createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 220: /* col_name ::= column_name */ - case 273: /* tag_item ::= column_name */ yytestcase(yyruleno==273); -{ yylhsminor.yy712 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy129); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 222: /* col_name ::= column_name */ + case 275: /* tag_item ::= column_name */ yytestcase(yyruleno==275); +{ yylhsminor.yy448 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy113); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 221: /* cmd ::= SHOW DNODES */ + case 223: /* cmd ::= SHOW DNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DNODES_STMT); } break; - case 222: /* cmd ::= SHOW USERS */ + case 224: /* cmd ::= SHOW USERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USERS_STMT); } break; - case 223: /* cmd ::= SHOW USER PRIVILEGES */ + case 225: /* cmd ::= SHOW USER PRIVILEGES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_USER_PRIVILEGES_STMT); } break; - case 224: /* cmd ::= SHOW DATABASES */ + case 226: /* cmd ::= SHOW DATABASES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_DATABASES_STMT); } break; - case 225: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy712, yymsp[0].minor.yy712, OP_TYPE_LIKE); } + case 227: /* cmd ::= SHOW db_name_cond_opt TABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TABLES_STMT, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, OP_TYPE_LIKE); } break; - case 226: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy712, yymsp[0].minor.yy712, OP_TYPE_LIKE); } + case 228: /* cmd ::= SHOW db_name_cond_opt STABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_STABLES_STMT, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, OP_TYPE_LIKE); } break; - case 227: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy712, NULL, OP_TYPE_LIKE); } + case 229: /* cmd ::= SHOW db_name_cond_opt VGROUPS */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_VGROUPS_STMT, yymsp[-1].minor.yy448, NULL, OP_TYPE_LIKE); } break; - case 228: /* cmd ::= SHOW MNODES */ + case 230: /* cmd ::= SHOW MNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_MNODES_STMT); } break; - case 229: /* cmd ::= SHOW QNODES */ + case 231: /* cmd ::= SHOW QNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QNODES_STMT); } break; - case 230: /* cmd ::= SHOW FUNCTIONS */ + case 232: /* cmd ::= SHOW FUNCTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_FUNCTIONS_STMT); } break; - case 231: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy712, yymsp[-1].minor.yy712, OP_TYPE_EQUAL); } + case 233: /* cmd ::= SHOW INDEXES FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_INDEXES_STMT, yymsp[0].minor.yy448, yymsp[-1].minor.yy448, OP_TYPE_EQUAL); } break; - case 232: /* cmd ::= SHOW STREAMS */ + case 234: /* cmd ::= SHOW STREAMS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_STREAMS_STMT); } break; - case 233: /* cmd ::= SHOW ACCOUNTS */ + case 235: /* cmd ::= SHOW ACCOUNTS */ { pCxt->errCode = generateSyntaxErrMsg(&pCxt->msgBuf, TSDB_CODE_PAR_EXPRIE_STATEMENT); } break; - case 234: /* cmd ::= SHOW APPS */ + case 236: /* cmd ::= SHOW APPS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_APPS_STMT); } break; - case 235: /* cmd ::= SHOW CONNECTIONS */ + case 237: /* cmd ::= SHOW CONNECTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONNECTIONS_STMT); } break; - case 236: /* cmd ::= SHOW LICENCES */ - case 237: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==237); + case 238: /* cmd ::= SHOW LICENCES */ + case 239: /* cmd ::= SHOW GRANTS */ yytestcase(yyruleno==239); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LICENCES_STMT); } break; - case 238: /* cmd ::= SHOW CREATE DATABASE db_name */ -{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy129); } + case 240: /* cmd ::= SHOW CREATE DATABASE db_name */ +{ pCxt->pRootNode = createShowCreateDatabaseStmt(pCxt, &yymsp[0].minor.yy113); } break; - case 239: /* cmd ::= SHOW CREATE TABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy712); } + case 241: /* cmd ::= SHOW CREATE TABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_TABLE_STMT, yymsp[0].minor.yy448); } break; - case 240: /* cmd ::= SHOW CREATE STABLE full_table_name */ -{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy712); } + case 242: /* cmd ::= SHOW CREATE STABLE full_table_name */ +{ pCxt->pRootNode = createShowCreateTableStmt(pCxt, QUERY_NODE_SHOW_CREATE_STABLE_STMT, yymsp[0].minor.yy448); } break; - case 241: /* cmd ::= SHOW QUERIES */ + case 243: /* cmd ::= SHOW QUERIES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_QUERIES_STMT); } break; - case 242: /* cmd ::= SHOW SCORES */ + case 244: /* cmd ::= SHOW SCORES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SCORES_STMT); } break; - case 243: /* cmd ::= SHOW TOPICS */ + case 245: /* cmd ::= SHOW TOPICS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TOPICS_STMT); } break; - case 244: /* cmd ::= SHOW VARIABLES */ - case 245: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==245); + case 246: /* cmd ::= SHOW VARIABLES */ + case 247: /* cmd ::= SHOW CLUSTER VARIABLES */ yytestcase(yyruleno==247); { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_VARIABLES_STMT); } break; - case 246: /* cmd ::= SHOW LOCAL VARIABLES */ + case 248: /* cmd ::= SHOW LOCAL VARIABLES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_LOCAL_VARIABLES_STMT); } break; - case 247: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ -{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy712); } + case 249: /* cmd ::= SHOW DNODE NK_INTEGER VARIABLES like_pattern_opt */ +{ pCxt->pRootNode = createShowDnodeVariablesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[-2].minor.yy0), yymsp[0].minor.yy448); } break; - case 248: /* cmd ::= SHOW BNODES */ + case 250: /* cmd ::= SHOW BNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_BNODES_STMT); } break; - case 249: /* cmd ::= SHOW SNODES */ + case 251: /* cmd ::= SHOW SNODES */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SNODES_STMT); } break; - case 250: /* cmd ::= SHOW CLUSTER */ + case 252: /* cmd ::= SHOW CLUSTER */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CLUSTER_STMT); } break; - case 251: /* cmd ::= SHOW TRANSACTIONS */ + case 253: /* cmd ::= SHOW TRANSACTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_TRANSACTIONS_STMT); } break; - case 252: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ -{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy712); } + case 254: /* cmd ::= SHOW TABLE DISTRIBUTED full_table_name */ +{ pCxt->pRootNode = createShowTableDistributedStmt(pCxt, yymsp[0].minor.yy448); } break; - case 253: /* cmd ::= SHOW CONSUMERS */ + case 255: /* cmd ::= SHOW CONSUMERS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_CONSUMERS_STMT); } break; - case 254: /* cmd ::= SHOW SUBSCRIPTIONS */ + case 256: /* cmd ::= SHOW SUBSCRIPTIONS */ { pCxt->pRootNode = createShowStmt(pCxt, QUERY_NODE_SHOW_SUBSCRIPTIONS_STMT); } break; - case 255: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy712, yymsp[-1].minor.yy712, OP_TYPE_EQUAL); } + case 257: /* cmd ::= SHOW TAGS FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowStmtWithCond(pCxt, QUERY_NODE_SHOW_TAGS_STMT, yymsp[0].minor.yy448, yymsp[-1].minor.yy448, OP_TYPE_EQUAL); } break; - case 256: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ -{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy712, yymsp[0].minor.yy712, yymsp[-3].minor.yy274); } + case 258: /* cmd ::= SHOW TABLE TAGS tag_list_opt FROM table_name_cond from_db_opt */ +{ pCxt->pRootNode = createShowTableTagsStmt(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy448, yymsp[-3].minor.yy432); } break; - case 257: /* cmd ::= SHOW VNODES NK_INTEGER */ + case 259: /* cmd ::= SHOW VNODES NK_INTEGER */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0), NULL); } break; - case 258: /* cmd ::= SHOW VNODES NK_STRING */ + case 260: /* cmd ::= SHOW VNODES NK_STRING */ { pCxt->pRootNode = createShowVnodesStmt(pCxt, NULL, createValueNode(pCxt, TSDB_DATA_TYPE_VARCHAR, &yymsp[0].minor.yy0)); } break; - case 259: /* cmd ::= SHOW db_name_cond_opt ALIVE */ -{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy712, QUERY_NODE_SHOW_DB_ALIVE_STMT); } + case 261: /* cmd ::= SHOW db_name_cond_opt ALIVE */ +{ pCxt->pRootNode = createShowAliveStmt(pCxt, yymsp[-1].minor.yy448, QUERY_NODE_SHOW_DB_ALIVE_STMT); } break; - case 260: /* cmd ::= SHOW CLUSTER ALIVE */ + case 262: /* cmd ::= SHOW CLUSTER ALIVE */ { pCxt->pRootNode = createShowAliveStmt(pCxt, NULL, QUERY_NODE_SHOW_CLUSTER_ALIVE_STMT); } break; - case 261: /* db_name_cond_opt ::= */ - case 266: /* from_db_opt ::= */ yytestcase(yyruleno==266); -{ yymsp[1].minor.yy712 = createDefaultDatabaseCondValue(pCxt); } + case 263: /* db_name_cond_opt ::= */ + case 268: /* from_db_opt ::= */ yytestcase(yyruleno==268); +{ yymsp[1].minor.yy448 = createDefaultDatabaseCondValue(pCxt); } break; - case 262: /* db_name_cond_opt ::= db_name NK_DOT */ -{ yylhsminor.yy712 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy129); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 264: /* db_name_cond_opt ::= db_name NK_DOT */ +{ yylhsminor.yy448 = createIdentifierValueNode(pCxt, &yymsp[-1].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 264: /* like_pattern_opt ::= LIKE NK_STRING */ -{ yymsp[-1].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + case 266: /* like_pattern_opt ::= LIKE NK_STRING */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } break; - case 265: /* table_name_cond ::= table_name */ -{ yylhsminor.yy712 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy129); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 267: /* table_name_cond ::= table_name */ +{ yylhsminor.yy448 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy113); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 267: /* from_db_opt ::= FROM db_name */ -{ yymsp[-1].minor.yy712 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy129); } + case 269: /* from_db_opt ::= FROM db_name */ +{ yymsp[-1].minor.yy448 = createIdentifierValueNode(pCxt, &yymsp[0].minor.yy113); } break; - case 271: /* tag_item ::= TBNAME */ -{ yylhsminor.yy712 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 273: /* tag_item ::= TBNAME */ +{ yylhsminor.yy448 = setProjectionAlias(pCxt, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL), &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 274: /* tag_item ::= column_name column_alias */ -{ yylhsminor.yy712 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy129), &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 276: /* tag_item ::= column_name column_alias */ +{ yylhsminor.yy448 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-1].minor.yy113), &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 275: /* tag_item ::= column_name AS column_alias */ -{ yylhsminor.yy712 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy129), &yymsp[0].minor.yy129); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 277: /* tag_item ::= column_name AS column_alias */ +{ yylhsminor.yy448 = setProjectionAlias(pCxt, createColumnNode(pCxt, NULL, &yymsp[-2].minor.yy113), &yymsp[0].minor.yy113); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 276: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy337, yymsp[-3].minor.yy712, yymsp[-1].minor.yy712, NULL, yymsp[0].minor.yy712); } + case 278: /* cmd ::= CREATE SMA INDEX not_exists_opt full_index_name ON full_table_name index_options */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_SMA, yymsp[-4].minor.yy369, yymsp[-3].minor.yy448, yymsp[-1].minor.yy448, NULL, yymsp[0].minor.yy448); } break; - case 277: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ -{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy337, yymsp[-5].minor.yy712, yymsp[-3].minor.yy712, yymsp[-1].minor.yy274, NULL); } + case 279: /* cmd ::= CREATE INDEX not_exists_opt full_index_name ON full_table_name NK_LP col_name_list NK_RP */ +{ pCxt->pRootNode = createCreateIndexStmt(pCxt, INDEX_TYPE_NORMAL, yymsp[-6].minor.yy369, yymsp[-5].minor.yy448, yymsp[-3].minor.yy448, yymsp[-1].minor.yy432, NULL); } break; - case 278: /* cmd ::= DROP INDEX exists_opt full_index_name */ -{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy337, yymsp[0].minor.yy712); } + case 280: /* cmd ::= DROP INDEX exists_opt full_index_name */ +{ pCxt->pRootNode = createDropIndexStmt(pCxt, yymsp[-1].minor.yy369, yymsp[0].minor.yy448); } break; - case 279: /* full_index_name ::= index_name */ -{ yylhsminor.yy712 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy129); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 281: /* full_index_name ::= index_name */ +{ yylhsminor.yy448 = createRealTableNodeForIndexName(pCxt, NULL, &yymsp[0].minor.yy113); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 280: /* full_index_name ::= db_name NK_DOT index_name */ -{ yylhsminor.yy712 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 282: /* full_index_name ::= db_name NK_DOT index_name */ +{ yylhsminor.yy448 = createRealTableNodeForIndexName(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 281: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-9].minor.yy712 = createIndexOption(pCxt, yymsp[-7].minor.yy274, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), NULL, yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } + case 283: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-9].minor.yy448 = createIndexOption(pCxt, yymsp[-7].minor.yy432, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), NULL, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 282: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ -{ yymsp[-11].minor.yy712 = createIndexOption(pCxt, yymsp[-9].minor.yy274, releaseRawExprNode(pCxt, yymsp[-5].minor.yy712), releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } + case 284: /* index_options ::= FUNCTION NK_LP func_list NK_RP INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt sma_stream_opt */ +{ yymsp[-11].minor.yy448 = createIndexOption(pCxt, yymsp[-9].minor.yy432, releaseRawExprNode(pCxt, yymsp[-5].minor.yy448), releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 285: /* func ::= sma_func_name NK_LP expression_list NK_RP */ -{ yylhsminor.yy712 = createFunctionNode(pCxt, &yymsp[-3].minor.yy129, yymsp[-1].minor.yy274); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 287: /* func ::= sma_func_name NK_LP expression_list NK_RP */ +{ yylhsminor.yy448 = createFunctionNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy432); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 291: /* sma_stream_opt ::= */ - case 329: /* stream_options ::= */ yytestcase(yyruleno==329); -{ yymsp[1].minor.yy712 = createStreamOptions(pCxt); } + case 288: /* sma_func_name ::= function_name */ + case 497: /* alias_opt ::= table_alias */ yytestcase(yyruleno==497); +{ yylhsminor.yy113 = yymsp[0].minor.yy113; } + yymsp[0].minor.yy113 = yylhsminor.yy113; break; - case 292: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy712)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy712); yylhsminor.yy712 = yymsp[-2].minor.yy712; } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 293: /* sma_stream_opt ::= */ + case 331: /* stream_options ::= */ yytestcase(yyruleno==331); +{ yymsp[1].minor.yy448 = createStreamOptions(pCxt); } break; - case 293: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy712)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy712); yylhsminor.yy712 = yymsp[-2].minor.yy712; } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 294: /* sma_stream_opt ::= sma_stream_opt WATERMARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy448)->pWatermark = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); yylhsminor.yy448 = yymsp[-2].minor.yy448; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 294: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ -{ ((SStreamOptions*)yymsp[-2].minor.yy712)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy712); yylhsminor.yy712 = yymsp[-2].minor.yy712; } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 295: /* sma_stream_opt ::= sma_stream_opt MAX_DELAY duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy448)->pDelay = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); yylhsminor.yy448 = yymsp[-2].minor.yy448; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 295: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ -{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy337, &yymsp[-2].minor.yy129, yymsp[0].minor.yy712); } + case 296: /* sma_stream_opt ::= sma_stream_opt DELETE_MARK duration_literal */ +{ ((SStreamOptions*)yymsp[-2].minor.yy448)->pDeleteMark = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); yylhsminor.yy448 = yymsp[-2].minor.yy448; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 296: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy337, &yymsp[-3].minor.yy129, &yymsp[0].minor.yy129, false); } + case 297: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS query_or_subquery */ +{ pCxt->pRootNode = createCreateTopicStmtUseQuery(pCxt, yymsp[-3].minor.yy369, &yymsp[-2].minor.yy113, yymsp[0].minor.yy448); } break; - case 297: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy337, &yymsp[-5].minor.yy129, &yymsp[0].minor.yy129, true); } + case 298: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-4].minor.yy369, &yymsp[-3].minor.yy113, &yymsp[0].minor.yy113, false); } break; - case 298: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy337, &yymsp[-3].minor.yy129, yymsp[0].minor.yy712, false); } + case 299: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS DATABASE db_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseDb(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, &yymsp[0].minor.yy113, true); } break; - case 299: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ -{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy337, &yymsp[-5].minor.yy129, yymsp[0].minor.yy712, true); } + case 300: /* cmd ::= CREATE TOPIC not_exists_opt topic_name AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-4].minor.yy369, &yymsp[-3].minor.yy113, yymsp[0].minor.yy448, false); } break; - case 300: /* cmd ::= DROP TOPIC exists_opt topic_name */ -{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy337, &yymsp[0].minor.yy129); } + case 301: /* cmd ::= CREATE TOPIC not_exists_opt topic_name WITH META AS STABLE full_table_name */ +{ pCxt->pRootNode = createCreateTopicStmtUseTable(pCxt, yymsp[-6].minor.yy369, &yymsp[-5].minor.yy113, yymsp[0].minor.yy448, true); } break; - case 301: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ -{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy337, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129); } + case 302: /* cmd ::= DROP TOPIC exists_opt topic_name */ +{ pCxt->pRootNode = createDropTopicStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 302: /* cmd ::= DESC full_table_name */ - case 303: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==303); -{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy712); } + case 303: /* cmd ::= DROP CONSUMER GROUP exists_opt cgroup_name ON topic_name */ +{ pCxt->pRootNode = createDropCGroupStmt(pCxt, yymsp[-3].minor.yy369, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113); } break; - case 304: /* cmd ::= RESET QUERY CACHE */ + case 304: /* cmd ::= DESC full_table_name */ + case 305: /* cmd ::= DESCRIBE full_table_name */ yytestcase(yyruleno==305); +{ pCxt->pRootNode = createDescribeStmt(pCxt, yymsp[0].minor.yy448); } + break; + case 306: /* cmd ::= RESET QUERY CACHE */ { pCxt->pRootNode = createResetQueryCacheStmt(pCxt); } break; - case 305: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ - case 306: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==306); -{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy337, yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } + case 307: /* cmd ::= EXPLAIN analyze_opt explain_options query_or_subquery */ + case 308: /* cmd ::= EXPLAIN analyze_opt explain_options insert_query */ yytestcase(yyruleno==308); +{ pCxt->pRootNode = createExplainStmt(pCxt, yymsp[-2].minor.yy369, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 309: /* explain_options ::= */ -{ yymsp[1].minor.yy712 = createDefaultExplainOptions(pCxt); } + case 311: /* explain_options ::= */ +{ yymsp[1].minor.yy448 = createDefaultExplainOptions(pCxt); } break; - case 310: /* explain_options ::= explain_options VERBOSE NK_BOOL */ -{ yylhsminor.yy712 = setExplainVerbose(pCxt, yymsp[-2].minor.yy712, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 312: /* explain_options ::= explain_options VERBOSE NK_BOOL */ +{ yylhsminor.yy448 = setExplainVerbose(pCxt, yymsp[-2].minor.yy448, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 311: /* explain_options ::= explain_options RATIO NK_FLOAT */ -{ yylhsminor.yy712 = setExplainRatio(pCxt, yymsp[-2].minor.yy712, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 313: /* explain_options ::= explain_options RATIO NK_FLOAT */ +{ yylhsminor.yy448 = setExplainRatio(pCxt, yymsp[-2].minor.yy448, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 312: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ -{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy337, yymsp[-9].minor.yy337, &yymsp[-6].minor.yy129, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy184, yymsp[-1].minor.yy310, &yymsp[0].minor.yy129, yymsp[-10].minor.yy337); } + case 314: /* cmd ::= CREATE or_replace_opt agg_func_opt FUNCTION not_exists_opt function_name AS NK_STRING OUTPUTTYPE type_name bufsize_opt language_opt */ +{ pCxt->pRootNode = createCreateFunctionStmt(pCxt, yymsp[-7].minor.yy369, yymsp[-9].minor.yy369, &yymsp[-6].minor.yy113, &yymsp[-4].minor.yy0, yymsp[-2].minor.yy728, yymsp[-1].minor.yy788, &yymsp[0].minor.yy113, yymsp[-10].minor.yy369); } break; - case 313: /* cmd ::= DROP FUNCTION exists_opt function_name */ -{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy337, &yymsp[0].minor.yy129); } + case 315: /* cmd ::= DROP FUNCTION exists_opt function_name */ +{ pCxt->pRootNode = createDropFunctionStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 318: /* language_opt ::= */ -{ yymsp[1].minor.yy129 = nil_token; } + case 320: /* language_opt ::= */ +{ yymsp[1].minor.yy113 = nil_token; } break; - case 319: /* language_opt ::= LANGUAGE NK_STRING */ -{ yymsp[-1].minor.yy129 = yymsp[0].minor.yy0; } + case 321: /* language_opt ::= LANGUAGE NK_STRING */ +{ yymsp[-1].minor.yy113 = yymsp[0].minor.yy0; } break; - case 322: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ -{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy337, &yymsp[-8].minor.yy129, yymsp[-5].minor.yy712, yymsp[-7].minor.yy712, yymsp[-3].minor.yy274, yymsp[-2].minor.yy712, yymsp[0].minor.yy712, yymsp[-4].minor.yy274); } + case 324: /* cmd ::= CREATE STREAM not_exists_opt stream_name stream_options INTO full_table_name col_list_opt tag_def_or_ref_opt subtable_opt AS query_or_subquery */ +{ pCxt->pRootNode = createCreateStreamStmt(pCxt, yymsp[-9].minor.yy369, &yymsp[-8].minor.yy113, yymsp[-5].minor.yy448, yymsp[-7].minor.yy448, yymsp[-3].minor.yy432, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, yymsp[-4].minor.yy432); } break; - case 323: /* cmd ::= DROP STREAM exists_opt stream_name */ -{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy337, &yymsp[0].minor.yy129); } + case 325: /* cmd ::= DROP STREAM exists_opt stream_name */ +{ pCxt->pRootNode = createDropStreamStmt(pCxt, yymsp[-1].minor.yy369, &yymsp[0].minor.yy113); } break; - case 330: /* stream_options ::= stream_options TRIGGER AT_ONCE */ - case 331: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==331); -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-2].minor.yy712, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 332: /* stream_options ::= stream_options TRIGGER AT_ONCE */ + case 333: /* stream_options ::= stream_options TRIGGER WINDOW_CLOSE */ yytestcase(yyruleno==333); +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_TRIGGER_TYPE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 332: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-3].minor.yy712, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 334: /* stream_options ::= stream_options TRIGGER MAX_DELAY duration_literal */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-3].minor.yy448, SOPT_TRIGGER_TYPE_SET, &yymsp[-1].minor.yy0, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 333: /* stream_options ::= stream_options WATERMARK duration_literal */ -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-2].minor.yy712, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 335: /* stream_options ::= stream_options WATERMARK duration_literal */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_WATERMARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 334: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-3].minor.yy712, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 336: /* stream_options ::= stream_options IGNORE EXPIRED NK_INTEGER */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-3].minor.yy448, SOPT_IGNORE_EXPIRED_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 335: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-2].minor.yy712, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 337: /* stream_options ::= stream_options FILL_HISTORY NK_INTEGER */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_FILL_HISTORY_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 336: /* stream_options ::= stream_options DELETE_MARK duration_literal */ -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-2].minor.yy712, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 338: /* stream_options ::= stream_options DELETE_MARK duration_literal */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-2].minor.yy448, SOPT_DELETE_MARK_SET, NULL, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 337: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ -{ yylhsminor.yy712 = setStreamOptions(pCxt, yymsp[-3].minor.yy712, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 339: /* stream_options ::= stream_options IGNORE UPDATE NK_INTEGER */ +{ yylhsminor.yy448 = setStreamOptions(pCxt, yymsp[-3].minor.yy448, SOPT_IGNORE_UPDATE_SET, &yymsp[0].minor.yy0, NULL); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 339: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ - case 529: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==529); - case 549: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==549); -{ yymsp[-3].minor.yy712 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy712); } + case 341: /* subtable_opt ::= SUBTABLE NK_LP expression NK_RP */ + case 531: /* sliding_opt ::= SLIDING NK_LP duration_literal NK_RP */ yytestcase(yyruleno==531); + case 551: /* every_opt ::= EVERY NK_LP duration_literal NK_RP */ yytestcase(yyruleno==551); +{ yymsp[-3].minor.yy448 = releaseRawExprNode(pCxt, yymsp[-1].minor.yy448); } break; - case 340: /* cmd ::= KILL CONNECTION NK_INTEGER */ + case 342: /* cmd ::= KILL CONNECTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_CONNECTION_STMT, &yymsp[0].minor.yy0); } break; - case 341: /* cmd ::= KILL QUERY NK_STRING */ + case 343: /* cmd ::= KILL QUERY NK_STRING */ { pCxt->pRootNode = createKillQueryStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 342: /* cmd ::= KILL TRANSACTION NK_INTEGER */ + case 344: /* cmd ::= KILL TRANSACTION NK_INTEGER */ { pCxt->pRootNode = createKillStmt(pCxt, QUERY_NODE_KILL_TRANSACTION_STMT, &yymsp[0].minor.yy0); } break; - case 343: /* cmd ::= BALANCE VGROUP */ + case 345: /* cmd ::= BALANCE VGROUP */ { pCxt->pRootNode = createBalanceVgroupStmt(pCxt); } break; - case 344: /* cmd ::= BALANCE VGROUP LEADER */ + case 346: /* cmd ::= BALANCE VGROUP LEADER */ { pCxt->pRootNode = createBalanceVgroupLeaderStmt(pCxt); } break; - case 345: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ + case 347: /* cmd ::= MERGE VGROUP NK_INTEGER NK_INTEGER */ { pCxt->pRootNode = createMergeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0); } break; - case 346: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ -{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy274); } + case 348: /* cmd ::= REDISTRIBUTE VGROUP NK_INTEGER dnode_list */ +{ pCxt->pRootNode = createRedistributeVgroupStmt(pCxt, &yymsp[-1].minor.yy0, yymsp[0].minor.yy432); } break; - case 347: /* cmd ::= SPLIT VGROUP NK_INTEGER */ + case 349: /* cmd ::= SPLIT VGROUP NK_INTEGER */ { pCxt->pRootNode = createSplitVgroupStmt(pCxt, &yymsp[0].minor.yy0); } break; - case 348: /* dnode_list ::= DNODE NK_INTEGER */ -{ yymsp[-1].minor.yy274 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } - break; - case 350: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ -{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } - break; - case 353: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ -{ yymsp[-6].minor.yy712 = createInsertStmt(pCxt, yymsp[-4].minor.yy712, yymsp[-2].minor.yy274, yymsp[0].minor.yy712); } - break; - case 354: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ -{ yymsp[-3].minor.yy712 = createInsertStmt(pCxt, yymsp[-1].minor.yy712, NULL, yymsp[0].minor.yy712); } - break; - case 355: /* literal ::= NK_INTEGER */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 356: /* literal ::= NK_FLOAT */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 357: /* literal ::= NK_STRING */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 358: /* literal ::= NK_BOOL */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 359: /* literal ::= TIMESTAMP NK_STRING */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; - break; - case 360: /* literal ::= duration_literal */ - case 370: /* signed_literal ::= signed */ yytestcase(yyruleno==370); - case 391: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==391); - case 392: /* expression ::= literal */ yytestcase(yyruleno==392); - case 393: /* expression ::= pseudo_column */ yytestcase(yyruleno==393); - case 394: /* expression ::= column_reference */ yytestcase(yyruleno==394); - case 395: /* expression ::= function_expression */ yytestcase(yyruleno==395); - case 396: /* expression ::= case_when_expression */ yytestcase(yyruleno==396); - case 427: /* function_expression ::= literal_func */ yytestcase(yyruleno==427); - case 476: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==476); - case 480: /* boolean_primary ::= predicate */ yytestcase(yyruleno==480); - case 482: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==482); - case 483: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==483); - case 486: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==486); - case 488: /* table_reference ::= table_primary */ yytestcase(yyruleno==488); - case 489: /* table_reference ::= joined_table */ yytestcase(yyruleno==489); - case 493: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==493); - case 551: /* query_simple ::= query_specification */ yytestcase(yyruleno==551); - case 552: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==552); - case 555: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==555); - case 557: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==557); -{ yylhsminor.yy712 = yymsp[0].minor.yy712; } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 361: /* literal ::= NULL */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 362: /* literal ::= NK_QUESTION */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 363: /* duration_literal ::= NK_VARIABLE */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 364: /* signed ::= NK_INTEGER */ -{ yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 365: /* signed ::= NK_PLUS NK_INTEGER */ -{ yymsp[-1].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } - break; - case 366: /* signed ::= NK_MINUS NK_INTEGER */ + case 350: /* dnode_list ::= DNODE NK_INTEGER */ +{ yymsp[-1].minor.yy432 = createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &yymsp[0].minor.yy0)); } + break; + case 352: /* cmd ::= DELETE FROM full_table_name where_clause_opt */ +{ pCxt->pRootNode = createDeleteStmt(pCxt, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } + break; + case 355: /* insert_query ::= INSERT INTO full_table_name NK_LP col_name_list NK_RP query_or_subquery */ +{ yymsp[-6].minor.yy448 = createInsertStmt(pCxt, yymsp[-4].minor.yy448, yymsp[-2].minor.yy432, yymsp[0].minor.yy448); } + break; + case 356: /* insert_query ::= INSERT INTO full_table_name query_or_subquery */ +{ yymsp[-3].minor.yy448 = createInsertStmt(pCxt, yymsp[-1].minor.yy448, NULL, yymsp[0].minor.yy448); } + break; + case 357: /* literal ::= NK_INTEGER */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 358: /* literal ::= NK_FLOAT */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 359: /* literal ::= NK_STRING */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 360: /* literal ::= NK_BOOL */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 361: /* literal ::= TIMESTAMP NK_STRING */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0)); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; + break; + case 362: /* literal ::= duration_literal */ + case 372: /* signed_literal ::= signed */ yytestcase(yyruleno==372); + case 393: /* expr_or_subquery ::= expression */ yytestcase(yyruleno==393); + case 394: /* expression ::= literal */ yytestcase(yyruleno==394); + case 395: /* expression ::= pseudo_column */ yytestcase(yyruleno==395); + case 396: /* expression ::= column_reference */ yytestcase(yyruleno==396); + case 397: /* expression ::= function_expression */ yytestcase(yyruleno==397); + case 398: /* expression ::= case_when_expression */ yytestcase(yyruleno==398); + case 429: /* function_expression ::= literal_func */ yytestcase(yyruleno==429); + case 478: /* boolean_value_expression ::= boolean_primary */ yytestcase(yyruleno==478); + case 482: /* boolean_primary ::= predicate */ yytestcase(yyruleno==482); + case 484: /* common_expression ::= expr_or_subquery */ yytestcase(yyruleno==484); + case 485: /* common_expression ::= boolean_value_expression */ yytestcase(yyruleno==485); + case 488: /* table_reference_list ::= table_reference */ yytestcase(yyruleno==488); + case 490: /* table_reference ::= table_primary */ yytestcase(yyruleno==490); + case 491: /* table_reference ::= joined_table */ yytestcase(yyruleno==491); + case 495: /* table_primary ::= parenthesized_joined_table */ yytestcase(yyruleno==495); + case 553: /* query_simple ::= query_specification */ yytestcase(yyruleno==553); + case 554: /* query_simple ::= union_query_expression */ yytestcase(yyruleno==554); + case 557: /* query_simple_or_subquery ::= query_simple */ yytestcase(yyruleno==557); + case 559: /* query_or_subquery ::= query_expression */ yytestcase(yyruleno==559); +{ yylhsminor.yy448 = yymsp[0].minor.yy448; } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 363: /* literal ::= NULL */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 364: /* literal ::= NK_QUESTION */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 365: /* duration_literal ::= NK_VARIABLE */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createDurationValueNode(pCxt, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 366: /* signed ::= NK_INTEGER */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 367: /* signed ::= NK_PLUS NK_INTEGER */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_UBIGINT, &yymsp[0].minor.yy0); } + break; + case 368: /* signed ::= NK_MINUS NK_INTEGER */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); + yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BIGINT, &t); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 367: /* signed ::= NK_FLOAT */ -{ yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 369: /* signed ::= NK_FLOAT */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 368: /* signed ::= NK_PLUS NK_FLOAT */ -{ yymsp[-1].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } + case 370: /* signed ::= NK_PLUS NK_FLOAT */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &yymsp[0].minor.yy0); } break; - case 369: /* signed ::= NK_MINUS NK_FLOAT */ + case 371: /* signed ::= NK_MINUS NK_FLOAT */ { SToken t = yymsp[-1].minor.yy0; t.n = (yymsp[0].minor.yy0.z + yymsp[0].minor.yy0.n) - yymsp[-1].minor.yy0.z; - yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); + yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_DOUBLE, &t); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; - break; - case 371: /* signed_literal ::= NK_STRING */ -{ yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 372: /* signed_literal ::= NK_BOOL */ -{ yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 373: /* signed_literal ::= TIMESTAMP NK_STRING */ -{ yymsp[-1].minor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } - break; - case 374: /* signed_literal ::= duration_literal */ - case 376: /* signed_literal ::= literal_func */ yytestcase(yyruleno==376); - case 447: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==447); - case 509: /* select_item ::= common_expression */ yytestcase(yyruleno==509); - case 519: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==519); - case 556: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==556); - case 558: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==558); - case 571: /* search_condition ::= common_expression */ yytestcase(yyruleno==571); -{ yylhsminor.yy712 = releaseRawExprNode(pCxt, yymsp[0].minor.yy712); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 375: /* signed_literal ::= NULL */ -{ yylhsminor.yy712 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 377: /* signed_literal ::= NK_QUESTION */ -{ yylhsminor.yy712 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 397: /* expression ::= NK_LP expression NK_RP */ - case 481: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==481); - case 570: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==570); -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy712)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 398: /* expression ::= NK_PLUS expr_or_subquery */ + yymsp[-1].minor.yy448 = yylhsminor.yy448; + break; + case 373: /* signed_literal ::= NK_STRING */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 374: /* signed_literal ::= NK_BOOL */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_BOOL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 375: /* signed_literal ::= TIMESTAMP NK_STRING */ +{ yymsp[-1].minor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_TIMESTAMP, &yymsp[0].minor.yy0); } + break; + case 376: /* signed_literal ::= duration_literal */ + case 378: /* signed_literal ::= literal_func */ yytestcase(yyruleno==378); + case 449: /* star_func_para ::= expr_or_subquery */ yytestcase(yyruleno==449); + case 511: /* select_item ::= common_expression */ yytestcase(yyruleno==511); + case 521: /* partition_item ::= expr_or_subquery */ yytestcase(yyruleno==521); + case 558: /* query_simple_or_subquery ::= subquery */ yytestcase(yyruleno==558); + case 560: /* query_or_subquery ::= subquery */ yytestcase(yyruleno==560); + case 573: /* search_condition ::= common_expression */ yytestcase(yyruleno==573); +{ yylhsminor.yy448 = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 377: /* signed_literal ::= NULL */ +{ yylhsminor.yy448 = createValueNode(pCxt, TSDB_DATA_TYPE_NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 379: /* signed_literal ::= NK_QUESTION */ +{ yylhsminor.yy448 = createPlaceholderValueNode(pCxt, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 399: /* expression ::= NK_LP expression NK_RP */ + case 483: /* boolean_primary ::= NK_LP boolean_value_expression NK_RP */ yytestcase(yyruleno==483); + case 572: /* subquery ::= NK_LP subquery NK_RP */ yytestcase(yyruleno==572); +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 400: /* expression ::= NK_PLUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 399: /* expression ::= NK_MINUS expr_or_subquery */ + case 401: /* expression ::= NK_MINUS expr_or_subquery */ { - SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy712), NULL)); + SToken t = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &t, createOperatorNode(pCxt, OP_TYPE_MINUS, releaseRawExprNode(pCxt, yymsp[0].minor.yy448), NULL)); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 400: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ + case 402: /* expression ::= expr_or_subquery NK_PLUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_ADD, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 401: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ + case 403: /* expression ::= expr_or_subquery NK_MINUS expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_SUB, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 402: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ + case 404: /* expression ::= expr_or_subquery NK_STAR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_MULTI, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 403: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ + case 405: /* expression ::= expr_or_subquery NK_SLASH expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_DIV, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 404: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ + case 406: /* expression ::= expr_or_subquery NK_REM expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_REM, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 405: /* expression ::= column_reference NK_ARROW NK_STRING */ + case 407: /* expression ::= column_reference NK_ARROW NK_STRING */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_JSON_GET_VALUE, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[0].minor.yy0))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 406: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ + case 408: /* expression ::= expr_or_subquery NK_BITAND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 407: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ + case 409: /* expression ::= expr_or_subquery NK_BITOR expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, OP_TYPE_BIT_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 410: /* column_reference ::= column_name */ -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy129, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy129)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 411: /* column_reference ::= table_name NK_DOT column_name */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129, createColumnNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy129)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 412: /* pseudo_column ::= ROWTS */ - case 413: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==413); - case 415: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==415); - case 416: /* pseudo_column ::= QEND */ yytestcase(yyruleno==416); - case 417: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==417); - case 418: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==418); - case 419: /* pseudo_column ::= WEND */ yytestcase(yyruleno==419); - case 420: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==420); - case 421: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==421); - case 422: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==422); - case 423: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==423); - case 429: /* literal_func ::= NOW */ yytestcase(yyruleno==429); -{ yylhsminor.yy712 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } - yymsp[0].minor.yy712 = yylhsminor.yy712; - break; - case 414: /* pseudo_column ::= table_name NK_DOT TBNAME */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy129)))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 424: /* function_expression ::= function_name NK_LP expression_list NK_RP */ - case 425: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==425); -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy129, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy129, yymsp[-1].minor.yy274)); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; - break; - case 426: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), yymsp[-1].minor.yy184)); } - yymsp[-5].minor.yy712 = yylhsminor.yy712; - break; - case 428: /* literal_func ::= noarg_func NK_LP NK_RP */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy129, NULL)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 443: /* star_func_para_list ::= NK_STAR */ -{ yylhsminor.yy274 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } - yymsp[0].minor.yy274 = yylhsminor.yy274; - break; - case 448: /* star_func_para ::= table_name NK_DOT NK_STAR */ - case 512: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==512); -{ yylhsminor.yy712 = createColumnNode(pCxt, &yymsp[-2].minor.yy129, &yymsp[0].minor.yy0); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 449: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy274, yymsp[-1].minor.yy712)); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; - break; - case 450: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), yymsp[-2].minor.yy274, yymsp[-1].minor.yy712)); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; - break; - case 453: /* when_then_expr ::= WHEN common_expression THEN common_expression */ -{ yymsp[-3].minor.yy712 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712)); } - break; - case 455: /* case_when_else_opt ::= ELSE common_expression */ -{ yymsp[-1].minor.yy712 = releaseRawExprNode(pCxt, yymsp[0].minor.yy712); } - break; - case 456: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ - case 461: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==461); + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 412: /* column_reference ::= column_name */ +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy113, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy113)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 413: /* column_reference ::= table_name NK_DOT column_name */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113, createColumnNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy113)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 414: /* pseudo_column ::= ROWTS */ + case 415: /* pseudo_column ::= TBNAME */ yytestcase(yyruleno==415); + case 417: /* pseudo_column ::= QSTART */ yytestcase(yyruleno==417); + case 418: /* pseudo_column ::= QEND */ yytestcase(yyruleno==418); + case 419: /* pseudo_column ::= QDURATION */ yytestcase(yyruleno==419); + case 420: /* pseudo_column ::= WSTART */ yytestcase(yyruleno==420); + case 421: /* pseudo_column ::= WEND */ yytestcase(yyruleno==421); + case 422: /* pseudo_column ::= WDURATION */ yytestcase(yyruleno==422); + case 423: /* pseudo_column ::= IROWTS */ yytestcase(yyruleno==423); + case 424: /* pseudo_column ::= ISFILLED */ yytestcase(yyruleno==424); + case 425: /* pseudo_column ::= QTAGS */ yytestcase(yyruleno==425); + case 431: /* literal_func ::= NOW */ yytestcase(yyruleno==431); +{ yylhsminor.yy448 = createRawExprNode(pCxt, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, NULL)); } + yymsp[0].minor.yy448 = yylhsminor.yy448; + break; + case 416: /* pseudo_column ::= table_name NK_DOT TBNAME */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[0].minor.yy0, createNodeList(pCxt, createValueNode(pCxt, TSDB_DATA_TYPE_BINARY, &yymsp[-2].minor.yy113)))); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 426: /* function_expression ::= function_name NK_LP expression_list NK_RP */ + case 427: /* function_expression ::= star_func NK_LP star_func_para_list NK_RP */ yytestcase(yyruleno==427); +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-3].minor.yy113, yymsp[-1].minor.yy432)); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 428: /* function_expression ::= CAST NK_LP expr_or_subquery AS type_name NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-5].minor.yy0, &yymsp[0].minor.yy0, createCastFunctionNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-1].minor.yy728)); } + yymsp[-5].minor.yy448 = yylhsminor.yy448; + break; + case 430: /* literal_func ::= noarg_func NK_LP NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0, createFunctionNode(pCxt, &yymsp[-2].minor.yy113, NULL)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 445: /* star_func_para_list ::= NK_STAR */ +{ yylhsminor.yy432 = createNodeList(pCxt, createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0)); } + yymsp[0].minor.yy432 = yylhsminor.yy432; + break; + case 450: /* star_func_para ::= table_name NK_DOT NK_STAR */ + case 514: /* select_item ::= table_name NK_DOT NK_STAR */ yytestcase(yyruleno==514); +{ yylhsminor.yy448 = createColumnNode(pCxt, &yymsp[-2].minor.yy113, &yymsp[0].minor.yy0); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; + break; + case 451: /* case_when_expression ::= CASE when_then_list case_when_else_opt END */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-3].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, NULL, yymsp[-2].minor.yy432, yymsp[-1].minor.yy448)); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; + break; + case 452: /* case_when_expression ::= CASE common_expression when_then_list case_when_else_opt END */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-4].minor.yy0, &yymsp[0].minor.yy0, createCaseWhenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-2].minor.yy432, yymsp[-1].minor.yy448)); } + yymsp[-4].minor.yy448 = yylhsminor.yy448; + break; + case 455: /* when_then_expr ::= WHEN common_expression THEN common_expression */ +{ yymsp[-3].minor.yy448 = createWhenThenNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448)); } + break; + case 457: /* case_when_else_opt ::= ELSE common_expression */ +{ yymsp[-1].minor.yy448 = releaseRawExprNode(pCxt, yymsp[0].minor.yy448); } + break; + case 458: /* predicate ::= expr_or_subquery compare_op expr_or_subquery */ + case 463: /* predicate ::= expr_or_subquery in_op in_predicate_value */ yytestcase(yyruleno==463); { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy440, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createOperatorNode(pCxt, yymsp[-1].minor.yy156, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 457: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ + case 459: /* predicate ::= expr_or_subquery BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy712), releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-4].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-4].minor.yy448), releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-4].minor.yy712 = yylhsminor.yy712; + yymsp[-4].minor.yy448 = yylhsminor.yy448; break; - case 458: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ + case 460: /* predicate ::= expr_or_subquery NOT BETWEEN expr_or_subquery AND expr_or_subquery */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy712), releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-5].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createNotBetweenAnd(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy448), releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-5].minor.yy712 = yylhsminor.yy712; + yymsp[-5].minor.yy448 = yylhsminor.yy448; break; - case 459: /* predicate ::= expr_or_subquery IS NULL */ + case 461: /* predicate ::= expr_or_subquery IS NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NULL, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), NULL)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 460: /* predicate ::= expr_or_subquery IS NOT NULL */ + case 462: /* predicate ::= expr_or_subquery IS NOT NULL */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), NULL)); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-3].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &yymsp[0].minor.yy0, createOperatorNode(pCxt, OP_TYPE_IS_NOT_NULL, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), NULL)); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 462: /* compare_op ::= NK_LT */ -{ yymsp[0].minor.yy440 = OP_TYPE_LOWER_THAN; } + case 464: /* compare_op ::= NK_LT */ +{ yymsp[0].minor.yy156 = OP_TYPE_LOWER_THAN; } break; - case 463: /* compare_op ::= NK_GT */ -{ yymsp[0].minor.yy440 = OP_TYPE_GREATER_THAN; } + case 465: /* compare_op ::= NK_GT */ +{ yymsp[0].minor.yy156 = OP_TYPE_GREATER_THAN; } break; - case 464: /* compare_op ::= NK_LE */ -{ yymsp[0].minor.yy440 = OP_TYPE_LOWER_EQUAL; } + case 466: /* compare_op ::= NK_LE */ +{ yymsp[0].minor.yy156 = OP_TYPE_LOWER_EQUAL; } break; - case 465: /* compare_op ::= NK_GE */ -{ yymsp[0].minor.yy440 = OP_TYPE_GREATER_EQUAL; } + case 467: /* compare_op ::= NK_GE */ +{ yymsp[0].minor.yy156 = OP_TYPE_GREATER_EQUAL; } break; - case 466: /* compare_op ::= NK_NE */ -{ yymsp[0].minor.yy440 = OP_TYPE_NOT_EQUAL; } + case 468: /* compare_op ::= NK_NE */ +{ yymsp[0].minor.yy156 = OP_TYPE_NOT_EQUAL; } break; - case 467: /* compare_op ::= NK_EQ */ -{ yymsp[0].minor.yy440 = OP_TYPE_EQUAL; } + case 469: /* compare_op ::= NK_EQ */ +{ yymsp[0].minor.yy156 = OP_TYPE_EQUAL; } break; - case 468: /* compare_op ::= LIKE */ -{ yymsp[0].minor.yy440 = OP_TYPE_LIKE; } + case 470: /* compare_op ::= LIKE */ +{ yymsp[0].minor.yy156 = OP_TYPE_LIKE; } break; - case 469: /* compare_op ::= NOT LIKE */ -{ yymsp[-1].minor.yy440 = OP_TYPE_NOT_LIKE; } + case 471: /* compare_op ::= NOT LIKE */ +{ yymsp[-1].minor.yy156 = OP_TYPE_NOT_LIKE; } break; - case 470: /* compare_op ::= MATCH */ -{ yymsp[0].minor.yy440 = OP_TYPE_MATCH; } + case 472: /* compare_op ::= MATCH */ +{ yymsp[0].minor.yy156 = OP_TYPE_MATCH; } break; - case 471: /* compare_op ::= NMATCH */ -{ yymsp[0].minor.yy440 = OP_TYPE_NMATCH; } + case 473: /* compare_op ::= NMATCH */ +{ yymsp[0].minor.yy156 = OP_TYPE_NMATCH; } break; - case 472: /* compare_op ::= CONTAINS */ -{ yymsp[0].minor.yy440 = OP_TYPE_JSON_CONTAINS; } + case 474: /* compare_op ::= CONTAINS */ +{ yymsp[0].minor.yy156 = OP_TYPE_JSON_CONTAINS; } break; - case 473: /* in_op ::= IN */ -{ yymsp[0].minor.yy440 = OP_TYPE_IN; } + case 475: /* in_op ::= IN */ +{ yymsp[0].minor.yy156 = OP_TYPE_IN; } break; - case 474: /* in_op ::= NOT IN */ -{ yymsp[-1].minor.yy440 = OP_TYPE_NOT_IN; } + case 476: /* in_op ::= NOT IN */ +{ yymsp[-1].minor.yy156 = OP_TYPE_NOT_IN; } break; - case 475: /* in_predicate_value ::= NK_LP literal_list NK_RP */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy274)); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 477: /* in_predicate_value ::= NK_LP literal_list NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 477: /* boolean_value_expression ::= NOT boolean_primary */ + case 479: /* boolean_value_expression ::= NOT boolean_primary */ { - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy712), NULL)); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-1].minor.yy0, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_NOT, releaseRawExprNode(pCxt, yymsp[0].minor.yy448), NULL)); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 478: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ + case 480: /* boolean_value_expression ::= boolean_value_expression OR boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_OR, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 479: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ + case 481: /* boolean_value_expression ::= boolean_value_expression AND boolean_value_expression */ { - SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy712); - SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy712); - yylhsminor.yy712 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); + SToken s = getTokenFromRawExprNode(pCxt, yymsp[-2].minor.yy448); + SToken e = getTokenFromRawExprNode(pCxt, yymsp[0].minor.yy448); + yylhsminor.yy448 = createRawExprNodeExt(pCxt, &s, &e, createLogicConditionNode(pCxt, LOGIC_COND_TYPE_AND, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; - break; - case 485: /* from_clause_opt ::= FROM table_reference_list */ - case 514: /* where_clause_opt ::= WHERE search_condition */ yytestcase(yyruleno==514); - case 545: /* having_clause_opt ::= HAVING search_condition */ yytestcase(yyruleno==545); -{ yymsp[-1].minor.yy712 = yymsp[0].minor.yy712; } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 487: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ -{ yylhsminor.yy712 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy712, yymsp[0].minor.yy712, NULL); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 489: /* table_reference_list ::= table_reference_list NK_COMMA table_reference */ +{ yylhsminor.yy448 = createJoinTableNode(pCxt, JOIN_TYPE_INNER, yymsp[-2].minor.yy448, yymsp[0].minor.yy448, NULL); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 490: /* table_primary ::= table_name alias_opt */ -{ yylhsminor.yy712 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 492: /* table_primary ::= table_name alias_opt */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, NULL, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 491: /* table_primary ::= db_name NK_DOT table_name alias_opt */ -{ yylhsminor.yy712 = createRealTableNode(pCxt, &yymsp[-3].minor.yy129, &yymsp[-1].minor.yy129, &yymsp[0].minor.yy129); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 493: /* table_primary ::= db_name NK_DOT table_name alias_opt */ +{ yylhsminor.yy448 = createRealTableNode(pCxt, &yymsp[-3].minor.yy113, &yymsp[-1].minor.yy113, &yymsp[0].minor.yy113); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 492: /* table_primary ::= subquery alias_opt */ -{ yylhsminor.yy712 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy712), &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 494: /* table_primary ::= subquery alias_opt */ +{ yylhsminor.yy448 = createTempTableNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448), &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 494: /* alias_opt ::= */ -{ yymsp[1].minor.yy129 = nil_token; } + case 496: /* alias_opt ::= */ +{ yymsp[1].minor.yy113 = nil_token; } break; - case 496: /* alias_opt ::= AS table_alias */ -{ yymsp[-1].minor.yy129 = yymsp[0].minor.yy129; } + case 498: /* alias_opt ::= AS table_alias */ +{ yymsp[-1].minor.yy113 = yymsp[0].minor.yy113; } break; - case 497: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ - case 498: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==498); -{ yymsp[-2].minor.yy712 = yymsp[-1].minor.yy712; } + case 499: /* parenthesized_joined_table ::= NK_LP joined_table NK_RP */ + case 500: /* parenthesized_joined_table ::= NK_LP parenthesized_joined_table NK_RP */ yytestcase(yyruleno==500); +{ yymsp[-2].minor.yy448 = yymsp[-1].minor.yy448; } break; - case 499: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ -{ yylhsminor.yy712 = createJoinTableNode(pCxt, yymsp[-4].minor.yy912, yymsp[-5].minor.yy712, yymsp[-2].minor.yy712, yymsp[0].minor.yy712); } - yymsp[-5].minor.yy712 = yylhsminor.yy712; + case 501: /* joined_table ::= table_reference join_type JOIN table_reference ON search_condition */ +{ yylhsminor.yy448 = createJoinTableNode(pCxt, yymsp[-4].minor.yy596, yymsp[-5].minor.yy448, yymsp[-2].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-5].minor.yy448 = yylhsminor.yy448; break; - case 500: /* join_type ::= */ -{ yymsp[1].minor.yy912 = JOIN_TYPE_INNER; } + case 502: /* join_type ::= */ +{ yymsp[1].minor.yy596 = JOIN_TYPE_INNER; } break; - case 501: /* join_type ::= INNER */ -{ yymsp[0].minor.yy912 = JOIN_TYPE_INNER; } + case 503: /* join_type ::= INNER */ +{ yymsp[0].minor.yy596 = JOIN_TYPE_INNER; } break; - case 502: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ + case 504: /* query_specification ::= SELECT set_quantifier_opt select_list from_clause_opt where_clause_opt partition_by_clause_opt range_opt every_opt fill_opt twindow_clause_opt group_by_clause_opt having_clause_opt */ { - yymsp[-11].minor.yy712 = createSelectStmt(pCxt, yymsp[-10].minor.yy337, yymsp[-9].minor.yy274, yymsp[-8].minor.yy712); - yymsp[-11].minor.yy712 = addWhereClause(pCxt, yymsp[-11].minor.yy712, yymsp[-7].minor.yy712); - yymsp[-11].minor.yy712 = addPartitionByClause(pCxt, yymsp[-11].minor.yy712, yymsp[-6].minor.yy274); - yymsp[-11].minor.yy712 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy712, yymsp[-2].minor.yy712); - yymsp[-11].minor.yy712 = addGroupByClause(pCxt, yymsp[-11].minor.yy712, yymsp[-1].minor.yy274); - yymsp[-11].minor.yy712 = addHavingClause(pCxt, yymsp[-11].minor.yy712, yymsp[0].minor.yy712); - yymsp[-11].minor.yy712 = addRangeClause(pCxt, yymsp[-11].minor.yy712, yymsp[-5].minor.yy712); - yymsp[-11].minor.yy712 = addEveryClause(pCxt, yymsp[-11].minor.yy712, yymsp[-4].minor.yy712); - yymsp[-11].minor.yy712 = addFillClause(pCxt, yymsp[-11].minor.yy712, yymsp[-3].minor.yy712); + yymsp[-11].minor.yy448 = createSelectStmt(pCxt, yymsp[-10].minor.yy369, yymsp[-9].minor.yy432, yymsp[-8].minor.yy448); + yymsp[-11].minor.yy448 = addWhereClause(pCxt, yymsp[-11].minor.yy448, yymsp[-7].minor.yy448); + yymsp[-11].minor.yy448 = addPartitionByClause(pCxt, yymsp[-11].minor.yy448, yymsp[-6].minor.yy432); + yymsp[-11].minor.yy448 = addWindowClauseClause(pCxt, yymsp[-11].minor.yy448, yymsp[-2].minor.yy448); + yymsp[-11].minor.yy448 = addGroupByClause(pCxt, yymsp[-11].minor.yy448, yymsp[-1].minor.yy432); + yymsp[-11].minor.yy448 = addHavingClause(pCxt, yymsp[-11].minor.yy448, yymsp[0].minor.yy448); + yymsp[-11].minor.yy448 = addRangeClause(pCxt, yymsp[-11].minor.yy448, yymsp[-5].minor.yy448); + yymsp[-11].minor.yy448 = addEveryClause(pCxt, yymsp[-11].minor.yy448, yymsp[-4].minor.yy448); + yymsp[-11].minor.yy448 = addFillClause(pCxt, yymsp[-11].minor.yy448, yymsp[-3].minor.yy448); } break; - case 505: /* set_quantifier_opt ::= ALL */ -{ yymsp[0].minor.yy337 = false; } + case 507: /* set_quantifier_opt ::= ALL */ +{ yymsp[0].minor.yy369 = false; } break; - case 508: /* select_item ::= NK_STAR */ -{ yylhsminor.yy712 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } - yymsp[0].minor.yy712 = yylhsminor.yy712; + case 510: /* select_item ::= NK_STAR */ +{ yylhsminor.yy448 = createColumnNode(pCxt, NULL, &yymsp[0].minor.yy0); } + yymsp[0].minor.yy448 = yylhsminor.yy448; break; - case 510: /* select_item ::= common_expression column_alias */ - case 520: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==520); -{ yylhsminor.yy712 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy712), &yymsp[0].minor.yy129); } - yymsp[-1].minor.yy712 = yylhsminor.yy712; + case 512: /* select_item ::= common_expression column_alias */ + case 522: /* partition_item ::= expr_or_subquery column_alias */ yytestcase(yyruleno==522); +{ yylhsminor.yy448 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448), &yymsp[0].minor.yy113); } + yymsp[-1].minor.yy448 = yylhsminor.yy448; break; - case 511: /* select_item ::= common_expression AS column_alias */ - case 521: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==521); -{ yylhsminor.yy712 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), &yymsp[0].minor.yy129); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 513: /* select_item ::= common_expression AS column_alias */ + case 523: /* partition_item ::= expr_or_subquery AS column_alias */ yytestcase(yyruleno==523); +{ yylhsminor.yy448 = setProjectionAlias(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), &yymsp[0].minor.yy113); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 516: /* partition_by_clause_opt ::= PARTITION BY partition_list */ - case 541: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==541); - case 560: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==560); -{ yymsp[-2].minor.yy274 = yymsp[0].minor.yy274; } + case 518: /* partition_by_clause_opt ::= PARTITION BY partition_list */ + case 543: /* group_by_clause_opt ::= GROUP BY group_by_list */ yytestcase(yyruleno==543); + case 562: /* order_by_clause_opt ::= ORDER BY sort_specification_list */ yytestcase(yyruleno==562); +{ yymsp[-2].minor.yy432 = yymsp[0].minor.yy432; } break; - case 523: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ -{ yymsp[-5].minor.yy712 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), releaseRawExprNode(pCxt, yymsp[-1].minor.yy712)); } + case 525: /* twindow_clause_opt ::= SESSION NK_LP column_reference NK_COMMA duration_literal NK_RP */ +{ yymsp[-5].minor.yy448 = createSessionWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } break; - case 524: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ -{ yymsp[-3].minor.yy712 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy712)); } + case 526: /* twindow_clause_opt ::= STATE_WINDOW NK_LP expr_or_subquery NK_RP */ +{ yymsp[-3].minor.yy448 = createStateWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } break; - case 525: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-5].minor.yy712 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), NULL, yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } + case 527: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-5].minor.yy448 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), NULL, yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 526: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ -{ yymsp[-7].minor.yy712 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy712), releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), yymsp[-1].minor.yy712, yymsp[0].minor.yy712); } + case 528: /* twindow_clause_opt ::= INTERVAL NK_LP duration_literal NK_COMMA duration_literal NK_RP sliding_opt fill_opt */ +{ yymsp[-7].minor.yy448 = createIntervalWindowNode(pCxt, releaseRawExprNode(pCxt, yymsp[-5].minor.yy448), releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), yymsp[-1].minor.yy448, yymsp[0].minor.yy448); } break; - case 527: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ -{ yymsp[-6].minor.yy712 = createEventWindowNode(pCxt, yymsp[-3].minor.yy712, yymsp[0].minor.yy712); } + case 529: /* twindow_clause_opt ::= EVENT_WINDOW START WITH search_condition END WITH search_condition */ +{ yymsp[-6].minor.yy448 = createEventWindowNode(pCxt, yymsp[-3].minor.yy448, yymsp[0].minor.yy448); } break; - case 531: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ -{ yymsp[-3].minor.yy712 = createFillNode(pCxt, yymsp[-1].minor.yy94, NULL); } + case 533: /* fill_opt ::= FILL NK_LP fill_mode NK_RP */ +{ yymsp[-3].minor.yy448 = createFillNode(pCxt, yymsp[-1].minor.yy46, NULL); } break; - case 532: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy712 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy274)); } + case 534: /* fill_opt ::= FILL NK_LP VALUE NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; - case 533: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA literal_list NK_RP */ -{ yymsp[-5].minor.yy712 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy274)); } + case 535: /* fill_opt ::= FILL NK_LP VALUE_F NK_COMMA expression_list NK_RP */ +{ yymsp[-5].minor.yy448 = createFillNode(pCxt, FILL_MODE_VALUE_F, createNodeListNode(pCxt, yymsp[-1].minor.yy432)); } break; - case 534: /* fill_mode ::= NONE */ -{ yymsp[0].minor.yy94 = FILL_MODE_NONE; } + case 536: /* fill_mode ::= NONE */ +{ yymsp[0].minor.yy46 = FILL_MODE_NONE; } break; - case 535: /* fill_mode ::= PREV */ -{ yymsp[0].minor.yy94 = FILL_MODE_PREV; } + case 537: /* fill_mode ::= PREV */ +{ yymsp[0].minor.yy46 = FILL_MODE_PREV; } break; - case 536: /* fill_mode ::= NULL */ -{ yymsp[0].minor.yy94 = FILL_MODE_NULL; } + case 538: /* fill_mode ::= NULL */ +{ yymsp[0].minor.yy46 = FILL_MODE_NULL; } break; - case 537: /* fill_mode ::= NULL_F */ -{ yymsp[0].minor.yy94 = FILL_MODE_NULL_F; } + case 539: /* fill_mode ::= NULL_F */ +{ yymsp[0].minor.yy46 = FILL_MODE_NULL_F; } break; - case 538: /* fill_mode ::= LINEAR */ -{ yymsp[0].minor.yy94 = FILL_MODE_LINEAR; } + case 540: /* fill_mode ::= LINEAR */ +{ yymsp[0].minor.yy46 = FILL_MODE_LINEAR; } break; - case 539: /* fill_mode ::= NEXT */ -{ yymsp[0].minor.yy94 = FILL_MODE_NEXT; } + case 541: /* fill_mode ::= NEXT */ +{ yymsp[0].minor.yy46 = FILL_MODE_NEXT; } break; - case 542: /* group_by_list ::= expr_or_subquery */ -{ yylhsminor.yy274 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); } - yymsp[0].minor.yy274 = yylhsminor.yy274; + case 544: /* group_by_list ::= expr_or_subquery */ +{ yylhsminor.yy432 = createNodeList(pCxt, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } + yymsp[0].minor.yy432 = yylhsminor.yy432; break; - case 543: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ -{ yylhsminor.yy274 = addNodeToList(pCxt, yymsp[-2].minor.yy274, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy712))); } - yymsp[-2].minor.yy274 = yylhsminor.yy274; + case 545: /* group_by_list ::= group_by_list NK_COMMA expr_or_subquery */ +{ yylhsminor.yy432 = addNodeToList(pCxt, yymsp[-2].minor.yy432, createGroupingSetNode(pCxt, releaseRawExprNode(pCxt, yymsp[0].minor.yy448))); } + yymsp[-2].minor.yy432 = yylhsminor.yy432; break; - case 547: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ -{ yymsp[-5].minor.yy712 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy712), releaseRawExprNode(pCxt, yymsp[-1].minor.yy712)); } + case 549: /* range_opt ::= RANGE NK_LP expr_or_subquery NK_COMMA expr_or_subquery NK_RP */ +{ yymsp[-5].minor.yy448 = createInterpTimeRange(pCxt, releaseRawExprNode(pCxt, yymsp[-3].minor.yy448), releaseRawExprNode(pCxt, yymsp[-1].minor.yy448)); } break; - case 550: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ + case 552: /* query_expression ::= query_simple order_by_clause_opt slimit_clause_opt limit_clause_opt */ { - yylhsminor.yy712 = addOrderByClause(pCxt, yymsp[-3].minor.yy712, yymsp[-2].minor.yy274); - yylhsminor.yy712 = addSlimitClause(pCxt, yylhsminor.yy712, yymsp[-1].minor.yy712); - yylhsminor.yy712 = addLimitClause(pCxt, yylhsminor.yy712, yymsp[0].minor.yy712); + yylhsminor.yy448 = addOrderByClause(pCxt, yymsp[-3].minor.yy448, yymsp[-2].minor.yy432); + yylhsminor.yy448 = addSlimitClause(pCxt, yylhsminor.yy448, yymsp[-1].minor.yy448); + yylhsminor.yy448 = addLimitClause(pCxt, yylhsminor.yy448, yymsp[0].minor.yy448); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 553: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ -{ yylhsminor.yy712 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy712, yymsp[0].minor.yy712); } - yymsp[-3].minor.yy712 = yylhsminor.yy712; + case 555: /* union_query_expression ::= query_simple_or_subquery UNION ALL query_simple_or_subquery */ +{ yylhsminor.yy448 = createSetOperator(pCxt, SET_OP_TYPE_UNION_ALL, yymsp[-3].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-3].minor.yy448 = yylhsminor.yy448; break; - case 554: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ -{ yylhsminor.yy712 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy712, yymsp[0].minor.yy712); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 556: /* union_query_expression ::= query_simple_or_subquery UNION query_simple_or_subquery */ +{ yylhsminor.yy448 = createSetOperator(pCxt, SET_OP_TYPE_UNION, yymsp[-2].minor.yy448, yymsp[0].minor.yy448); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 562: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ - case 566: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==566); -{ yymsp[-1].minor.yy712 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } + case 564: /* slimit_clause_opt ::= SLIMIT NK_INTEGER */ + case 568: /* limit_clause_opt ::= LIMIT NK_INTEGER */ yytestcase(yyruleno==568); +{ yymsp[-1].minor.yy448 = createLimitNode(pCxt, &yymsp[0].minor.yy0, NULL); } break; - case 563: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ - case 567: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==567); -{ yymsp[-3].minor.yy712 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } + case 565: /* slimit_clause_opt ::= SLIMIT NK_INTEGER SOFFSET NK_INTEGER */ + case 569: /* limit_clause_opt ::= LIMIT NK_INTEGER OFFSET NK_INTEGER */ yytestcase(yyruleno==569); +{ yymsp[-3].minor.yy448 = createLimitNode(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0); } break; - case 564: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ - case 568: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==568); -{ yymsp[-3].minor.yy712 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } + case 566: /* slimit_clause_opt ::= SLIMIT NK_INTEGER NK_COMMA NK_INTEGER */ + case 570: /* limit_clause_opt ::= LIMIT NK_INTEGER NK_COMMA NK_INTEGER */ yytestcase(yyruleno==570); +{ yymsp[-3].minor.yy448 = createLimitNode(pCxt, &yymsp[0].minor.yy0, &yymsp[-2].minor.yy0); } break; - case 569: /* subquery ::= NK_LP query_expression NK_RP */ -{ yylhsminor.yy712 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy712); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 571: /* subquery ::= NK_LP query_expression NK_RP */ +{ yylhsminor.yy448 = createRawExprNodeExt(pCxt, &yymsp[-2].minor.yy0, &yymsp[0].minor.yy0, yymsp[-1].minor.yy448); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 574: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ -{ yylhsminor.yy712 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy712), yymsp[-1].minor.yy88, yymsp[0].minor.yy907); } - yymsp[-2].minor.yy712 = yylhsminor.yy712; + case 576: /* sort_specification ::= expr_or_subquery ordering_specification_opt null_ordering_opt */ +{ yylhsminor.yy448 = createOrderByExprNode(pCxt, releaseRawExprNode(pCxt, yymsp[-2].minor.yy448), yymsp[-1].minor.yy666, yymsp[0].minor.yy585); } + yymsp[-2].minor.yy448 = yylhsminor.yy448; break; - case 575: /* ordering_specification_opt ::= */ -{ yymsp[1].minor.yy88 = ORDER_ASC; } + case 577: /* ordering_specification_opt ::= */ +{ yymsp[1].minor.yy666 = ORDER_ASC; } break; - case 576: /* ordering_specification_opt ::= ASC */ -{ yymsp[0].minor.yy88 = ORDER_ASC; } + case 578: /* ordering_specification_opt ::= ASC */ +{ yymsp[0].minor.yy666 = ORDER_ASC; } break; - case 577: /* ordering_specification_opt ::= DESC */ -{ yymsp[0].minor.yy88 = ORDER_DESC; } + case 579: /* ordering_specification_opt ::= DESC */ +{ yymsp[0].minor.yy666 = ORDER_DESC; } break; - case 578: /* null_ordering_opt ::= */ -{ yymsp[1].minor.yy907 = NULL_ORDER_DEFAULT; } + case 580: /* null_ordering_opt ::= */ +{ yymsp[1].minor.yy585 = NULL_ORDER_DEFAULT; } break; - case 579: /* null_ordering_opt ::= NULLS FIRST */ -{ yymsp[-1].minor.yy907 = NULL_ORDER_FIRST; } + case 581: /* null_ordering_opt ::= NULLS FIRST */ +{ yymsp[-1].minor.yy585 = NULL_ORDER_FIRST; } break; - case 580: /* null_ordering_opt ::= NULLS LAST */ -{ yymsp[-1].minor.yy907 = NULL_ORDER_LAST; } + case 582: /* null_ordering_opt ::= NULLS LAST */ +{ yymsp[-1].minor.yy585 = NULL_ORDER_LAST; } break; default: break; diff --git a/source/libs/parser/test/mockCatalog.cpp b/source/libs/parser/test/mockCatalog.cpp index 1af214bfb4033502ece07a0f03455f60bd3bed18..be4f3cd80b08d86d52da4160daf99d0657341d18 100644 --- a/source/libs/parser/test/mockCatalog.cpp +++ b/source/libs/parser/test/mockCatalog.cpp @@ -279,15 +279,13 @@ int32_t __catalogGetDBCfg(SCatalog* pCtg, SRequestConnInfo* pConn, const char* d return g_mockCatalogService->catalogGetDBCfg(dbFName, pDbCfg); } -int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, const char* user, const char* dbFName, AUTH_TYPE type, - bool* pass) { - *pass = true; +int32_t __catalogChkAuth(SCatalog* pCtg, SRequestConnInfo* pConn, SUserAuthInfo *pAuth, SUserAuthRes* pRes) { + pRes->pass = true; return 0; } -int32_t __catalogChkAuthFromCache(SCatalog* pCtg, const char* user, const char* dbFName, AUTH_TYPE type, bool* pass, - bool* exists) { - *pass = true; +int32_t __catalogChkAuthFromCache(SCatalog* pCtg, SUserAuthInfo *pAuth, SUserAuthRes* pRes, bool* exists) { + pRes->pass = true; *exists = true; return 0; } diff --git a/source/libs/parser/test/mockCatalogService.cpp b/source/libs/parser/test/mockCatalogService.cpp index 4d1ef597d0b5907e05dba6cc9c565d3b5ec7d75c..f6a8b407d76071ceb0324ee492655d2959565f7c 100644 --- a/source/libs/parser/test/mockCatalogService.cpp +++ b/source/libs/parser/test/mockCatalogService.cpp @@ -589,8 +589,8 @@ class MockCatalogServiceImpl { *pUserAuthData = taosArrayInit(num, sizeof(SMetaRes)); for (int32_t i = 0; i < num; ++i) { SMetaRes res = {0}; - res.pRes = taosMemoryCalloc(1, sizeof(bool)); - *(bool*)(res.pRes) = true; + res.pRes = taosMemoryCalloc(1, sizeof(SUserAuthRes)); + ((SUserAuthRes*)res.pRes)->pass = true; taosArrayPush(*pUserAuthData, &res); } } diff --git a/source/libs/parser/test/parInitialCTest.cpp b/source/libs/parser/test/parInitialCTest.cpp index 88e66997eb85ff6a65595b7749a38a69762e5239..b7ca944ebb22770b848951f833b00b0e37618244 100644 --- a/source/libs/parser/test/parInitialCTest.cpp +++ b/source/libs/parser/test/parInitialCTest.cpp @@ -760,10 +760,10 @@ TEST_F(ParserInitialCTest, createStable) { addFieldToCreateStbReq(false, "a15", TSDB_DATA_TYPE_VARCHAR, 50 + VARSTR_HEADER_SIZE); run("CREATE STABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, " - "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, " + "a8 BINARY(20), a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, " "a12 TINYINT UNSIGNED, a13 BOOL, a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN) MAX_DELAY 100s,10m WATERMARK 10a,1m " "DELETE_MARK 1000s,200m"); @@ -1023,16 +1023,16 @@ TEST_F(ParserInitialCTest, createTable) { run("CREATE TABLE IF NOT EXISTS test.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3)"); run("CREATE TABLE IF NOT EXISTS rollup_db.t1(" "ts TIMESTAMP, c1 INT, c2 INT UNSIGNED, c3 BIGINT, c4 BIGINT UNSIGNED, c5 FLOAT, c6 DOUBLE, c7 BINARY(20), " - "c8 SMALLINT, c9 SMALLINT UNSIGNED COMMENT 'test column comment', c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " + "c8 SMALLINT, c9 SMALLINT UNSIGNED, c10 TINYINT, c11 TINYINT UNSIGNED, c12 BOOL, " "c13 NCHAR(30), c14 VARCHAR(50)) " "TAGS (a1 TIMESTAMP, a2 INT, a3 INT UNSIGNED, a4 BIGINT, a5 BIGINT UNSIGNED, a6 FLOAT, a7 DOUBLE, a8 BINARY(20), " - "a9 SMALLINT, a10 SMALLINT UNSIGNED COMMENT 'test column comment', a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " + "a9 SMALLINT, a10 SMALLINT UNSIGNED, a11 TINYINT, a12 TINYINT UNSIGNED, a13 BOOL, " "a14 NCHAR(30), a15 VARCHAR(50)) " "TTL 100 COMMENT 'test create table' SMA(c1, c2, c3) ROLLUP (MIN)"); diff --git a/source/libs/planner/src/planLogicCreater.c b/source/libs/planner/src/planLogicCreater.c index c9ee83a64757b6fe951d93fd9c9d917ba357f58b..6544898be9ff389cf441d8824ef06b235fd7cc35 100644 --- a/source/libs/planner/src/planLogicCreater.c +++ b/source/libs/planner/src/planLogicCreater.c @@ -740,6 +740,13 @@ static int32_t createWindowLogicNodeFinalize(SLogicPlanContext* pCxt, SSelectStm code = createColumnByRewriteExprs(pWindow->pFuncs, &pWindow->node.pTargets); } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving) { + pWindow->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pWindow->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + pSelect->hasAggFuncs = false; if (TSDB_CODE_SUCCESS == code) { @@ -1132,6 +1139,14 @@ static int32_t createPartitionLogicNode(SLogicPlanContext* pCxt, SSelectStmt* pS } } + if (TSDB_CODE_SUCCESS == code && NULL != pSelect->pHaving && !pSelect->hasAggFuncs && NULL == pSelect->pGroupByList && + NULL == pSelect->pWindow) { + pPartition->node.pConditions = nodesCloneNode(pSelect->pHaving); + if (NULL == pPartition->node.pConditions) { + code = TSDB_CODE_OUT_OF_MEMORY; + } + } + if (TSDB_CODE_SUCCESS == code) { *pLogicNode = (SLogicNode*)pPartition; } else { diff --git a/source/libs/planner/src/planOptimizer.c b/source/libs/planner/src/planOptimizer.c index 8fee17d96811227d7831b20c629999d3428074e7..4f8b57de5fd1d75d0faae8df5a5a2a2e08108b4b 100644 --- a/source/libs/planner/src/planOptimizer.c +++ b/source/libs/planner/src/planOptimizer.c @@ -1079,11 +1079,23 @@ static bool sortPriKeyOptMayBeOptimized(SLogicNode* pNode) { if (!sortPriKeyOptIsPriKeyOrderBy(pSort->pSortKeys) || 1 != LIST_LENGTH(pSort->node.pChildren)) { return false; } + SNode* pChild; + FOREACH(pChild, pSort->node.pChildren) { + SLogicNode* pSortDescendent = optFindPossibleNode((SLogicNode*)pChild, sortPriKeyOptMayBeOptimized); + if (pSortDescendent != NULL) { + return false; + } + } return true; } static int32_t sortPriKeyOptGetSequencingNodesImpl(SLogicNode* pNode, bool groupSort, bool* pNotOptimize, SNodeList** pSequencingNodes) { + if (NULL != pNode->pLimit || NULL != pNode->pSlimit) { + *pNotOptimize = false; + return TSDB_CODE_SUCCESS; + } + switch (nodeType(pNode)) { case QUERY_NODE_LOGIC_PLAN_SCAN: { SScanLogicNode* pScan = (SScanLogicNode*)pNode; @@ -2127,7 +2139,7 @@ typedef struct SLastRowScanOptLastParaCkCxt { bool hasCol; } SLastRowScanOptLastParaCkCxt; -static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { +static EDealRes lastRowScanOptLastParaIsTagImpl(SNode* pNode, void* pContext) { if (QUERY_NODE_COLUMN == nodeType(pNode)) { SLastRowScanOptLastParaCkCxt* pCxt = pContext; if (COLUMN_TYPE_TAG == ((SColumnNode*)pNode)->colType || COLUMN_TYPE_TBNAME == ((SColumnNode*)pNode)->colType) { @@ -2140,10 +2152,10 @@ static EDealRes lastRowScanOptLastParaCheckImpl(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static bool lastRowScanOptLastParaCheck(SNode* pExpr) { +static bool lastRowScanOptLastParaIsTag(SNode* pExpr) { SLastRowScanOptLastParaCkCxt cxt = {.hasTag = false, .hasCol = false}; - nodesWalkExpr(pExpr, lastRowScanOptLastParaCheckImpl, &cxt); - return !cxt.hasTag && cxt.hasCol; + nodesWalkExpr(pExpr, lastRowScanOptLastParaIsTagImpl, &cxt); + return cxt.hasTag && !cxt.hasCol; } static bool hasSuitableCache(int8_t cacheLastMode, bool hasLastRow, bool hasLast) { @@ -2183,15 +2195,19 @@ static bool lastRowScanOptMayBeOptimized(SLogicNode* pNode) { FOREACH(pFunc, ((SAggLogicNode*)pNode)->pAggFuncs) { SFunctionNode* pAggFunc = (SFunctionNode*)pFunc; if (FUNCTION_TYPE_LAST == pAggFunc->funcType) { - if (hasSelectFunc || !lastRowScanOptLastParaCheck(nodesListGetNode(pAggFunc->pParameterList, 0))) { + if (hasSelectFunc || QUERY_NODE_VALUE == nodeType(nodesListGetNode(pAggFunc->pParameterList, 0))) { return false; } hasLastFunc = true; - } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType || FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + } else if (FUNCTION_TYPE_SELECT_VALUE == pAggFunc->funcType) { if (hasLastFunc) { return false; } hasSelectFunc = true; + } else if (FUNCTION_TYPE_GROUP_KEY == pAggFunc->funcType) { + if (!lastRowScanOptLastParaIsTag(nodesListGetNode(pAggFunc->pParameterList, 0))) { + return false; + } } else if (FUNCTION_TYPE_LAST_ROW != pAggFunc->funcType) { return false; } @@ -2225,7 +2241,7 @@ static EDealRes lastRowScanOptSetColDataType(SNode* pNode, void* pContext) { return DEAL_RES_CONTINUE; } -static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols) { +static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCols, bool erase) { SNode* pTarget = NULL; WHERE_EACH(pTarget, pTargets) { bool found = false; @@ -2237,7 +2253,7 @@ static void lastRowScanOptSetLastTargets(SNodeList* pTargets, SNodeList* pLastCo break; } } - if (!found) { + if (!found && erase) { ERASE_NODE(pTargets); continue; } @@ -2278,9 +2294,8 @@ static int32_t lastRowScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogic pScan->igLastNull = pAgg->hasLast ? true : false; if (NULL != cxt.pLastCols) { cxt.doAgg = false; - lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols); - NODES_DESTORY_LIST(pScan->pScanPseudoCols); - lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols); + lastRowScanOptSetLastTargets(pScan->pScanCols, cxt.pLastCols, true); + lastRowScanOptSetLastTargets(pScan->node.pTargets, cxt.pLastCols, false); nodesClearList(cxt.pLastCols); } pAgg->hasLastRow = false; @@ -2403,6 +2418,36 @@ static bool tagScanOptShouldBeOptimized(SLogicNode* pNode) { return true; } +static SLogicNode* tagScanOptFindAncestorWithSlimit(SLogicNode* pTableScanNode) { + SLogicNode* pNode = pTableScanNode->pParent; + while (NULL != pNode) { + if (QUERY_NODE_LOGIC_PLAN_PARTITION == nodeType(pNode) || QUERY_NODE_LOGIC_PLAN_AGG == nodeType(pNode) || + QUERY_NODE_LOGIC_PLAN_WINDOW == nodeType(pNode) || QUERY_NODE_LOGIC_PLAN_SORT == nodeType(pNode)) { + return NULL; + } + if (NULL != pNode->pSlimit) { + return pNode; + } + pNode = pNode->pParent; + } + return NULL; +} + +static void tagScanOptCloneAncestorSlimit(SLogicNode* pTableScanNode) { + if (NULL != pTableScanNode->pSlimit) { + return; + } + + SLogicNode* pNode = tagScanOptFindAncestorWithSlimit(pTableScanNode); + if (NULL != pNode) { + //TODO: only set the slimit now. push down slimit later + pTableScanNode->pSlimit = nodesCloneNode(pNode->pSlimit); + ((SLimitNode*)pTableScanNode->pSlimit)->limit += ((SLimitNode*)pTableScanNode->pSlimit)->offset; + ((SLimitNode*)pTableScanNode->pSlimit)->offset = 0; + } + return; +} + static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubplan) { SScanLogicNode* pScanNode = (SScanLogicNode*)optFindPossibleNode(pLogicSubplan->pNode, tagScanOptShouldBeOptimized); if (NULL == pScanNode) { @@ -2443,6 +2488,7 @@ static int32_t tagScanOptimize(SOptimizeContext* pCxt, SLogicSubplan* pLogicSubp NODES_CLEAR_LIST(pAgg->pChildren); } nodesDestroyNode((SNode*)pAgg); + tagScanOptCloneAncestorSlimit((SLogicNode*)pScanNode); pCxt->optimized = true; return TSDB_CODE_SUCCESS; } diff --git a/source/libs/qcom/src/querymsg.c b/source/libs/qcom/src/querymsg.c index d2934e1ff86d964cc80e391e53052c25900b0d4a..b62a3e49326181ba6d32b07f307e1e18f5fbc76f 100644 --- a/source/libs/qcom/src/querymsg.c +++ b/source/libs/qcom/src/querymsg.c @@ -587,7 +587,8 @@ int32_t queryProcessRetrieveFuncRsp(void *output, char *msg, int32_t msgSize) { memcpy(output, funcInfo, sizeof(*funcInfo)); taosArrayDestroy(out.pFuncInfos); - + taosArrayDestroy(out.pFuncExtraInfos); + return TSDB_CODE_SUCCESS; } diff --git a/source/libs/scalar/src/scalar.c b/source/libs/scalar/src/scalar.c index fe01977b2e2803e6797df2bc730177eb28aa6617..0521076d23d9662219574388e86960b5dddbea6f 100644 --- a/source/libs/scalar/src/scalar.c +++ b/source/libs/scalar/src/scalar.c @@ -1038,23 +1038,40 @@ bool sclContainsAggFuncNode(SNode *pNode) { return aggFunc; } +static uint8_t sclGetOpValueNodeTsPrecision(SNode *pLeft, SNode *pRight) { + uint8_t lPrec = ((SExprNode *)pLeft)->resType.precision; + uint8_t rPrec = ((SExprNode *)pRight)->resType.precision; + + uint8_t lType = ((SExprNode *)pLeft)->resType.type; + uint8_t rType = ((SExprNode *)pRight)->resType.type; + + if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP == rType) { + return TMAX(lPrec, rPrec); + } else if (TSDB_DATA_TYPE_TIMESTAMP == lType && TSDB_DATA_TYPE_TIMESTAMP != rType) { + return lPrec; + } else if (TSDB_DATA_TYPE_TIMESTAMP == rType && TSDB_DATA_TYPE_TIMESTAMP != lType) { + return rPrec; + } + + return 0; +} int32_t sclConvertOpValueNodeTs(SOperatorNode *node, SScalarCtx *ctx) { int32_t code = 0; - + if (node->pLeft && SCL_IS_VAR_VALUE_NODE(node->pLeft)) { if (node->pRight && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pRight)->resType.type)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pRight)->resType.precision, (SValueNode*)node->pLeft)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pLeft)); } } else if (node->pRight && SCL_IS_NOTNULL_CONST_NODE(node->pRight)) { if (node->pLeft && (TSDB_DATA_TYPE_TIMESTAMP == ((SExprNode *)node->pLeft)->resType.type)) { if (SCL_IS_VAR_VALUE_NODE(node->pRight)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)node->pRight)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, node->pRight), (SValueNode*)node->pRight)); } else if (QUERY_NODE_NODE_LIST == node->pRight->type) { SNode* pNode; FOREACH(pNode, ((SNodeListNode*)node->pRight)->pNodeList) { if (SCL_IS_VAR_VALUE_NODE(pNode)) { - SCL_ERR_JRET(sclConvertToTsValueNode(((SExprNode *)node->pLeft)->resType.precision, (SValueNode*)pNode)); + SCL_ERR_JRET(sclConvertToTsValueNode(sclGetOpValueNodeTsPrecision(node->pLeft, pNode), (SValueNode*)pNode)); } } } @@ -1077,7 +1094,7 @@ int32_t sclConvertCaseWhenValueNodeTs(SCaseWhenNode *node, SScalarCtx *ctx) { if (NULL == node->pCase) { return TSDB_CODE_SUCCESS; } - + if (SCL_IS_VAR_VALUE_NODE(node->pCase)) { SNode* pNode; FOREACH(pNode, node->pWhenThenList) { diff --git a/source/libs/scalar/src/sclfunc.c b/source/libs/scalar/src/sclfunc.c index 24b25cec80ce58390b7a83d71a607bbaba18fe50..835264eabe165f9eaf2fe3869ebb03e13cf2dd05 100644 --- a/source/libs/scalar/src/sclfunc.c +++ b/source/libs/scalar/src/sclfunc.c @@ -1124,7 +1124,8 @@ _end: int32_t toUnixtimestampFunction(SScalarParam *pInput, int32_t inputNum, SScalarParam *pOutput) { int32_t type = GET_PARAM_TYPE(pInput); int64_t timePrec; - GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[1]), pInput[1].columnData->pData); + int32_t idx = (inputNum == 2) ? 1 : 2; + GET_TYPED_DATA(timePrec, int64_t, GET_PARAM_TYPE(&pInput[idx]), pInput[idx].columnData->pData); for (int32_t i = 0; i < pInput[0].numOfRows; ++i) { if (colDataIsNull_s(pInput[0].columnData, i)) { diff --git a/source/libs/scheduler/inc/schInt.h b/source/libs/scheduler/inc/schInt.h index d002b5dfa96f182d343d85390f32263fcd4912a8..7840fe2017a6f384e70f1553105c7293a4abb2d7 100644 --- a/source/libs/scheduler/inc/schInt.h +++ b/source/libs/scheduler/inc/schInt.h @@ -54,7 +54,6 @@ typedef enum { #define SCHEDULE_DEFAULT_MAX_JOB_NUM 1000 #define SCHEDULE_DEFAULT_MAX_TASK_NUM 1000 -#define SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM 200 // unit is TSDB_TABLE_NUM_UNIT #define SCHEDULE_DEFAULT_POLICY SCH_LOAD_SEQ #define SCHEDULE_DEFAULT_MAX_NODE_NUM 20 @@ -134,7 +133,7 @@ typedef struct SSchStatusFps { typedef struct SSchedulerCfg { uint32_t maxJobNum; - int32_t maxNodeTableNum; + int64_t maxNodeTableNum; SCH_POLICY schPolicy; bool enableReSchedule; } SSchedulerCfg; @@ -175,7 +174,7 @@ typedef struct SSchHbCallbackParam { typedef struct SSchFlowControl { SRWLatch lock; bool sorted; - int32_t tableNumSum; + int64_t tableNumSum; uint32_t execTaskNum; SArray *taskList; // Element is SSchTask* } SSchFlowControl; diff --git a/source/libs/scheduler/src/schFlowCtrl.c b/source/libs/scheduler/src/schFlowCtrl.c index 9cb95a6bbe7aad0bf9a6576dac13db3114d1559d..8c2b65e1259685fcb1fb8502cbf3710faa4e426c 100644 --- a/source/libs/scheduler/src/schFlowCtrl.c +++ b/source/libs/scheduler/src/schFlowCtrl.c @@ -46,7 +46,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { return TSDB_CODE_SUCCESS; } - int32_t sum = 0; + int64_t sum = 0; int32_t taskNum = taosArrayGetSize(pJob->dataSrcTasks); for (int32_t i = 0; i < taskNum; ++i) { SSchTask *pTask = *(SSchTask **)taosArrayGet(pJob->dataSrcTasks, i); @@ -55,7 +55,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { } if (schMgmt.cfg.maxNodeTableNum <= 0 || sum < schMgmt.cfg.maxNodeTableNum) { - SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%d", sum); + SCH_JOB_DLOG("job no need flow ctrl, totalTableNum:%" PRId64, sum); return TSDB_CODE_SUCCESS; } @@ -68,7 +68,7 @@ int32_t schChkJobNeedFlowCtrl(SSchJob *pJob, SSchLevel *pLevel) { SCH_SET_JOB_NEED_FLOW_CTRL(pJob); - SCH_JOB_DLOG("job NEED flow ctrl, totalTableNum:%d", sum); + SCH_JOB_DLOG("job NEED flow ctrl, totalTableNum:%" PRId64, sum); return TSDB_CODE_SUCCESS; } @@ -94,7 +94,7 @@ int32_t schDecTaskFlowQuota(SSchJob *pJob, SSchTask *pTask) { --ctrl->execTaskNum; ctrl->tableNumSum -= pTask->plan->execNodeStat.tableNum; - SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task quota removed, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); _return: @@ -125,7 +125,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { SCH_ERR_RET(TSDB_CODE_OUT_OF_MEMORY); } - SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task quota added, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, nctrl.tableNumSum, nctrl.execTaskNum); *enough = true; @@ -142,7 +142,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { break; } - int32_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum; + int64_t sum = pTask->plan->execNodeStat.tableNum + ctrl->tableNumSum; if (sum <= schMgmt.cfg.maxNodeTableNum) { ctrl->tableNumSum = sum; @@ -173,7 +173,7 @@ int32_t schCheckIncTaskFlowQuota(SSchJob *pJob, SSchTask *pTask, bool *enough) { _return: - SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", + SCH_TASK_DLOG("task quota %s added, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ((*enough) ? "" : "NOT"), ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); @@ -203,7 +203,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { return TSDB_CODE_SUCCESS; } - int32_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; + int64_t remainNum = schMgmt.cfg.maxNodeTableNum - ctrl->tableNumSum; int32_t taskNum = taosArrayGetSize(ctrl->taskList); int32_t code = 0; SSchTask *pTask = NULL; @@ -217,7 +217,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { SEp *ep = SCH_GET_CUR_EP(&pTask->plan->execNode); if (pTask->plan->execNodeStat.tableNum > remainNum && ctrl->execTaskNum > 0) { - SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task NOT to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); continue; @@ -228,14 +228,14 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { taosArrayRemove(ctrl->taskList, i); - SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, + SCH_TASK_DLOG("task to launch, fqdn:%s, port:%d, tableNum:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, pTask->plan->execNodeStat.tableNum, ctrl->tableNumSum, ctrl->execTaskNum); SCH_ERR_JRET(schAsyncLaunchTaskImpl(pJob, pTask)); remainNum -= pTask->plan->execNodeStat.tableNum; if (remainNum <= 0) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d", ep->fqdn, ep->port, + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d", ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum); break; @@ -244,7 +244,7 @@ int32_t schLaunchTasksInFlowCtrlListImpl(SSchJob *pJob, SSchFlowControl *ctrl) { if (i < (taskNum - 1)) { SSchTask *pLastTask = *(SSchTask **)taosArrayGetLast(ctrl->taskList); if (remainNum < pLastTask->plan->execNodeStat.tableNum) { - SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%d, remainExecTaskNum:%d, smallestInList:%d", + SCH_TASK_DLOG("no more task to launch, fqdn:%s, port:%d, remainNum:%" PRId64 ", remainExecTaskNum:%d, smallestInList:%d", ep->fqdn, ep->port, ctrl->tableNumSum, ctrl->execTaskNum, pLastTask->plan->execNodeStat.tableNum); break; diff --git a/source/libs/scheduler/src/scheduler.c b/source/libs/scheduler/src/scheduler.c index 2b46a4710e24215e9b32009f4fb97c8204963fe6..e7561ccb7ecf68c5030e12a19c04f3d1a9661cb5 100644 --- a/source/libs/scheduler/src/scheduler.c +++ b/source/libs/scheduler/src/scheduler.c @@ -18,6 +18,7 @@ #include "schInt.h" #include "tmsg.h" #include "tref.h" +#include "tglobal.h" SSchedulerMgmt schMgmt = { .jobRef = -1, @@ -30,11 +31,12 @@ int32_t schedulerInit() { } schMgmt.cfg.maxJobNum = SCHEDULE_DEFAULT_MAX_JOB_NUM; - schMgmt.cfg.maxNodeTableNum = SCHEDULE_DEFAULT_MAX_NODE_TABLE_NUM; + schMgmt.cfg.maxNodeTableNum = tsQueryMaxConcurrentTables; schMgmt.cfg.schPolicy = SCHEDULE_DEFAULT_POLICY; schMgmt.cfg.enableReSchedule = true; - qDebug("schedule policy init to %d", schMgmt.cfg.schPolicy); + qDebug("schedule init, policy: %d, maxNodeTableNum: %" PRId64", reSchedule:%d", + schMgmt.cfg.schPolicy, schMgmt.cfg.maxNodeTableNum, schMgmt.cfg.enableReSchedule); schMgmt.jobRef = taosOpenRef(schMgmt.cfg.maxJobNum, schFreeJobImpl); if (schMgmt.jobRef < 0) { diff --git a/source/libs/stream/CMakeLists.txt b/source/libs/stream/CMakeLists.txt index ceddf4f2153735c4d8423dde806272419dc35fc5..790547bb6154e6a53847f609d918fe4fac784c29 100644 --- a/source/libs/stream/CMakeLists.txt +++ b/source/libs/stream/CMakeLists.txt @@ -9,7 +9,7 @@ target_include_directories( target_link_libraries( stream PUBLIC tdb - PRIVATE os util transport qcom executor + PRIVATE os util transport qcom executor wal ) if(${BUILD_TEST}) diff --git a/source/libs/stream/inc/streamInc.h b/source/libs/stream/inc/streamInc.h index 66496f11f8b2da8ad46d91637efdeb1444f26319..876b80697a9679125350f8b9b2a1e26e4fdb0572 100644 --- a/source/libs/stream/inc/streamInc.h +++ b/source/libs/stream/inc/streamInc.h @@ -44,7 +44,7 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, SEpSet* pEpSet); -SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem); +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem); #ifdef __cplusplus } diff --git a/source/libs/stream/src/stream.c b/source/libs/stream/src/stream.c index a4a02b7d65b77a6b27539a9b9df513264c2ebcaf..0f000f1f5017e6bf1cd737ee424a4ed72654ac5f 100644 --- a/source/libs/stream/src/stream.c +++ b/source/libs/stream/src/stream.c @@ -16,6 +16,8 @@ #include "streamInc.h" #include "ttimer.h" +#define STREAM_TASK_INPUT_QUEUEU_CAPACITY 100000 + int32_t streamInit() { int8_t old; while (1) { @@ -50,7 +52,7 @@ void streamCleanUp() { void streamSchedByTimer(void* param, void* tmrId) { SStreamTask* pTask = (void*)param; - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { streamMetaReleaseTask(NULL, pTask); return; } @@ -64,15 +66,16 @@ void streamSchedByTimer(void* param, void* tmrId) { taosFreeQitem(trigger); return; } - trigger->pBlock->info.type = STREAM_GET_ALL; + trigger->pBlock->info.type = STREAM_GET_ALL; atomic_store_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE); - if (streamTaskInput(pTask, (SStreamQueueItem*)trigger) < 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)trigger) < 0) { taosFreeQitem(trigger); taosTmrReset(streamSchedByTimer, (int32_t)pTask->triggerParam, pTask, streamEnv.timer, &pTask->timer); return; } + streamSchedExec(pTask); } @@ -91,31 +94,33 @@ int32_t streamSetupTrigger(SStreamTask* pTask) { int32_t streamSchedExec(SStreamTask* pTask) { int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE, TASK_SCHED_STATUS__WAITING); + if (schedStatus == TASK_SCHED_STATUS__INACTIVE) { SStreamTaskRunReq* pRunReq = rpcMallocCont(sizeof(SStreamTaskRunReq)); if (pRunReq == NULL) { - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); return -1; } + pRunReq->head.vgId = pTask->nodeId; - pRunReq->streamId = pTask->streamId; - pRunReq->taskId = pTask->taskId; - SRpcMsg msg = { - .msgType = TDMT_STREAM_TASK_RUN, - .pCont = pRunReq, - .contLen = sizeof(SStreamTaskRunReq), - }; + pRunReq->streamId = pTask->id.streamId; + pRunReq->taskId = pTask->id.taskId; + + SRpcMsg msg = { .msgType = TDMT_STREAM_TASK_RUN, .pCont = pRunReq, .contLen = sizeof(SStreamTaskRunReq) }; tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &msg); + qDebug("trigger to run s-task:%s", pTask->id.idStr); } + return 0; } -int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { +int32_t streamTaskEnqueueBlocks(SStreamTask* pTask, const SStreamDispatchReq* pReq, SRpcMsg* pRsp) { SStreamDataBlock* pData = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); int8_t status; - // enqueue + // enqueue data block if (pData != NULL) { pData->type = STREAM_INPUT__DATA_BLOCK; pData->srcVgId = pReq->dataSrcVgId; @@ -123,10 +128,10 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamDispatchReqToData(pReq, pData); - if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; - } else { - status = TASK_INPUT_STATUS__FAILED; + } else { // input queue is full, upstream is blocked now + status = TASK_INPUT_STATUS__BLOCKED; } } else { streamTaskInputFail(pTask); @@ -142,10 +147,12 @@ int32_t streamTaskEnqueue(SStreamTask* pTask, const SStreamDispatchReq* pReq, SR pCont->upstreamNodeId = htonl(pReq->upstreamNodeId); pCont->upstreamTaskId = htonl(pReq->upstreamTaskId); pCont->downstreamNodeId = htonl(pTask->nodeId); - pCont->downstreamTaskId = htonl(pTask->taskId); + pCont->downstreamTaskId = htonl(pTask->id.taskId); pRsp->pCont = buf; + pRsp->contLen = sizeof(SMsgHead) + sizeof(SStreamDispatchRsp); tmsgSendRsp(pRsp); + return status == TASK_INPUT_STATUS__NORMAL ? 0 : -1; } @@ -155,7 +162,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, // enqueue if (pData != NULL) { - qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->taskId, pTask->selfChildId, + qDebug("task %d(child %d) recv retrieve req from task %d, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, pReq->srcTaskId, pReq->reqId); pData->type = STREAM_INPUT__DATA_RETRIEVE; @@ -164,7 +171,7 @@ int32_t streamTaskEnqueueRetrieve(SStreamTask* pTask, SStreamRetrieveReq* pReq, /*pData->blocks = pReq->data;*/ /*pBlock->sourceVer = pReq->sourceVer;*/ streamRetrieveReqToData(pReq, pData); - if (streamTaskInput(pTask, (SStreamQueueItem*)pData) == 0) { + if (tAppendDataToInputQueue(pTask, (SStreamQueueItem*)pData) == 0) { status = TASK_INPUT_STATUS__NORMAL; } else { status = TASK_INPUT_STATUS__FAILED; @@ -205,10 +212,10 @@ int32_t streamTaskOutput(SStreamTask* pTask, SStreamDataBlock* pBlock) { } int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, SRpcMsg* pRsp, bool exec) { - qDebug("task %d receive dispatch req from node %d task %d", pTask->taskId, pReq->upstreamNodeId, + qDebug("vgId:%d s-task:%s receive dispatch req from taskId:%d", pReq->upstreamNodeId, pTask->id.idStr, pReq->upstreamTaskId); - streamTaskEnqueue(pTask, pReq, pRsp); + streamTaskEnqueueBlocks(pTask, pReq, pRsp); tDeleteStreamDispatchReq(pReq); if (exec) { @@ -228,13 +235,14 @@ int32_t streamProcessDispatchReq(SStreamTask* pTask, SStreamDispatchReq* pReq, S int32_t streamProcessDispatchRsp(SStreamTask* pTask, SStreamDispatchRsp* pRsp, int32_t code) { ASSERT(pRsp->inputStatus == TASK_OUTPUT_STATUS__NORMAL || pRsp->inputStatus == TASK_OUTPUT_STATUS__BLOCKED); - - qDebug("task %d receive dispatch rsp, code: %x", pTask->taskId, code); + qDebug("s-task:%s receive dispatch rsp, code: %x", pTask->id.idStr, code); if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { int32_t leftRsp = atomic_sub_fetch_32(&pTask->shuffleDispatcher.waitingRspCnt, 1); - qDebug("task %d is shuffle, left waiting rsp %d", pTask->taskId, leftRsp); - if (leftRsp > 0) return 0; + qDebug("task %d is shuffle, left waiting rsp %d", pTask->id.taskId, leftRsp); + if (leftRsp > 0) { + return 0; + } } int8_t old = atomic_exchange_8(&pTask->outputStatus, pRsp->inputStatus); @@ -261,7 +269,7 @@ int32_t streamProcessRunReq(SStreamTask* pTask) { } int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, SRpcMsg* pRsp) { - qDebug("task %d receive retrieve req from node %d task %d", pTask->taskId, pReq->srcNodeId, pReq->srcTaskId); + qDebug("task %d receive retrieve req from node %d task %d", pTask->id.taskId, pReq->srcNodeId, pReq->srcTaskId); streamTaskEnqueueRetrieve(pTask, pReq, pRsp); @@ -275,7 +283,77 @@ int32_t streamProcessRetrieveReq(SStreamTask* pTask, SStreamRetrieveReq* pReq, S return 0; } -// int32_t streamProcessRetrieveRsp(SStreamTask* pTask, SStreamRetrieveRsp* pRsp) { -// // -// return 0; -// } +bool tInputQueueIsFull(const SStreamTask* pTask) { + return taosQueueItemSize((pTask->inputQueue->queue)) >= STREAM_TASK_INPUT_QUEUEU_CAPACITY; +} + +int32_t tAppendDataToInputQueue(SStreamTask* pTask, SStreamQueueItem* pItem) { + int8_t type = pItem->type; + + if (type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit2* pSubmitBlock = streamSubmitBlockClone((SStreamDataSubmit2*)pItem); + if (pSubmitBlock == NULL) { + qDebug("task %d %p submit enqueue failed since out of memory", pTask->id.taskId, pTask); + terrno = TSDB_CODE_OUT_OF_MEMORY; + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); + return -1; + } + + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; + qDebug("s-task:%s submit enqueue %p %p msgLen:%d ver:%" PRId64 ", total in queue:%d", pTask->id.idStr, + pItem, pSubmitBlock->submit.msgStr, pSubmitBlock->submit.msgLen, + pSubmitBlock->submit.ver, total); + + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { + qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); + streamDataSubmitDestroy(pSubmitBlock); + return -1; + } + + taosWriteQitem(pTask->inputQueue->queue, pSubmitBlock); + } else if (type == STREAM_INPUT__DATA_BLOCK || type == STREAM_INPUT__DATA_RETRIEVE || + type == STREAM_INPUT__REF_DATA_BLOCK) { + int32_t total = taosQueueItemSize(pTask->inputQueue->queue) + 1; + if ((pTask->taskLevel == TASK_LEVEL__SOURCE) && total > STREAM_TASK_INPUT_QUEUEU_CAPACITY) { + qError("s-task:%s input queue is full, capacity:%d, abort", pTask->id.idStr, STREAM_TASK_INPUT_QUEUEU_CAPACITY); + return -1; + } + + qDebug("s-task:%s data block enqueue, total in queue:%d", pTask->id.idStr, total); + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (type == STREAM_INPUT__CHECKPOINT) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } else if (type == STREAM_INPUT__GET_RES) { + taosWriteQitem(pTask->inputQueue->queue, pItem); + } + + if (type != STREAM_INPUT__GET_RES && type != STREAM_INPUT__CHECKPOINT && pTask->triggerParam != 0) { + atomic_val_compare_exchange_8(&pTask->triggerStatus, TASK_TRIGGER_STATUS__INACTIVE, TASK_TRIGGER_STATUS__ACTIVE); + } + +#if 0 + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__NORMAL); +#endif + + return 0; +} + +void* streamQueueNextItem(SStreamQueue* queue) { + int8_t dequeueFlag = atomic_exchange_8(&queue->status, STREAM_QUEUE__PROCESSING); + if (dequeueFlag == STREAM_QUEUE__FAILED) { + ASSERT(queue->qItem != NULL); + return streamQueueCurItem(queue); + } else { + queue->qItem = NULL; + taosGetQitem(queue->qall, &queue->qItem); + if (queue->qItem == NULL) { + taosReadAllQitems(queue->queue, queue->qall); + taosGetQitem(queue->qall, &queue->qItem); + } + return streamQueueCurItem(queue); + } +} + +void streamTaskInputFail(SStreamTask* pTask) { + atomic_store_8(&pTask->inputStatus, TASK_INPUT_STATUS__FAILED); +} \ No newline at end of file diff --git a/source/libs/stream/src/streamData.c b/source/libs/stream/src/streamData.c index 8baebaee4224e6b9af5d2eacf62910412171bc63..ae616260f3b38038f103b661b4a05aabace9ac31 100644 --- a/source/libs/stream/src/streamData.c +++ b/source/libs/stream/src/streamData.c @@ -48,10 +48,12 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock if (pArray == NULL) { return -1; } + taosArrayPush(pArray, &(SSDataBlock){0}); SRetrieveTableRsp* pRetrieve = pReq->pRetrieve; SSDataBlock* pDataBlock = taosArrayGet(pArray, 0); blockDecode(pDataBlock, pRetrieve->data); + // TODO: refactor pDataBlock->info.window.skey = be64toh(pRetrieve->skey); pDataBlock->info.window.ekey = be64toh(pRetrieve->ekey); @@ -65,35 +67,53 @@ int32_t streamRetrieveReqToData(const SStreamRetrieveReq* pReq, SStreamDataBlock return 0; } -SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit) { +SStreamDataSubmit2* streamDataSubmitNew(SPackedData submit, int32_t type) { SStreamDataSubmit2* pDataSubmit = (SStreamDataSubmit2*)taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); + if (pDataSubmit == NULL) { + return NULL; + } - if (pDataSubmit == NULL) return NULL; pDataSubmit->dataRef = (int32_t*)taosMemoryMalloc(sizeof(int32_t)); - if (pDataSubmit->dataRef == NULL) goto FAIL; + if (pDataSubmit->dataRef == NULL) { + taosFreeQitem(pDataSubmit); + return NULL; + } + pDataSubmit->submit = submit; - *pDataSubmit->dataRef = 1; - pDataSubmit->type = STREAM_INPUT__DATA_SUBMIT; + *pDataSubmit->dataRef = 1; // initialize the reference count to be 1 + pDataSubmit->type = type; + return pDataSubmit; -FAIL: - taosFreeQitem(pDataSubmit); - return NULL; +} + +void streamDataSubmitDestroy(SStreamDataSubmit2* pDataSubmit) { + int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); + ASSERT(ref >= 0 && pDataSubmit->type == STREAM_INPUT__DATA_SUBMIT); + + if (ref == 0) { + taosMemoryFree(pDataSubmit->submit.msgStr); + taosMemoryFree(pDataSubmit->dataRef); + } } SStreamMergedSubmit2* streamMergedSubmitNew() { SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)taosAllocateQitem(sizeof(SStreamMergedSubmit2), DEF_QITEM, 0); + if (pMerged == NULL) { + return NULL; + } - if (pMerged == NULL) return NULL; pMerged->submits = taosArrayInit(0, sizeof(SPackedData)); pMerged->dataRefs = taosArrayInit(0, sizeof(void*)); - if (pMerged->dataRefs == NULL || pMerged->submits == NULL) goto FAIL; + + if (pMerged->dataRefs == NULL || pMerged->submits == NULL) { + taosArrayDestroy(pMerged->submits); + taosArrayDestroy(pMerged->dataRefs); + taosFreeQitem(pMerged); + return NULL; + } + pMerged->type = STREAM_INPUT__MERGED_SUBMIT; return pMerged; -FAIL: - if (pMerged->submits) taosArrayDestroy(pMerged->submits); - if (pMerged->dataRefs) taosArrayDestroy(pMerged->dataRefs); - taosFreeQitem(pMerged); - return NULL; } int32_t streamMergeSubmit(SStreamMergedSubmit2* pMerged, SStreamDataSubmit2* pSubmit) { @@ -107,48 +127,38 @@ static FORCE_INLINE void streamDataSubmitRefInc(SStreamDataSubmit2* pDataSubmit) atomic_add_fetch_32(pDataSubmit->dataRef, 1); } -SStreamDataSubmit2* streamSubmitRefClone(SStreamDataSubmit2* pSubmit) { +SStreamDataSubmit2* streamSubmitBlockClone(SStreamDataSubmit2* pSubmit) { SStreamDataSubmit2* pSubmitClone = taosAllocateQitem(sizeof(SStreamDataSubmit2), DEF_QITEM, 0); - if (pSubmitClone == NULL) { return NULL; } + streamDataSubmitRefInc(pSubmit); memcpy(pSubmitClone, pSubmit, sizeof(SStreamDataSubmit2)); return pSubmitClone; } -void streamDataSubmitRefDec(SStreamDataSubmit2* pDataSubmit) { - int32_t ref = atomic_sub_fetch_32(pDataSubmit->dataRef, 1); - ASSERT(ref >= 0); - if (ref == 0) { - taosMemoryFree(pDataSubmit->submit.msgStr); - taosMemoryFree(pDataSubmit->dataRef); - } -} - -SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* elem) { - ASSERT(elem); - if (dst->type == STREAM_INPUT__DATA_BLOCK && elem->type == STREAM_INPUT__DATA_BLOCK) { +SStreamQueueItem* streamMergeQueueItem(SStreamQueueItem* dst, SStreamQueueItem* pElem) { + if (dst->type == STREAM_INPUT__DATA_BLOCK && pElem->type == STREAM_INPUT__DATA_BLOCK) { SStreamDataBlock* pBlock = (SStreamDataBlock*)dst; - SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)elem; + SStreamDataBlock* pBlockSrc = (SStreamDataBlock*)pElem; taosArrayAddAll(pBlock->blocks, pBlockSrc->blocks); taosArrayDestroy(pBlockSrc->blocks); - taosFreeQitem(elem); + taosFreeQitem(pElem); return dst; - } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { + } else if (dst->type == STREAM_INPUT__MERGED_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)dst; - SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)elem; + SStreamDataSubmit2* pBlockSrc = (SStreamDataSubmit2*)pElem; streamMergeSubmit(pMerged, pBlockSrc); - taosFreeQitem(elem); + taosFreeQitem(pElem); return dst; - } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && elem->type == STREAM_INPUT__DATA_SUBMIT) { + } else if (dst->type == STREAM_INPUT__DATA_SUBMIT && pElem->type == STREAM_INPUT__DATA_SUBMIT) { SStreamMergedSubmit2* pMerged = streamMergedSubmitNew(); ASSERT(pMerged); streamMergeSubmit(pMerged, (SStreamDataSubmit2*)dst); - streamMergeSubmit(pMerged, (SStreamDataSubmit2*)elem); + streamMergeSubmit(pMerged, (SStreamDataSubmit2*)pElem); taosFreeQitem(dst); - taosFreeQitem(elem); + taosFreeQitem(pElem); return (SStreamQueueItem*)pMerged; } else { return NULL; @@ -164,7 +174,7 @@ void streamFreeQitem(SStreamQueueItem* data) { taosArrayDestroyEx(((SStreamDataBlock*)data)->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(data); } else if (type == STREAM_INPUT__DATA_SUBMIT) { - streamDataSubmitRefDec((SStreamDataSubmit2*)data); + streamDataSubmitDestroy((SStreamDataSubmit2*)data); taosFreeQitem(data); } else if (type == STREAM_INPUT__MERGED_SUBMIT) { SStreamMergedSubmit2* pMerge = (SStreamMergedSubmit2*)data; diff --git a/source/libs/stream/src/streamDispatch.c b/source/libs/stream/src/streamDispatch.c index 7e7c23f98addfae5689093bedb1a86196cd380c9..549374ed9402b3abb775e804d472845a47c44f53 100644 --- a/source/libs/stream/src/streamDispatch.c +++ b/source/libs/stream/src/streamDispatch.c @@ -121,9 +121,9 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) int32_t actualLen = blockEncode(pBlock, pRetrieve->data, numOfCols); SStreamRetrieveReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .srcNodeId = pTask->nodeId, - .srcTaskId = pTask->taskId, + .srcTaskId = pTask->id.taskId, .pRetrieve = pRetrieve, .retrieveLen = dataStrLen, }; @@ -168,7 +168,7 @@ int32_t streamBroadcastToChildren(SStreamTask* pTask, const SSDataBlock* pBlock) } buf = NULL; - qDebug("task %d(child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->taskId, + qDebug("s-task:%s (child %d) send retrieve req to task %d at node %d, reqId %" PRId64, pTask->id.idStr, pTask->selfChildId, pEpInfo->taskId, pEpInfo->nodeId, req.reqId); } code = 0; @@ -230,22 +230,21 @@ int32_t streamDispatchOneCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* SEncoder encoder; tEncoderInit(&encoder, abuf, tlen); if ((code = tEncodeSStreamTaskCheckReq(&encoder, pReq)) < 0) { - goto FAIL; + rpcFreeCont(buf); + return code; } + tEncoderClear(&encoder); msg.contLen = tlen + sizeof(SMsgHead); msg.pCont = buf; msg.msgType = TDMT_STREAM_TASK_CHECK; - qDebug("dispatch from task %d to task %d node %d: check msg", pTask->taskId, pReq->downstreamTaskId, nodeId); + qDebug("dispatch from s-task:%s to downstream s-task:%" PRIx64 ":%d node %d: check msg", pTask->id.idStr, + pReq->streamId, pReq->downstreamTaskId, nodeId); tmsgSendReq(pEpSet, &msg); - return 0; -FAIL: - if (buf) rpcFreeCont(buf); - return code; } int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecoverFinishReq* pReq, int32_t vgId, @@ -282,7 +281,7 @@ int32_t streamDispatchOneRecoverFinishReq(SStreamTask* pTask, const SStreamRecov tmsgSendReq(pEpSet, &msg); - qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->taskId, pReq->taskId, vgId); + qDebug("dispatch from task %d to task %d node %d: recover finish msg", pTask->id.taskId, pReq->taskId, vgId); return 0; FAIL: @@ -319,8 +318,7 @@ int32_t streamDispatchOneDataReq(SStreamTask* pTask, const SStreamDispatchReq* p msg.pCont = buf; msg.msgType = pTask->dispatchMsgType; - qDebug("dispatch from task %d to task %d node %d: data msg", pTask->taskId, pReq->taskId, vgId); - + qDebug("dispatch from s-task:%s to taskId:%d vgId:%d data msg", pTask->id.idStr, pReq->taskId, vgId); tmsgSendReq(pEpSet, &msg); code = 0; @@ -382,9 +380,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { SStreamDispatchReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .dataSrcVgId = pData->srcVgId, - .upstreamTaskId = pTask->taskId, + .upstreamTaskId = pTask->id.taskId, .upstreamChildId = pTask->selfChildId, .upstreamNodeId = pTask->nodeId, .blockNum = blockNum, @@ -402,14 +400,15 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat goto FAIL_FIXED_DISPATCH; } } + int32_t vgId = pTask->fixedEpDispatcher.nodeId; SEpSet* pEpSet = &pTask->fixedEpDispatcher.epSet; int32_t downstreamTaskId = pTask->fixedEpDispatcher.taskId; req.taskId = downstreamTaskId; - qDebug("dispatch from task %d (child id %d) to down stream task %d in vnode %d", pTask->taskId, pTask->selfChildId, - downstreamTaskId, vgId); + qDebug("s-task:%s (child taskId:%d) dispatch blocks:%d to down stream s-task:%d in vgId:%d", pTask->id.idStr, + pTask->selfChildId, blockNum, downstreamTaskId, vgId); if (streamDispatchOneDataReq(pTask, &req, vgId, pEpSet) < 0) { goto FAIL_FIXED_DISPATCH; @@ -432,9 +431,9 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat } for (int32_t i = 0; i < vgSz; i++) { - pReqs[i].streamId = pTask->streamId; + pReqs[i].streamId = pTask->id.streamId; pReqs[i].dataSrcVgId = pData->srcVgId; - pReqs[i].upstreamTaskId = pTask->taskId; + pReqs[i].upstreamTaskId = pTask->id.taskId; pReqs[i].upstreamChildId = pTask->selfChildId; pReqs[i].upstreamNodeId = pTask->nodeId; pReqs[i].blockNum = 0; @@ -494,6 +493,8 @@ int32_t streamDispatchAllBlocks(SStreamTask* pTask, const SStreamDataBlock* pDat int32_t streamDispatch(SStreamTask* pTask) { ASSERT(pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH); + qDebug("s-task:%s try to dispatch intermediate result block to downstream, numofBlocks in outputQ:%d", pTask->id.idStr, + taosQueueItemSize(pTask->outputQueue->queue)); int8_t old = atomic_val_compare_exchange_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL, TASK_OUTPUT_STATUS__WAIT); @@ -503,13 +504,12 @@ int32_t streamDispatch(SStreamTask* pTask) { SStreamDataBlock* pBlock = streamQueueNextItem(pTask->outputQueue); if (pBlock == NULL) { - qDebug("stream stop dispatching since no output: task %d", pTask->taskId); + qDebug("s-task:%s stream stop dispatching since no output in output queue", pTask->id.idStr); atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); return 0; } - ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); - qDebug("stream dispatching: task %d", pTask->taskId); + ASSERT(pBlock->type == STREAM_INPUT__DATA_BLOCK); int32_t code = 0; if (streamDispatchAllBlocks(pTask, pBlock) < 0) { @@ -518,6 +518,7 @@ int32_t streamDispatch(SStreamTask* pTask) { atomic_store_8(&pTask->outputStatus, TASK_OUTPUT_STATUS__NORMAL); goto FREE; } + FREE: taosArrayDestroyEx(pBlock->blocks, (FDelete)blockDataFreeRes); taosFreeQitem(pBlock); diff --git a/source/libs/stream/src/streamExec.c b/source/libs/stream/src/streamExec.c index 25b265636586bf74e6cb836b89308f6cd20d4122..9a6ff302efc3b470f9e71caf07e2d0ae9d70a13f 100644 --- a/source/libs/stream/src/streamExec.c +++ b/source/libs/stream/src/streamExec.c @@ -18,70 +18,82 @@ #define STREAM_EXEC_MAX_BATCH_NUM 100 static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* pRes) { - int32_t code; - void* exec = pTask->exec.executor; - while(pTask->taskLevel == TASK_LEVEL__SOURCE && atomic_load_8(&pTask->taskStatus) != TASK_STATUS__NORMAL) { - qError("stream task wait for the end of fill history"); - taosMsleep(2); - continue; + int32_t code = TSDB_CODE_SUCCESS; + void* pExecutor = pTask->exec.pExecutor; + + while (pTask->taskLevel == TASK_LEVEL__SOURCE) { + int8_t status = atomic_load_8(&pTask->status.taskStatus); + if (status != TASK_STATUS__NORMAL && status != TASK_STATUS__RESTORE) { + qError("stream task wait for the end of fill history, s-task:%s, status:%d", pTask->id.idStr, + atomic_load_8(&pTask->status.taskStatus)); + taosMsleep(2); + } else { + break; + } } // set input const SStreamQueueItem* pItem = (const SStreamQueueItem*)data; if (pItem->type == STREAM_INPUT__GET_RES) { const SStreamTrigger* pTrigger = (const SStreamTrigger*)data; - qSetMultiStreamInput(exec, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pExecutor, pTrigger->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__DATA_SUBMIT) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); const SStreamDataSubmit2* pSubmit = (const SStreamDataSubmit2*)data; - qDebug("task %d %p set submit input %p %p %d %" PRId64, pTask->taskId, pTask, pSubmit, pSubmit->submit.msgStr, + qSetMultiStreamInput(pExecutor, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); + qDebug("s-task:%s set submit blocks as source block completed, %p %p len:%d ver:%" PRId64, pTask->id.idStr, pSubmit, pSubmit->submit.msgStr, pSubmit->submit.msgLen, pSubmit->submit.ver); - qSetMultiStreamInput(exec, &pSubmit->submit, 1, STREAM_INPUT__DATA_SUBMIT); } else if (pItem->type == STREAM_INPUT__DATA_BLOCK || pItem->type == STREAM_INPUT__DATA_RETRIEVE) { const SStreamDataBlock* pBlock = (const SStreamDataBlock*)data; - SArray* blocks = pBlock->blocks; - qDebug("task %d %p set ssdata input", pTask->taskId, pTask); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__DATA_BLOCK); + + SArray* pBlockList = pBlock->blocks; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("s-task:%s set sdata blocks as input num:%d, ver:%"PRId64, pTask->id.idStr, numOfBlocks, pBlock->sourceVer); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__DATA_BLOCK); } else if (pItem->type == STREAM_INPUT__MERGED_SUBMIT) { const SStreamMergedSubmit2* pMerged = (const SStreamMergedSubmit2*)data; - SArray* blocks = pMerged->submits; - qDebug("task %d %p set submit input (merged), batch num: %d", pTask->taskId, pTask, (int32_t)blocks->size); - qSetMultiStreamInput(exec, blocks->pData, blocks->size, STREAM_INPUT__MERGED_SUBMIT); + + SArray* pBlockList = pMerged->submits; + int32_t numOfBlocks = taosArrayGetSize(pBlockList); + qDebug("st-task:%s %p set submit input (merged), batch num:%d", pTask->id.idStr, pTask, numOfBlocks); + qSetMultiStreamInput(pExecutor, pBlockList->pData, numOfBlocks, STREAM_INPUT__MERGED_SUBMIT); } else if (pItem->type == STREAM_INPUT__REF_DATA_BLOCK) { const SStreamRefDataBlock* pRefBlock = (const SStreamRefDataBlock*)data; - qSetMultiStreamInput(exec, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); + qSetMultiStreamInput(pExecutor, pRefBlock->pBlock, 1, STREAM_INPUT__DATA_BLOCK); } else { ASSERT(0); } - // exec + // pExecutor while (1) { - if (pTask->taskStatus == TASK_STATUS__DROPPING) { + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { return 0; } SSDataBlock* output = NULL; uint64_t ts = 0; - if ((code = qExecTask(exec, &output, &ts)) < 0) { + if ((code = qExecTask(pExecutor, &output, &ts)) < 0) { if (code == TSDB_CODE_QRY_IN_EXEC) { - resetTaskInfo(exec); + resetTaskInfo(pExecutor); } - /*ASSERT(false);*/ - qError("unexpected stream execution, stream %" PRId64 " task: %d, since %s", pTask->streamId, pTask->taskId, - terrstr()); + + qError("unexpected stream execution, s-task:%s since %s", pTask->id.idStr, terrstr()); continue; } + if (output == NULL) { if (pItem->type == STREAM_INPUT__DATA_RETRIEVE) { - SSDataBlock block = {0}; + SSDataBlock block = {0}; + const SStreamDataBlock* pRetrieveBlock = (const SStreamDataBlock*)data; ASSERT(taosArrayGetSize(pRetrieveBlock->blocks) == 1); + assignOneDataBlock(&block, taosArrayGet(pRetrieveBlock->blocks, 0)); block.info.type = STREAM_PULL_OVER; block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); - qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->taskId, pTask->selfChildId, + qDebug("task %d(child %d) processed retrieve, reqId %" PRId64, pTask->id.taskId, pTask->selfChildId, pRetrieveBlock->reqId); } break; @@ -94,20 +106,21 @@ static int32_t streamTaskExecImpl(SStreamTask* pTask, const void* data, SArray* continue; } - qDebug("task %d(child %d) executed and get block", pTask->taskId, pTask->selfChildId); + qDebug("task %d(child %d) executed and get block", pTask->id.taskId, pTask->selfChildId); SSDataBlock block = {0}; assignOneDataBlock(&block, output); block.info.childId = pTask->selfChildId; taosArrayPush(pRes, &block); } + return 0; } int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { ASSERT(pTask->taskLevel == TASK_LEVEL__SOURCE); - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; qSetStreamOpOpen(exec); bool finished = false; @@ -121,7 +134,7 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { int32_t batchCnt = 0; while (1) { - if (atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING) { + if (atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING) { taosArrayDestroy(pRes); return 0; } @@ -147,17 +160,17 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { batchCnt++; - qDebug("task %d scan exec block num %d, block limit %d", pTask->taskId, batchCnt, batchSz); + qDebug("task %d scan exec block num %d, block limit %d", pTask->id.taskId, batchCnt, batchSz); if (batchCnt >= batchSz) break; } if (taosArrayGetSize(pRes) == 0) { if (finished) { taosArrayDestroy(pRes); - qDebug("task %d finish recover exec task ", pTask->taskId); + qDebug("task %d finish recover exec task ", pTask->id.taskId); break; } else { - qDebug("task %d continue recover exec task ", pTask->taskId); + qDebug("task %d continue recover exec task ", pTask->id.taskId); continue; } } @@ -173,10 +186,13 @@ int32_t streamScanExec(SStreamTask* pTask, int32_t batchSz) { streamTaskOutput(pTask, qRes); if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH || pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { - qDebug("task %d scan exec dispatch block num %d", pTask->taskId, batchCnt); + qDebug("task %d scan exec dispatch block num %d", pTask->id.taskId, batchCnt); streamDispatch(pTask); } - if (finished) break; + + if (finished) { + break; + } } return 0; } @@ -186,7 +202,7 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { // fetch all queue item, merge according to batchLimit int32_t numOfItems = taosReadAllQitems(pTask->inputQueue1, pTask->inputQall); if (numOfItems == 0) { - qDebug("task: %d, stream task exec over, queue empty", pTask->taskId); + qDebug("task: %d, stream task exec over, queue empty", pTask->id.taskId); return 0; } SStreamQueueItem* pMerged = NULL; @@ -221,105 +237,141 @@ int32_t streamBatchExec(SStreamTask* pTask, int32_t batchLimit) { int32_t streamExecForAll(SStreamTask* pTask) { while (1) { - int32_t batchCnt = 1; - void* input = NULL; + int32_t batchSize = 1; + void* pInput = NULL; + + // merge multiple input data if possible in the input queue. while (1) { SStreamQueueItem* qItem = streamQueueNextItem(pTask->inputQueue); if (qItem == NULL) { - qDebug("stream task exec over, queue empty, task: %d", pTask->taskId); +// qDebug("s-task:%s extract data from input queue, queue is empty, abort", pTask->id.idStr); break; } - if (input == NULL) { - input = qItem; + + if (pInput == NULL) { + pInput = qItem; streamQueueProcessSuccess(pTask->inputQueue); if (pTask->taskLevel == TASK_LEVEL__SINK) { break; } } else { - void* newRet; - if ((newRet = streamMergeQueueItem(input, qItem)) == NULL) { + void* newRet = NULL; + if ((newRet = streamMergeQueueItem(pInput, qItem)) == NULL) { streamQueueProcessFail(pTask->inputQueue); break; } else { - batchCnt++; - input = newRet; + batchSize++; + pInput = newRet; streamQueueProcessSuccess(pTask->inputQueue); - if (batchCnt > STREAM_EXEC_MAX_BATCH_NUM) { + if (batchSize > STREAM_EXEC_MAX_BATCH_NUM) { break; } } } } - if (pTask->taskStatus == TASK_STATUS__DROPPING) { - if (input) streamFreeQitem(input); + if (pTask->status.taskStatus == TASK_STATUS__DROPPING) { + if (pInput) { + streamFreeQitem(pInput); + } return 0; } - if (input == NULL) { + if (pInput == NULL) { break; } if (pTask->taskLevel == TASK_LEVEL__SINK) { - ASSERT(((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_BLOCK); - streamTaskOutput(pTask, input); + ASSERT(((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_BLOCK); + qDebug("s-task:%s sink node start to sink result. numOfBlocks:%d", pTask->id.idStr, batchSize); + streamTaskOutput(pTask, pInput); continue; } SArray* pRes = taosArrayInit(0, sizeof(SSDataBlock)); + qDebug("s-task:%s exec begin, numOfBlocks:%d", pTask->id.idStr, batchSize); + + streamTaskExecImpl(pTask, pInput, pRes); + + int64_t ckId = 0; + int64_t dataVer = 0; + qGetCheckpointVersion(pTask->exec.pExecutor, &dataVer, &ckId); + if (dataVer > pTask->chkInfo.version) { // save it since the checkpoint is updated + qDebug("s-task:%s exec end, start to update check point, ver from %" PRId64 " to %" PRId64 + ", checkPoint id:%" PRId64 " -> %" PRId64, + pTask->id.idStr, pTask->chkInfo.version, dataVer, pTask->chkInfo.id, ckId); - qDebug("stream task %d exec begin, msg batch: %d", pTask->taskId, batchCnt); - streamTaskExecImpl(pTask, input, pRes); - qDebug("stream task %d exec end", pTask->taskId); + pTask->chkInfo = (SCheckpointInfo) {.version = dataVer, .id = ckId}; + + taosWLockLatch(&pTask->pMeta->lock); + streamMetaSaveTask(pTask->pMeta, pTask); + if (streamMetaCommit(pTask->pMeta) < 0) { + taosWUnLockLatch(&pTask->pMeta->lock); + qError("s-task:%s failed to commit stream meta, since %s", pTask->id.idStr, terrstr()); + return -1; + } else { + taosWUnLockLatch(&pTask->pMeta->lock); + qDebug("s-task:%s update checkpoint ver succeed", pTask->id.idStr); + } + } else { + qDebug("s-task:%s exec end", pTask->id.idStr); + } if (taosArrayGetSize(pRes) != 0) { SStreamDataBlock* qRes = taosAllocateQitem(sizeof(SStreamDataBlock), DEF_QITEM, 0); if (qRes == NULL) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(input); + streamFreeQitem(pInput); return -1; } + qRes->type = STREAM_INPUT__DATA_BLOCK; qRes->blocks = pRes; - if (((SStreamQueueItem*)input)->type == STREAM_INPUT__DATA_SUBMIT) { - SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)input; + if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__DATA_SUBMIT) { + SStreamDataSubmit2* pSubmit = (SStreamDataSubmit2*)pInput; qRes->childId = pTask->selfChildId; qRes->sourceVer = pSubmit->ver; - } else if (((SStreamQueueItem*)input)->type == STREAM_INPUT__MERGED_SUBMIT) { - SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)input; + } else if (((SStreamQueueItem*)pInput)->type == STREAM_INPUT__MERGED_SUBMIT) { + SStreamMergedSubmit2* pMerged = (SStreamMergedSubmit2*)pInput; qRes->childId = pTask->selfChildId; qRes->sourceVer = pMerged->ver; } if (streamTaskOutput(pTask, qRes) < 0) { taosArrayDestroyEx(pRes, (FDelete)blockDataFreeRes); - streamFreeQitem(input); + streamFreeQitem(pInput); taosFreeQitem(qRes); return -1; } } else { taosArrayDestroy(pRes); } - streamFreeQitem(input); + streamFreeQitem(pInput); } return 0; } int32_t streamTryExec(SStreamTask* pTask) { + // this function may be executed by multi-threads, so status check is required. int8_t schedStatus = - atomic_val_compare_exchange_8(&pTask->schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + atomic_val_compare_exchange_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__WAITING, TASK_SCHED_STATUS__ACTIVE); + if (schedStatus == TASK_SCHED_STATUS__WAITING) { int32_t code = streamExecForAll(pTask); if (code < 0) { - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__FAILED); + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__FAILED); return -1; } - atomic_store_8(&pTask->schedStatus, TASK_SCHED_STATUS__INACTIVE); + + // todo the task should be commit here + atomic_store_8(&pTask->status.schedStatus, TASK_SCHED_STATUS__INACTIVE); + qDebug("s-task:%s exec completed", pTask->id.idStr); if (!taosQueueEmpty(pTask->inputQueue->queue)) { streamSchedExec(pTask); } } + return 0; } diff --git a/source/libs/stream/src/streamMeta.c b/source/libs/stream/src/streamMeta.c index 03391c0ba2b5dd52e4442c4f172c27e7c474c37a..860514bb44fc97c4c0191fc16c2ff070230d4085 100644 --- a/source/libs/stream/src/streamMeta.c +++ b/source/libs/stream/src/streamMeta.c @@ -24,6 +24,7 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF terrno = TSDB_CODE_OUT_OF_MEMORY; return NULL; } + int32_t len = strlen(path) + 20; char* streamPath = taosMemoryCalloc(1, len); sprintf(streamPath, "%s/%s", path, "stream"); @@ -50,7 +51,8 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } - pMeta->pTasks = taosHashInit(64, taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT), true, HASH_ENTRY_LOCK); + _hash_fn_t fp = taosGetDefaultHashFunction(TSDB_DATA_TYPE_INT); + pMeta->pTasks = taosHashInit(64, fp, true, HASH_ENTRY_LOCK); if (pMeta->pTasks == NULL) { goto _err; } @@ -59,9 +61,10 @@ SStreamMeta* streamMetaOpen(const char* path, void* ahandle, FTaskExpand expandF goto _err; } + pMeta->vgId = vgId; pMeta->ahandle = ahandle; pMeta->expandFunc = expandFunc; - + taosInitRWLatch(&pMeta->lock); return pMeta; _err: @@ -81,19 +84,28 @@ void streamMetaClose(SStreamMeta* pMeta) { tdbClose(pMeta->db); void* pIter = NULL; +// while(pMeta->walScan) { +// qDebug("wait stream daemon quit"); +// taosMsleep(100); +// } + while (1) { pIter = taosHashIterate(pMeta->pTasks, pIter); - if (pIter == NULL) break; + if (pIter == NULL) { + break; + } + SStreamTask* pTask = *(SStreamTask**)pIter; if (pTask->timer) { taosTmrStop(pTask->timer); pTask->timer = NULL; } - tFreeSStreamTask(pTask); + + tFreeStreamTask(pTask); /*streamMetaReleaseTask(pMeta, pTask);*/ } + taosHashCleanup(pMeta->pTasks); - taosHashCleanup(pMeta->pRecoverStatus); taosMemoryFree(pMeta->path); taosMemoryFree(pMeta); } @@ -106,7 +118,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, } SDecoder decoder; tDecoderInit(&decoder, (uint8_t*)msg, msgLen); - if (tDecodeSStreamTask(&decoder, pTask) < 0) { + if (tDecodeStreamTask(&decoder, pTask) < 0) { tDecoderClear(&decoder); goto FAIL; } @@ -117,12 +129,12 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, goto FAIL; } - if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { goto FAIL; } - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { - taosHashRemove(pMeta->pTasks, &pTask->taskId, sizeof(int32_t)); + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), msg, msgLen, pMeta->txn) < 0) { + taosHashRemove(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t)); ASSERT(0); goto FAIL; } @@ -130,7 +142,7 @@ int32_t streamMetaAddSerializedTask(SStreamMeta* pMeta, int64_t ver, char* msg, return 0; FAIL: - if (pTask) tFreeSStreamTask(pTask); + if (pTask) tFreeStreamTask(pTask); return -1; } #endif @@ -139,7 +151,7 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { void* buf = NULL; int32_t len; int32_t code; - tEncodeSize(tEncodeSStreamTask, pTask, len, code); + tEncodeSize(tEncodeStreamTask, pTask, len, code); if (code < 0) { return -1; } @@ -150,10 +162,10 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { SEncoder encoder = {0}; tEncoderInit(&encoder, buf, len); - tEncodeSStreamTask(&encoder, pTask); + tEncodeStreamTask(&encoder, pTask); tEncoderClear(&encoder); - if (tdbTbUpsert(pMeta->pTaskDb, &pTask->taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { + if (tdbTbUpsert(pMeta->pTaskDb, &pTask->id.taskId, sizeof(int32_t), buf, len, pMeta->txn) < 0) { return -1; } @@ -161,8 +173,8 @@ int32_t streamMetaSaveTask(SStreamMeta* pMeta, SStreamTask* pTask) { return 0; } -#if 1 -int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { +// add to the ready tasks hash map, not the restored tasks hash map +int32_t streamMetaAddDeployedTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { if (pMeta->expandFunc(pMeta->ahandle, pTask, ver) < 0) { return -1; } @@ -171,39 +183,24 @@ int32_t streamMetaAddTask(SStreamMeta* pMeta, int64_t ver, SStreamTask* pTask) { return -1; } - taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)); - + taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, POINTER_BYTES); return 0; } -#endif -#if 0 -SStreamTask* streamMetaGetTask(SStreamMeta* pMeta, int32_t taskId) { - SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - ASSERT((*ppTask)->taskId == taskId); - return *ppTask; - } else { - return NULL; - } +int32_t streamMetaGetNumOfTasks(const SStreamMeta* pMeta) { + return (int32_t) taosHashGetSize(pMeta->pTasks); } -#endif SStreamTask* streamMetaAcquireTask(SStreamMeta* pMeta, int32_t taskId) { taosRLockLatch(&pMeta->lock); SStreamTask** ppTask = (SStreamTask**)taosHashGet(pMeta->pTasks, &taskId, sizeof(int32_t)); - if (ppTask) { - SStreamTask* pTask = *ppTask; - if (atomic_load_8(&pTask->taskStatus) != TASK_STATUS__DROPPING) { - atomic_add_fetch_32(&pTask->refCnt, 1); - taosRUnLockLatch(&pMeta->lock); - return pTask; - } else { - taosRUnLockLatch(&pMeta->lock); - return NULL; - } + if (ppTask != NULL && (atomic_load_8(&((*ppTask)->status.taskStatus)) != TASK_STATUS__DROPPING)) { + atomic_add_fetch_32(&(*ppTask)->refCnt, 1); + taosRUnLockLatch(&pMeta->lock); + return *ppTask; } + taosRUnLockLatch(&pMeta->lock); return NULL; } @@ -212,8 +209,8 @@ void streamMetaReleaseTask(SStreamMeta* pMeta, SStreamTask* pTask) { int32_t left = atomic_sub_fetch_32(&pTask->refCnt, 1); ASSERT(left >= 0); if (left == 0) { - ASSERT(atomic_load_8(&pTask->taskStatus) == TASK_STATUS__DROPPING); - tFreeSStreamTask(pTask); + ASSERT(atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__DROPPING); + tFreeStreamTask(pTask); } } @@ -227,7 +224,7 @@ void streamMetaRemoveTask(SStreamMeta* pMeta, int32_t taskId) { * taosTmrStop(pTask->timer);*/ /*pTask->timer = NULL;*/ /*}*/ - atomic_store_8(&pTask->taskStatus, TASK_STATUS__DROPPING); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__DROPPING); taosWLockLatch(&pMeta->lock); streamMetaReleaseTask(pMeta, pTask); @@ -245,9 +242,12 @@ int32_t streamMetaBegin(SStreamMeta* pMeta) { int32_t streamMetaCommit(SStreamMeta* pMeta) { if (tdbCommit(pMeta->db, pMeta->txn) < 0) { + ASSERT(0); return -1; } + if (tdbPostCommit(pMeta->db, pMeta->txn) < 0) { + ASSERT(0); return -1; } @@ -293,25 +293,26 @@ int32_t streamLoadTasks(SStreamMeta* pMeta, int64_t ver) { return -1; } tDecoderInit(&decoder, (uint8_t*)pVal, vLen); - tDecodeSStreamTask(&decoder, pTask); + tDecodeStreamTask(&decoder, pTask); tDecoderClear(&decoder); - if (pMeta->expandFunc(pMeta->ahandle, pTask, -1) < 0) { + if (pMeta->expandFunc(pMeta->ahandle, pTask, pTask->chkInfo.version) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return -1; } - if (taosHashPut(pMeta->pTasks, &pTask->taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { + if (taosHashPut(pMeta->pTasks, &pTask->id.taskId, sizeof(int32_t), &pTask, sizeof(void*)) < 0) { tdbFree(pKey); tdbFree(pVal); tdbTbcClose(pCur); return -1; } - /*pTask->taskStatus = TASK_STATUS__NORMAL;*/ + + /*pTask->status.taskStatus = TASK_STATUS__NORMAL;*/ if (pTask->fillHistory) { - pTask->taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; + pTask->status.taskStatus = TASK_STATUS__WAIT_DOWNSTREAM; streamTaskCheckDownstream(pTask, ver); } } diff --git a/source/libs/stream/src/streamRecover.c b/source/libs/stream/src/streamRecover.c index 87058bf490d43f2cbbd1b18eb8411c034b40a9cd..55c745e4173ffadc6869d52627d43a8abafa84e9 100644 --- a/source/libs/stream/src/streamRecover.c +++ b/source/libs/stream/src/streamRecover.c @@ -16,9 +16,10 @@ #include "streamInc.h" int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { - qDebug("task %d at node %d launch recover", pTask->taskId, pTask->nodeId); + qDebug("s-task:%s at node %d launch recover", pTask->id.idStr, pTask->nodeId); + if (pTask->taskLevel == TASK_LEVEL__SOURCE) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__RECOVER_PREPARE); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__RECOVER_PREPARE); streamSetParamForRecover(pTask); streamSourceRecoverPrepareStep1(pTask, version); @@ -33,34 +34,31 @@ int32_t streamTaskLaunchRecover(SStreamTask* pTask, int64_t version) { memcpy(serializedReq, &req, len); - SRpcMsg rpcMsg = { - .contLen = len, - .pCont = serializedReq, - .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE, - }; - + SRpcMsg rpcMsg = { .contLen = len, .pCont = serializedReq, .msgType = TDMT_VND_STREAM_RECOVER_NONBLOCKING_STAGE }; if (tmsgPutToQueue(pTask->pMsgCb, STREAM_QUEUE, &rpcMsg) < 0) { /*ASSERT(0);*/ } } else if (pTask->taskLevel == TASK_LEVEL__AGG) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); streamSetParamForRecover(pTask); streamAggRecoverPrepare(pTask); } else if (pTask->taskLevel == TASK_LEVEL__SINK) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); } + return 0; } // checkstatus int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { SStreamTaskCheckReq req = { - .streamId = pTask->streamId, - .upstreamTaskId = pTask->taskId, + .streamId = pTask->id.streamId, + .upstreamTaskId = pTask->id.taskId, .upstreamNodeId = pTask->nodeId, .childId = pTask->selfChildId, }; + // serialize if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { req.reqId = tGenIdPI64(); @@ -68,7 +66,7 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { req.downstreamTaskId = pTask->fixedEpDispatcher.taskId; pTask->checkReqId = req.reqId; - qDebug("task %d at node %d check downstream task %d at node %d", pTask->taskId, pTask->nodeId, req.downstreamTaskId, + qDebug("task %d at node %d check downstream task %d at node %d", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pTask->fixedEpDispatcher.nodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -83,12 +81,12 @@ int32_t streamTaskCheckDownstream(SStreamTask* pTask, int64_t version) { taosArrayPush(pTask->checkReqIds, &req.reqId); req.downstreamNodeId = pVgInfo->vgId; req.downstreamTaskId = pVgInfo->taskId; - qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->taskId, pTask->nodeId, + qDebug("task %d at node %d check downstream task %d at node %d (shuffle)", pTask->id.taskId, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); streamDispatchOneCheckReq(pTask, &req, pVgInfo->vgId, &pVgInfo->epSet); } } else { - qDebug("task %d at node %d direct launch recover since no downstream", pTask->taskId, pTask->nodeId); + qDebug("task %d at node %d direct launch recover since no downstream", pTask->id.taskId, pTask->nodeId); streamTaskLaunchRecover(pTask, version); } return 0; @@ -104,8 +102,10 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp .downstreamNodeId = pRsp->downstreamNodeId, .childId = pRsp->childId, }; - qDebug("task %d at node %d check downstream task %d at node %d (recheck)", pTask->taskId, pTask->nodeId, + + qDebug("s-task:%s at node %d check downstream task %d at node %d (recheck)", pTask->id.idStr, pTask->nodeId, req.downstreamTaskId, req.downstreamNodeId); + if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { streamDispatchOneCheckReq(pTask, &req, pRsp->downstreamNodeId, &pTask->fixedEpDispatcher.epSet); } else if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { @@ -118,16 +118,18 @@ int32_t streamRecheckOneDownstream(SStreamTask* pTask, const SStreamTaskCheckRsp } } } + return 0; } int32_t streamProcessTaskCheckReq(SStreamTask* pTask, const SStreamTaskCheckReq* pReq) { - return atomic_load_8(&pTask->taskStatus) == TASK_STATUS__NORMAL; + return atomic_load_8(&pTask->status.taskStatus) == TASK_STATUS__NORMAL; } int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* pRsp, int64_t version) { qDebug("task %d at node %d recv check rsp from task %d at node %d: status %d", pRsp->upstreamTaskId, pRsp->upstreamNodeId, pRsp->downstreamTaskId, pRsp->downstreamNodeId, pRsp->status); + if (pRsp->status == 1) { if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { bool found = false; @@ -138,7 +140,11 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* break; } } - if (!found) return -1; + + if (!found) { + return -1; + } + int32_t left = atomic_sub_fetch_32(&pTask->recoverTryingDownstream, 1); ASSERT(left >= 0); if (left == 0) { @@ -147,41 +153,46 @@ int32_t streamProcessTaskCheckRsp(SStreamTask* pTask, const SStreamTaskCheckRsp* streamTaskLaunchRecover(pTask, version); } } else if (pTask->outputType == TASK_OUTPUT__FIXED_DISPATCH) { - if (pRsp->reqId != pTask->checkReqId) return -1; + if (pRsp->reqId != pTask->checkReqId) { + return -1; + } + streamTaskLaunchRecover(pTask, version); } else { ASSERT(0); } - } else { + } else { // not ready, it should wait for at least 100ms and then retry streamRecheckOneDownstream(pTask, pRsp); } + return 0; } // common int32_t streamSetParamForRecover(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamSetParamForRecover(exec); } int32_t streamRestoreParam(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamRestoreParam(exec); } + int32_t streamSetStatusNormal(SStreamTask* pTask) { - atomic_store_8(&pTask->taskStatus, TASK_STATUS__NORMAL); + atomic_store_8(&pTask->status.taskStatus, TASK_STATUS__NORMAL); return 0; } // source int32_t streamSourceRecoverPrepareStep1(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; return qStreamSourceRecoverStep1(exec, ver); } int32_t streamBuildSourceRecover1Req(SStreamTask* pTask, SStreamRecoverStep1Req* pReq) { pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; + pReq->streamId = pTask->id.streamId; + pReq->taskId = pTask->id.taskId; return 0; } @@ -192,13 +203,13 @@ int32_t streamSourceRecoverScanStep1(SStreamTask* pTask) { int32_t streamBuildSourceRecover2Req(SStreamTask* pTask, SStreamRecoverStep2Req* pReq) { pReq->msgHead.vgId = pTask->nodeId; - pReq->streamId = pTask->streamId; - pReq->taskId = pTask->taskId; + pReq->streamId = pTask->id.streamId; + pReq->taskId = pTask->id.taskId; return 0; } int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; if (qStreamSourceRecoverStep2(exec, ver) < 0) { } return streamScanExec(pTask, 100); @@ -206,7 +217,7 @@ int32_t streamSourceRecoverScanStep2(SStreamTask* pTask, int64_t ver) { int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { SStreamRecoverFinishReq req = { - .streamId = pTask->streamId, + .streamId = pTask->id.streamId, .childId = pTask->selfChildId, }; // serialize @@ -227,13 +238,13 @@ int32_t streamDispatchRecoverFinishReq(SStreamTask* pTask) { // agg int32_t streamAggRecoverPrepare(SStreamTask* pTask) { - void* exec = pTask->exec.executor; pTask->recoverWaitingUpstream = taosArrayGetSize(pTask->childEpInfo); + qDebug("s-task:%s wait for %d upstreams", pTask->id.idStr, pTask->recoverWaitingUpstream); return 0; } int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { - void* exec = pTask->exec.executor; + void* exec = pTask->exec.pExecutor; if (qStreamRestoreParam(exec) < 0) { return -1; } @@ -247,6 +258,7 @@ int32_t streamAggChildrenRecoverFinish(SStreamTask* pTask) { int32_t streamProcessRecoverFinishReq(SStreamTask* pTask, int32_t childId) { if (pTask->taskLevel == TASK_LEVEL__AGG) { int32_t left = atomic_sub_fetch_32(&pTask->recoverWaitingUpstream, 1); + qDebug("s-task:%s remain unfinished child tasks:%d", pTask->id.idStr, left); ASSERT(left >= 0); if (left == 0) { streamAggChildrenRecoverFinish(pTask); diff --git a/source/libs/stream/src/streamState.c b/source/libs/stream/src/streamState.c index 411726075e1b4488292868bb0bdf3e7ec942aa1e..7bea989e3a63be5110deb9e8dbb5b00ac6eed643 100644 --- a/source/libs/stream/src/streamState.c +++ b/source/libs/stream/src/streamState.c @@ -121,7 +121,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int char statePath[1024]; if (!specPath) { - sprintf(statePath, "%s/%d", path, pTask->taskId); + sprintf(statePath, "%s/%d", path, pTask->id.taskId); } else { memset(statePath, 0, 1024); tstrncpy(statePath, path, 1024); @@ -193,6 +193,7 @@ SStreamState* streamStateOpen(char* path, SStreamTask* pTask, bool specPath, int } pState->pTdbState->pOwner = pTask; + pState->checkPointId = 0; return pState; @@ -243,6 +244,7 @@ int32_t streamStateCommit(SStreamState* pState) { TDB_TXN_WRITE | TDB_TXN_READ_UNCOMMITTED) < 0) { return -1; } + pState->checkPointId++; return 0; } diff --git a/source/libs/stream/src/streamTask.c b/source/libs/stream/src/streamTask.c index e9aba0bc3941d501ba25a5ea28aaf02e11d42cbd..67c60008fdb98b5f341be5271668a17bd3563c92 100644 --- a/source/libs/stream/src/streamTask.c +++ b/source/libs/stream/src/streamTask.c @@ -15,15 +15,22 @@ #include "executor.h" #include "tstream.h" +#include "wal.h" -SStreamTask* tNewSStreamTask(int64_t streamId) { +SStreamTask* tNewStreamTask(int64_t streamId) { SStreamTask* pTask = (SStreamTask*)taosMemoryCalloc(1, sizeof(SStreamTask)); if (pTask == NULL) { return NULL; } - pTask->taskId = tGenIdPI32(); - pTask->streamId = streamId; - pTask->schedStatus = TASK_SCHED_STATUS__INACTIVE; + + pTask->id.taskId = tGenIdPI32(); + pTask->id.streamId = streamId; + + char buf[128] = {0}; + sprintf(buf, "0x%"PRIx64"-%d", pTask->id.streamId, pTask->id.taskId); + + pTask->id.idStr = taosStrdup(buf); + pTask->status.schedStatus = TASK_SCHED_STATUS__INACTIVE; pTask->inputStatus = TASK_INPUT_STATUS__NORMAL; pTask->outputStatus = TASK_OUTPUT_STATUS__NORMAL; @@ -48,24 +55,24 @@ int32_t tDecodeStreamEpInfo(SDecoder* pDecoder, SStreamChildEpInfo* pInfo) { return 0; } -int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { +int32_t tEncodeStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { if (tStartEncode(pEncoder) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->streamId) < 0) return -1; - if (tEncodeI32(pEncoder, pTask->taskId) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->id.streamId) < 0) return -1; + if (tEncodeI32(pEncoder, pTask->id.taskId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->totalLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->taskLevel) < 0) return -1; if (tEncodeI8(pEncoder, pTask->outputType) < 0) return -1; if (tEncodeI16(pEncoder, pTask->dispatchMsgType) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->taskStatus) < 0) return -1; - if (tEncodeI8(pEncoder, pTask->schedStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status.taskStatus) < 0) return -1; + if (tEncodeI8(pEncoder, pTask->status.schedStatus) < 0) return -1; if (tEncodeI32(pEncoder, pTask->selfChildId) < 0) return -1; if (tEncodeI32(pEncoder, pTask->nodeId) < 0) return -1; if (tEncodeSEpSet(pEncoder, &pTask->epSet) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->recoverSnapVer) < 0) return -1; - if (tEncodeI64(pEncoder, pTask->startVer) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->chkInfo.id) < 0) return -1; + if (tEncodeI64(pEncoder, pTask->chkInfo.version) < 0) return -1; if (tEncodeI8(pEncoder, pTask->fillHistory) < 0) return -1; int32_t epSz = taosArrayGetSize(pTask->childEpInfo); @@ -101,24 +108,24 @@ int32_t tEncodeSStreamTask(SEncoder* pEncoder, const SStreamTask* pTask) { return pEncoder->pos; } -int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { +int32_t tDecodeStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { if (tStartDecode(pDecoder) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->streamId) < 0) return -1; - if (tDecodeI32(pDecoder, &pTask->taskId) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->id.streamId) < 0) return -1; + if (tDecodeI32(pDecoder, &pTask->id.taskId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->totalLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->taskLevel) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->outputType) < 0) return -1; if (tDecodeI16(pDecoder, &pTask->dispatchMsgType) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->taskStatus) < 0) return -1; - if (tDecodeI8(pDecoder, &pTask->schedStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status.taskStatus) < 0) return -1; + if (tDecodeI8(pDecoder, &pTask->status.schedStatus) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->selfChildId) < 0) return -1; if (tDecodeI32(pDecoder, &pTask->nodeId) < 0) return -1; if (tDecodeSEpSet(pDecoder, &pTask->epSet) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->recoverSnapVer) < 0) return -1; - if (tDecodeI64(pDecoder, &pTask->startVer) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->chkInfo.id) < 0) return -1; + if (tDecodeI64(pDecoder, &pTask->chkInfo.version) < 0) return -1; if (tDecodeI8(pDecoder, &pTask->fillHistory) < 0) return -1; int32_t epSz; @@ -162,24 +169,47 @@ int32_t tDecodeSStreamTask(SDecoder* pDecoder, SStreamTask* pTask) { return 0; } -void tFreeSStreamTask(SStreamTask* pTask) { - qDebug("free stream task %d", pTask->taskId); - if (pTask->inputQueue) streamQueueClose(pTask->inputQueue); - if (pTask->outputQueue) streamQueueClose(pTask->outputQueue); - if (pTask->exec.qmsg) taosMemoryFree(pTask->exec.qmsg); - if (pTask->exec.executor) qDestroyTask(pTask->exec.executor); +void tFreeStreamTask(SStreamTask* pTask) { + qDebug("free s-task:%s", pTask->id.idStr); + + if (pTask->inputQueue) { + streamQueueClose(pTask->inputQueue); + } + if (pTask->outputQueue) { + streamQueueClose(pTask->outputQueue); + } + if (pTask->exec.qmsg) { + taosMemoryFree(pTask->exec.qmsg); + } + + if (pTask->exec.pExecutor) { + qDestroyTask(pTask->exec.pExecutor); + pTask->exec.pExecutor = NULL; + } + + if (pTask->exec.pWalReader != NULL) { + walCloseReader(pTask->exec.pWalReader); + } + taosArrayDestroyP(pTask->childEpInfo, taosMemoryFree); if (pTask->outputType == TASK_OUTPUT__TABLE) { tDeleteSSchemaWrapper(pTask->tbSink.pSchemaWrapper); taosMemoryFree(pTask->tbSink.pTSchema); } + if (pTask->outputType == TASK_OUTPUT__SHUFFLE_DISPATCH) { taosArrayDestroy(pTask->shuffleDispatcher.dbInfo.pVgroupInfos); taosArrayDestroy(pTask->checkReqIds); pTask->checkReqIds = NULL; } - if (pTask->pState) streamStateClose(pTask->pState); + if (pTask->pState) { + streamStateClose(pTask->pState); + } + + if (pTask->id.idStr != NULL) { + taosMemoryFree((void*)pTask->id.idStr); + } taosMemoryFree(pTask); } diff --git a/source/libs/sync/inc/syncIndexMgr.h b/source/libs/sync/inc/syncIndexMgr.h index 4e6ab284f8cf9b10a06f5880abd8f14e7c13de81..a4e83dc4956160648362b55900b5131ae80998cf 100644 --- a/source/libs/sync/inc/syncIndexMgr.h +++ b/source/libs/sync/inc/syncIndexMgr.h @@ -24,12 +24,13 @@ extern "C" { // SIndexMgr ----------------------------- typedef struct SSyncIndexMgr { - SRaftId (*replicas)[TSDB_MAX_REPLICA]; - SyncIndex index[TSDB_MAX_REPLICA]; - SyncTerm privateTerm[TSDB_MAX_REPLICA]; // for advanced function - int64_t startTimeArr[TSDB_MAX_REPLICA]; - int64_t recvTimeArr[TSDB_MAX_REPLICA]; + SRaftId (*replicas)[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncIndex index[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SyncTerm privateTerm[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // for advanced function + int64_t startTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + int64_t recvTimeArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; int32_t replicaNum; + int32_t totalReplicaNum; SSyncNode *pNode; } SSyncIndexMgr; diff --git a/source/libs/sync/inc/syncInt.h b/source/libs/sync/inc/syncInt.h index 6f2c1a1ad0e76d233d516e36ee419cae43fcc409..7d336c83135a20449f09431c2d9427a6dd6cc0c5 100644 --- a/source/libs/sync/inc/syncInt.h +++ b/source/libs/sync/inc/syncInt.h @@ -54,13 +54,13 @@ typedef struct SSyncLogReplMgr SSyncLogReplMgr; #define MAX_CONFIG_INDEX_COUNT 256 typedef struct SRaftCfg { - SSyncCfg cfg; - int32_t batchSize; - int8_t isStandBy; - int8_t snapshotStrategy; - SyncIndex lastConfigIndex; - int32_t configIndexCount; - SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; + SSyncCfg cfg; + int32_t batchSize; + int8_t isStandBy; + int8_t snapshotStrategy; + SyncIndex lastConfigIndex; + int32_t configIndexCount; + SyncIndex configIndexArr[MAX_CONFIG_INDEX_COUNT]; } SRaftCfg; typedef struct SRaftId { @@ -127,12 +127,13 @@ typedef struct SSyncNode { SRaftId myRaftId; int32_t peersNum; - SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA]; - SEpSet peersEpset[TSDB_MAX_REPLICA]; - SRaftId peersId[TSDB_MAX_REPLICA]; + SNodeInfo peersNodeInfo[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SEpSet peersEpset[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + SRaftId peersId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; int32_t replicaNum; - SRaftId replicasId[TSDB_MAX_REPLICA]; + int32_t totalReplicaNum; + SRaftId replicasId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // raft algorithm SSyncFSM* pFsm; @@ -188,7 +189,7 @@ typedef struct SSyncNode { uint64_t heartbeatTimerCounter; // peer heartbeat timer - SSyncTimer peerHeartbeatTimerArr[TSDB_MAX_REPLICA]; + SSyncTimer peerHeartbeatTimerArr[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // tools SSyncRespMgr* pSyncRespMgr; @@ -196,13 +197,13 @@ typedef struct SSyncNode { // restore state bool restoreFinish; // SSnapshot* pSnapshot; - SSyncSnapshotSender* senders[TSDB_MAX_REPLICA]; + SSyncSnapshotSender* senders[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; SSyncSnapshotReceiver* pNewNodeReceiver; // log replication mgr - SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA]; + SSyncLogReplMgr* logReplMgrs[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; - SPeerState peerStates[TSDB_MAX_REPLICA]; + SPeerState peerStates[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; // is config changing bool changing; @@ -275,6 +276,7 @@ void syncNodeUpdateTerm(SSyncNode* pSyncNode, SyncTerm term); void syncNodeUpdateTermWithoutStepDown(SSyncNode* pSyncNode, SyncTerm term); void syncNodeStepDown(SSyncNode* pSyncNode, SyncTerm newTerm); void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr); +void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr); void syncNodeBecomeLeader(SSyncNode* pSyncNode, const char* debugStr); void syncNodeCandidate2Leader(SSyncNode* pSyncNode); void syncNodeFollower2Candidate(SSyncNode* pSyncNode); diff --git a/source/libs/sync/inc/syncMessage.h b/source/libs/sync/inc/syncMessage.h index c3566c7c820b97c4484f15ce59239d423db9d480..f8c96d8be296f136292996daed85f78d2b4aa698 100644 --- a/source/libs/sync/inc/syncMessage.h +++ b/source/libs/sync/inc/syncMessage.h @@ -237,6 +237,7 @@ typedef struct SyncLeaderTransfer { typedef enum { SYNC_LOCAL_CMD_STEP_DOWN = 100, SYNC_LOCAL_CMD_FOLLOWER_CMT, + SYNC_LOCAL_CMD_LEARNER_CMT, } ESyncLocalCmd; typedef struct SyncLocalCmd { diff --git a/source/libs/sync/inc/syncPipeline.h b/source/libs/sync/inc/syncPipeline.h index a1de2ee71ac12fba4a43676c589c9acb323492ac..02790732a23e02e21a5708169780e222711754c2 100644 --- a/source/libs/sync/inc/syncPipeline.h +++ b/source/libs/sync/inc/syncPipeline.h @@ -56,39 +56,42 @@ typedef struct SSyncLogBuffer { int64_t size; TdThreadMutex mutex; TdThreadMutexAttr attr; + int64_t totalIndex; + bool isCatchup; } SSyncLogBuffer; // SSyncLogRepMgr -SSyncLogReplMgr* syncLogReplMgrCreate(); -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr); -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr); +SSyncLogReplMgr* syncLogReplCreate(); +void syncLogReplDestroy(SSyncLogReplMgr* pMgr); +void syncLogReplReset(SSyncLogReplMgr* pMgr); -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode); -void syncNodeLogReplMgrDestroy(SSyncNode* pNode); +int32_t syncNodeLogReplInit(SSyncNode* pNode); +void syncNodeLogReplDestroy(SSyncNode* pNode); // access -static FORCE_INLINE int64_t syncLogGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int64_t syncLogReplGetRetryBackoffTimeMs(SSyncLogReplMgr* pMgr) { return ((int64_t)1 << pMgr->retryBackoff) * SYNC_LOG_REPL_RETRY_WAIT_MS; } -static FORCE_INLINE int32_t syncLogGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { +static FORCE_INLINE int32_t syncLogReplGetNextRetryBackoff(SSyncLogReplMgr* pMgr) { return TMIN(pMgr->retryBackoff + 1, SYNC_MAX_RETRY_BACKOFF); } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier); -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); +int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index); -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, + bool* pBarrier); -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode); +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg); + +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg); // SSyncLogBuffer SSyncLogBuffer* syncLogBufferCreate(); @@ -100,6 +103,7 @@ int32_t syncLogBufferReInit(SSyncLogBuffer* pBuf, SSyncNode* pNode); int64_t syncLogBufferGetEndIndex(SSyncLogBuffer* pBuf); SyncTerm syncLogBufferGetLastMatchTerm(SSyncLogBuffer* pBuf); bool syncLogBufferIsEmpty(SSyncLogBuffer* pBuf); + int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry); int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEntry* pEntry, SyncTerm prevTerm); int64_t syncLogBufferProceed(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncTerm* pMatchTerm); @@ -111,8 +115,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, int32_t syncLogBufferValidate(SSyncLogBuffer* pBuf); int32_t syncLogBufferRollback(SSyncLogBuffer* pBuf, SSyncNode* pNode, SyncIndex toIndex); -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, - int32_t applyCode); +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, + int32_t applyCode); #ifdef __cplusplus } #endif diff --git a/source/libs/sync/inc/syncRaftEntry.h b/source/libs/sync/inc/syncRaftEntry.h index a39e043c52a0016a6e7242572667a9d02d62be5e..f9447e016882f9c219a74ef96394d00535bc411f 100644 --- a/source/libs/sync/inc/syncRaftEntry.h +++ b/source/libs/sync/inc/syncRaftEntry.h @@ -45,7 +45,7 @@ SSyncRaftEntry* syncEntryBuildNoop(SyncTerm term, SyncIndex index, int32_t vgId) void syncEntryDestroy(SSyncRaftEntry* pEntry); void syncEntry2OriginalRpc(const SSyncRaftEntry* pEntry, SRpcMsg* pRpcMsg); // step 7 -static FORCE_INLINE bool syncLogIsReplicationBarrier(SSyncRaftEntry* pEntry) { +static FORCE_INLINE bool syncLogReplBarrier(SSyncRaftEntry* pEntry) { return pEntry->originalRpcType == TDMT_SYNC_NOOP; } diff --git a/source/libs/sync/src/syncAppendEntries.c b/source/libs/sync/src/syncAppendEntries.c index 9ab545075cfc38281358e8847447f821a00393aa..b488455dba4cfe5e1c103cbf2319bd730e59e7b3 100644 --- a/source/libs/sync/src/syncAppendEntries.c +++ b/source/libs/sync/src/syncAppendEntries.c @@ -137,8 +137,10 @@ int32_t syncNodeOnAppendEntries(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pReply->term = pMsg->term; } - syncNodeStepDown(ths, pMsg->term); - resetElect = true; + if(ths->raftCfg.cfg.nodeInfo[ths->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){ + syncNodeStepDown(ths, pMsg->term); + resetElect = true; + } if (pMsg->dataLen < sizeof(SSyncRaftEntry)) { sError("vgId:%d, incomplete append entries received. prev index:%" PRId64 ", term:%" PRId64 ", datalen:%d", diff --git a/source/libs/sync/src/syncAppendEntriesReply.c b/source/libs/sync/src/syncAppendEntriesReply.c index a60f43cd5ed053fc00eba4b97c32e908698ea499..7c343c0e5d343e79d9f74e63b9548830e9b78a5f 100644 --- a/source/libs/sync/src/syncAppendEntriesReply.c +++ b/source/libs/sync/src/syncAppendEntriesReply.c @@ -85,7 +85,7 @@ int32_t syncNodeOnAppendEntriesReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, failed to get log repl mgr for src addr: 0x%016" PRIx64, ths->vgId, pMsg->srcId.addr); return -1; } - (void)syncLogReplMgrProcessReply(pMgr, ths, pMsg); + (void)syncLogReplProcessReply(pMgr, ths, pMsg); } return 0; } diff --git a/source/libs/sync/src/syncCommit.c b/source/libs/sync/src/syncCommit.c index 2501b4df8b813f438a4724097fff8eb15228b5e6..d3397f8e880105953cbd6ee4e42ce13a009add7e 100644 --- a/source/libs/sync/src/syncCommit.c +++ b/source/libs/sync/src/syncCommit.c @@ -75,10 +75,12 @@ bool syncNodeAgreedUpon(SSyncNode* pNode, SyncIndex index) { SSyncIndexMgr* pMatches = pNode->pMatchIndex; ASSERT(pNode->replicaNum == pMatches->replicaNum); - for (int i = 0; i < pNode->replicaNum; i++) { - SyncIndex matchIndex = pMatches->index[i]; - if (matchIndex >= index) { - count++; + for (int i = 0; i < pNode->totalReplicaNum; i++) { + if(pNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_VOTER){ + SyncIndex matchIndex = pMatches->index[i]; + if (matchIndex >= index) { + count++; + } } } diff --git a/source/libs/sync/src/syncElection.c b/source/libs/sync/src/syncElection.c index 3699efbc594c812c798c0c4b6aa38a6ae001edc5..86e28db90c93bf69a5ad40d0a6149bb7a7cad111 100644 --- a/source/libs/sync/src/syncElection.c +++ b/source/libs/sync/src/syncElection.c @@ -41,6 +41,8 @@ static int32_t syncNodeRequestVotePeers(SSyncNode* pNode) { int32_t ret = 0; for (int i = 0; i < pNode->peersNum; ++i) { + if(pNode->peersNodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; + SRpcMsg rpcMsg = {0}; ret = syncBuildRequestVote(&rpcMsg, pNode->vgId); if (ret < 0) { diff --git a/source/libs/sync/src/syncIndexMgr.c b/source/libs/sync/src/syncIndexMgr.c index 7ecb9d77825b496ea290d4bbca6f3c873edc716a..e3c3f63a4f89b4b0021325f54a8b595e28f56970 100644 --- a/source/libs/sync/src/syncIndexMgr.c +++ b/source/libs/sync/src/syncIndexMgr.c @@ -26,6 +26,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { pIndexMgr->replicas = &pNode->replicasId; pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->totalReplicaNum = pNode->totalReplicaNum; pIndexMgr->pNode = pNode; syncIndexMgrClear(pIndexMgr); @@ -35,6 +36,7 @@ SSyncIndexMgr *syncIndexMgrCreate(SSyncNode *pNode) { void syncIndexMgrUpdate(SSyncIndexMgr *pIndexMgr, SSyncNode *pNode) { pIndexMgr->replicas = &pNode->replicasId; pIndexMgr->replicaNum = pNode->replicaNum; + pIndexMgr->totalReplicaNum = pNode->totalReplicaNum; pIndexMgr->pNode = pNode; syncIndexMgrClear(pIndexMgr); } @@ -50,14 +52,14 @@ void syncIndexMgrClear(SSyncIndexMgr *pIndexMgr) { memset(pIndexMgr->privateTerm, 0, sizeof(pIndexMgr->privateTerm)); int64_t timeNow = taosGetTimestampMs(); - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { pIndexMgr->startTimeArr[i] = 0; pIndexMgr->recvTimeArr[i] = timeNow; } } void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncIndex index) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->index)[i] = index; return; @@ -69,7 +71,7 @@ void syncIndexMgrSetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, Sync } SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { - for (int i = 0; i < pNode->replicaNum; i++) { + for (int i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], pRaftId)) { return pNode->logReplMgrs[i]; } @@ -80,7 +82,7 @@ SSyncLogReplMgr *syncNodeGetLogReplMgr(SSyncNode *pNode, SRaftId *pRaftId) { } SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { SyncIndex idx = (pIndexMgr->index)[i]; return idx; @@ -93,7 +95,7 @@ SyncIndex syncIndexMgrGetIndex(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) } void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t startTime) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->startTimeArr)[i] = startTime; return; @@ -105,7 +107,7 @@ void syncIndexMgrSetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, } int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { int64_t startTime = (pIndexMgr->startTimeArr)[i]; return startTime; @@ -118,7 +120,7 @@ int64_t syncIndexMgrGetStartTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftI } void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, int64_t recvTime) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->recvTimeArr)[i] = recvTime; return; @@ -130,7 +132,7 @@ void syncIndexMgrSetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, i } int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { int64_t recvTime = (pIndexMgr->recvTimeArr)[i]; return recvTime; @@ -143,7 +145,7 @@ int64_t syncIndexMgrGetRecvTime(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId } void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncTerm term) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { (pIndexMgr->privateTerm)[i] = term; return; @@ -155,7 +157,7 @@ void syncIndexMgrSetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId, SyncT } SyncTerm syncIndexMgrGetTerm(SSyncIndexMgr *pIndexMgr, const SRaftId *pRaftId) { - for (int i = 0; i < pIndexMgr->replicaNum; ++i) { + for (int i = 0; i < pIndexMgr->totalReplicaNum; ++i) { if (syncUtilSameId(&((*(pIndexMgr->replicas))[i]), pRaftId)) { SyncTerm term = (pIndexMgr->privateTerm)[i]; return term; diff --git a/source/libs/sync/src/syncMain.c b/source/libs/sync/src/syncMain.c index 524daee7cf2e0ece65ba6ffac02ecaf9c9d2117f..499df4a98b6f6bc8fac65a916e2163313e020e5e 100644 --- a/source/libs/sync/src/syncMain.c +++ b/source/libs/sync/src/syncMain.c @@ -141,6 +141,13 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return -1; + if(pSyncNode->raftCfg.lastConfigIndex >= pNewCfg->lastIndex){ + syncNodeRelease(pSyncNode); + sInfo("vgId:%d, no need Reconfig, current index:%" PRId64 ", new index:%" PRId64, pSyncNode->vgId, + pSyncNode->raftCfg.lastConfigIndex, pNewCfg->lastIndex); + return 0; + } + if (!syncNodeCheckNewConfig(pSyncNode, pNewCfg)) { syncNodeRelease(pSyncNode); terrno = TSDB_CODE_SYN_NEW_CONFIG_ERROR; @@ -149,12 +156,12 @@ int32_t syncReconfig(int64_t rid, SSyncCfg* pNewCfg) { } syncNodeUpdateNewConfigIndex(pSyncNode, pNewCfg); - syncNodeDoConfigChange(pSyncNode, pNewCfg, SYNC_INDEX_INVALID); + syncNodeDoConfigChange(pSyncNode, pNewCfg, pNewCfg->lastIndex); if (pSyncNode->state == TAOS_SYNC_STATE_LEADER) { syncNodeStopHeartbeatTimer(pSyncNode); - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { syncHbTimerInit(pSyncNode, &pSyncNode->peerHeartbeatTimerArr[i], pSyncNode->replicasId[i]); } @@ -315,8 +322,9 @@ int32_t syncBeginSnapshot(int64_t rid, int64_t lastApplyIndex) { } } - if (pSyncNode->replicaNum > 1) { - if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pSyncNode->totalReplicaNum > 1) { + if (pSyncNode->state != TAOS_SYNC_STATE_LEADER && pSyncNode->state != TAOS_SYNC_STATE_FOLLOWER + && pSyncNode->state != TAOS_SYNC_STATE_LEARNER) { sNTrace(pSyncNode, "new-snapshot-index:%" PRId64 " candidate or unknown state, do not delete wal", lastApplyIndex); syncNodeRelease(pSyncNode); @@ -455,8 +463,7 @@ bool syncSnapshotRecving(int64_t rid) { int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { if (pSyncNode->peersNum == 0) { sDebug("vgId:%d, only one replica, cannot leader transfer", pSyncNode->vgId); - terrno = TSDB_CODE_SYN_ONE_REPLICA; - return -1; + return 0; } int32_t ret = 0; @@ -478,7 +485,6 @@ int32_t syncNodeLeaderTransfer(SSyncNode* pSyncNode) { int32_t syncNodeLeaderTransferTo(SSyncNode* pSyncNode, SNodeInfo newLeader) { if (pSyncNode->replicaNum == 1) { sDebug("vgId:%d, only one replica, cannot leader transfer", pSyncNode->vgId); - terrno = TSDB_CODE_SYN_ONE_REPLICA; return -1; } @@ -537,7 +543,8 @@ void syncGetRetryEpSet(int64_t rid, SEpSet* pEpSet) { SSyncNode* pSyncNode = syncNodeAcquire(rid); if (pSyncNode == NULL) return; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { + if(pSyncNode->raftCfg.cfg.nodeInfo[i].nodeRole == TAOS_SYNC_ROLE_LEARNER) continue; SEp* pEp = &pEpSet->eps[i]; tstrncpy(pEp->fqdn, pSyncNode->raftCfg.cfg.nodeInfo[i].nodeFqdn, TSDB_FQDN_LEN); pEp->port = (pSyncNode->raftCfg.cfg.nodeInfo)[i].nodePort; @@ -564,6 +571,46 @@ int32_t syncPropose(int64_t rid, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { return ret; } +int32_t syncIsCatchUp(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + sError("sync Node Acquire error since %d", errno); + return -1; + } + + int32_t isCatchUp = 0; + if(pSyncNode->pLogBuf->totalIndex < 0 || pSyncNode->pLogBuf->commitIndex < 0 || + pSyncNode->pLogBuf->totalIndex < pSyncNode->pLogBuf->commitIndex || + pSyncNode->pLogBuf->totalIndex - pSyncNode->pLogBuf->commitIndex > SYNC_LEARNER_CATCHUP){ + sInfo("vgId:%d, Not catch up, wait one second, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + isCatchUp = 0; + } + else{ + sInfo("vgId:%d, Catch up, totalIndex:%" PRId64 " commitIndex:%" PRId64 " matchIndex:%" PRId64, + pSyncNode->vgId, pSyncNode->pLogBuf->totalIndex, pSyncNode->pLogBuf->commitIndex, + pSyncNode->pLogBuf->matchIndex); + isCatchUp = 1; + } + + syncNodeRelease(pSyncNode); + return isCatchUp; +} + +ESyncRole syncGetRole(int64_t rid) { + SSyncNode* pSyncNode = syncNodeAcquire(rid); + if (pSyncNode == NULL) { + sError("sync Node Acquire error since %d", errno); + return -1; + } + + ESyncRole role = pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole; + + syncNodeRelease(pSyncNode); + return role; +} + int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_t* seq) { if (pSyncNode->state != TAOS_SYNC_STATE_LEADER) { terrno = TSDB_CODE_SYN_NOT_LEADER; @@ -617,7 +664,7 @@ int32_t syncNodePropose(SSyncNode* pSyncNode, SRpcMsg* pMsg, bool isWeak, int64_ sNTrace(pSyncNode, "propose msg, type:%s", TMSG_INFO(pMsg->msgType)); code = (*pSyncNode->syncEqMsg)(pSyncNode->msgcb, &rpcMsg); if (code != 0) { - sError("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); + sWarn("vgId:%d, failed to propose msg while enqueue since %s", pSyncNode->vgId, terrstr()); (void)syncRespMgrDel(pSyncNode->pSyncRespMgr, seqNum); } @@ -655,6 +702,8 @@ static int32_t syncHbTimerStart(SSyncNode* pSyncNode, SSyncTimer* pSyncTimer) { pData->logicClock = pSyncTimer->logicClock; pData->execTime = tsNow + pSyncTimer->timerMS; + sTrace("vgId:%d, start hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, pData->rid, pData->destId.addr); + taosTmrReset(pSyncTimer->timerCb, pSyncTimer->timerMS / HEARTBEAT_TICK_NUM, (void*)(pData->rid), syncEnv()->pTimerManager, &pSyncTimer->pTimer); } else { @@ -719,7 +768,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { sInfo("vgId:%d, create a new raft config file", pSyncNode->vgId); pSyncNode->raftCfg.isStandBy = pSyncInfo->isStandBy; pSyncNode->raftCfg.snapshotStrategy = pSyncInfo->snapshotStrategy; - pSyncNode->raftCfg.lastConfigIndex = SYNC_INDEX_INVALID; + pSyncNode->raftCfg.lastConfigIndex = pSyncInfo->syncCfg.lastIndex; pSyncNode->raftCfg.batchSize = pSyncInfo->batchSize; pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; pSyncNode->raftCfg.configIndexCount = 1; @@ -736,7 +785,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { goto _error; } - if (pSyncInfo->syncCfg.replicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { + if (pSyncInfo->syncCfg.totalReplicaNum > 0 && syncIsConfigChanged(&pSyncNode->raftCfg.cfg, &pSyncInfo->syncCfg)) { sInfo("vgId:%d, use sync config from input options and write to cfg file", pSyncNode->vgId); pSyncNode->raftCfg.cfg = pSyncInfo->syncCfg; if (syncWriteCfgFile(pSyncNode) != 0) { @@ -753,8 +802,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->vgId = pSyncInfo->vgId; SSyncCfg* pCfg = &pSyncNode->raftCfg.cfg; bool updated = false; - sInfo("vgId:%d, start to open sync node, replica:%d selfIndex:%d", pSyncNode->vgId, pCfg->replicaNum, pCfg->myIndex); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + sInfo("vgId:%d, start to open sync node, totalReplicaNum:%d replicaNum:%d selfIndex:%d", + pSyncNode->vgId, pCfg->totalReplicaNum, pCfg->replicaNum, pCfg->myIndex); + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SNodeInfo* pNode = &pCfg->nodeInfo[i]; if (tmsgUpdateDnodeInfo(&pNode->nodeId, &pNode->clusterId, pNode->nodeFqdn, &pNode->nodePort)) { updated = true; @@ -792,9 +842,9 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { } // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (i != pSyncNode->raftCfg.cfg.myIndex) { pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); @@ -810,7 +860,8 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (!syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i])) { sError("vgId:%d, failed to determine raft member id, replica:%d", pSyncNode->vgId, i); goto _error; @@ -934,7 +985,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->heartbeatTimerCounter = 0; // init peer heartbeat timer - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { syncHbTimerInit(pSyncNode, &(pSyncNode->peerHeartbeatTimerArr[i]), (pSyncNode->replicasId)[i]); } @@ -949,7 +1000,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->restoreFinish = false; // snapshot senders - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { SSyncSnapshotSender* pSender = snapshotSenderCreate(pSyncNode, i); if (pSender == NULL) return NULL; @@ -967,7 +1018,7 @@ SSyncNode* syncNodeOpen(SSyncInfo* pSyncInfo) { pSyncNode->changing = false; // replication mgr - if (syncNodeLogReplMgrInit(pSyncNode) < 0) { + if (syncNodeLogReplInit(pSyncNode) < 0) { sError("vgId:%d, failed to init repl mgr since %s.", pSyncNode->vgId, terrstr()); goto _error; } @@ -1059,14 +1110,19 @@ int32_t syncNodeRestore(SSyncNode* pSyncNode) { int32_t syncNodeStart(SSyncNode* pSyncNode) { // start raft - if (pSyncNode->replicaNum == 1) { - raftStoreNextTerm(pSyncNode); - syncNodeBecomeLeader(pSyncNode, "one replica start"); + if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER){ + syncNodeBecomeLearner(pSyncNode, "first start"); + } + else{ + if (pSyncNode->replicaNum == 1) { + raftStoreNextTerm(pSyncNode); + syncNodeBecomeLeader(pSyncNode, "one replica start"); - // Raft 3.6.2 Committing entries from previous terms - syncNodeAppendNoop(pSyncNode); - } else { - syncNodeBecomeFollower(pSyncNode, "first start"); + // Raft 3.6.2 Committing entries from previous terms + syncNodeAppendNoop(pSyncNode); + } else { + syncNodeBecomeFollower(pSyncNode, "first start"); + } } int32_t ret = 0; @@ -1140,7 +1196,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncNodeStopPingTimer(pSyncNode); syncNodeStopElectTimer(pSyncNode); syncNodeStopHeartbeatTimer(pSyncNode); - syncNodeLogReplMgrDestroy(pSyncNode); + syncNodeLogReplDestroy(pSyncNode); syncRespMgrDestroy(pSyncNode->pSyncRespMgr); pSyncNode->pSyncRespMgr = NULL; @@ -1157,7 +1213,7 @@ void syncNodeClose(SSyncNode* pSyncNode) { syncLogBufferDestroy(pSyncNode->pLogBuf); pSyncNode->pLogBuf = NULL; - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (pSyncNode->senders[i] != NULL) { sDebug("vgId:%d, snapshot sender destroy while close, data:%p", pSyncNode->vgId, pSyncNode->senders[i]); @@ -1350,7 +1406,7 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { bool b1 = false; bool b2 = false; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { if (strcmp(pCfg->nodeInfo[i].nodeFqdn, pNode->myNodeInfo.nodeFqdn) == 0 && pCfg->nodeInfo[i].nodePort == pNode->myNodeInfo.nodePort) { b1 = true; @@ -1358,7 +1414,7 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { } } - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SRaftId raftId = { .addr = SYNC_ADDR(&pCfg->nodeInfo[i]), .vgId = pNode->vgId, @@ -1375,13 +1431,14 @@ inline bool syncNodeInConfig(SSyncNode* pNode, const SSyncCfg* pCfg) { } static bool syncIsConfigChanged(const SSyncCfg* pOldCfg, const SSyncCfg* pNewCfg) { - if (pOldCfg->replicaNum != pNewCfg->replicaNum) return true; + if (pOldCfg->totalReplicaNum != pNewCfg->totalReplicaNum) return true; if (pOldCfg->myIndex != pNewCfg->myIndex) return true; - for (int32_t i = 0; i < pOldCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pOldCfg->totalReplicaNum; ++i) { const SNodeInfo* pOldInfo = &pOldCfg->nodeInfo[i]; const SNodeInfo* pNewInfo = &pNewCfg->nodeInfo[i]; if (strcmp(pOldInfo->nodeFqdn, pNewInfo->nodeFqdn) != 0) return true; if (pOldInfo->nodePort != pNewInfo->nodePort) return true; + if(pOldInfo->nodeRole != pNewInfo->nodeRole) return true; } return false; @@ -1418,8 +1475,10 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // log begin config change - sNInfo(pSyncNode, "begin do config change, from %d to %d, replicas:%d", pSyncNode->vgId, oldConfig.replicaNum, - pNewConfig->replicaNum); + sNInfo(pSyncNode, "begin do config change, from %d to %d, from %" PRId64 " to %" PRId64 ", replicas:%d", + pSyncNode->vgId, + oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum, + oldConfig.lastIndex, pNewConfig->lastIndex); if (IamInNew) { pSyncNode->raftCfg.isStandBy = 0; // change isStandBy to normal @@ -1436,11 +1495,10 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde int32_t ret = 0; // save snapshot senders - int32_t oldReplicaNum = pSyncNode->replicaNum; - SRaftId oldReplicasId[TSDB_MAX_REPLICA]; + SRaftId oldReplicasId[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; memcpy(oldReplicasId, pSyncNode->replicasId, sizeof(oldReplicasId)); - SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA]; - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + SSyncSnapshotSender* oldSenders[TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA]; + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { oldSenders[i] = pSyncNode->senders[i]; sSTrace(oldSenders[i], "snapshot sender save old"); } @@ -1450,9 +1508,9 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde syncUtilNodeInfo2RaftId(&pSyncNode->myNodeInfo, pSyncNode->vgId, &pSyncNode->myRaftId); // init peersNum, peers, peersId - pSyncNode->peersNum = pSyncNode->raftCfg.cfg.replicaNum - 1; + pSyncNode->peersNum = pSyncNode->raftCfg.cfg.totalReplicaNum - 1; int32_t j = 0; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { if (i != pSyncNode->raftCfg.cfg.myIndex) { pSyncNode->peersNodeInfo[j] = pSyncNode->raftCfg.cfg.nodeInfo[i]; syncUtilNodeInfo2EpSet(&pSyncNode->peersNodeInfo[j], &pSyncNode->peersEpset[j]); @@ -1465,7 +1523,8 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // init replicaNum, replicasId pSyncNode->replicaNum = pSyncNode->raftCfg.cfg.replicaNum; - for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.replicaNum; ++i) { + pSyncNode->totalReplicaNum = pSyncNode->raftCfg.cfg.totalReplicaNum; + for (int32_t i = 0; i < pSyncNode->raftCfg.cfg.totalReplicaNum; ++i) { syncUtilNodeInfo2RaftId(&pSyncNode->raftCfg.cfg.nodeInfo[i], pSyncNode->vgId, &pSyncNode->replicasId[i]); } @@ -1480,15 +1539,15 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde // reset snapshot senders // clear new - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { pSyncNode->senders[i] = NULL; } // reset new - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->totalReplicaNum; ++i) { // reset sender bool reset = false; - for (int32_t j = 0; j < TSDB_MAX_REPLICA; ++j) { + for (int32_t j = 0; j < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++j) { if (syncUtilSameId(&(pSyncNode->replicasId)[i], &oldReplicasId[j]) && oldSenders[j] != NULL) { sNTrace(pSyncNode, "snapshot sender reset for:%" PRId64 ", newIndex:%d, dnode:%d, %p", (pSyncNode->replicasId)[i].addr, i, DID(&pSyncNode->replicasId[i]), oldSenders[j]); @@ -1510,7 +1569,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // create new - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (pSyncNode->senders[i] == NULL) { pSyncNode->senders[i] = snapshotSenderCreate(pSyncNode, i); if (pSyncNode->senders[i] == NULL) { @@ -1525,7 +1584,7 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } // free old - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { if (oldSenders[i] != NULL) { sSDebug(oldSenders[i], "snapshot sender destroy old, data:%p replica-index:%d", oldSenders[i], i); snapshotSenderDestroy(oldSenders[i]); @@ -1550,12 +1609,12 @@ void syncNodeDoConfigChange(SSyncNode* pSyncNode, SSyncCfg* pNewConfig, SyncInde } else { // persist cfg syncWriteCfgFile(pSyncNode); - sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); + sNInfo(pSyncNode, "do not config change from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); } _END: // log end config change - sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.replicaNum, pNewConfig->replicaNum); + sNInfo(pSyncNode, "end do config change, from %d to %d", oldConfig.totalReplicaNum, pNewConfig->totalReplicaNum); } // raft state change -------------- @@ -1635,6 +1694,27 @@ void syncNodeBecomeFollower(SSyncNode* pSyncNode, const char* debugStr) { syncNodeResetElectTimer(pSyncNode); } +void syncNodeBecomeLearner(SSyncNode* pSyncNode, const char* debugStr) { + pSyncNode->hbSlowNum = 0; + + // state change + pSyncNode->state = TAOS_SYNC_STATE_LEARNER; + + // trace log + sNTrace(pSyncNode, "become learner %s", debugStr); + + // call back + if (pSyncNode->pFsm != NULL && pSyncNode->pFsm->FpBecomeLearnerCb != NULL) { + pSyncNode->pFsm->FpBecomeLearnerCb(pSyncNode->pFsm); + } + + // min match index + pSyncNode->minMatchIndex = SYNC_INDEX_INVALID; + + // reset log buffer + syncLogBufferReset(pSyncNode->pLogBuf, pSyncNode); +} + // TLA+ Spec // \* Candidate i transitions to leader. // BecomeLeader(i) == @@ -1752,7 +1832,7 @@ void syncNodeCandidate2Leader(SSyncNode* pSyncNode) { bool syncNodeIsMnode(SSyncNode* pSyncNode) { return (pSyncNode->vgId == 1); } int32_t syncNodePeerStateInit(SSyncNode* pSyncNode) { - for (int32_t i = 0; i < TSDB_MAX_REPLICA; ++i) { + for (int32_t i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; ++i) { pSyncNode->peerStates[i].lastSendIndex = SYNC_INDEX_INVALID; pSyncNode->peerStates[i].lastSendTime = 0; } @@ -2039,7 +2119,7 @@ static void syncNodeEqHeartbeatTimer(void* param, void* tmrId) { if (!syncIsInit()) return; SSyncNode* pNode = param; - if (pNode->replicaNum > 1) { + if (pNode->totalReplicaNum > 1) { if (atomic_load_64(&pNode->heartbeatTimerLogicClockUser) <= atomic_load_64(&pNode->heartbeatTimerLogicClock)) { SRpcMsg rpcMsg = {0}; int32_t code = syncBuildTimeout(&rpcMsg, SYNC_TIMEOUT_HEARTBEAT, atomic_load_64(&pNode->heartbeatTimerLogicClock), @@ -2101,9 +2181,9 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { return; } - // sTrace("vgId:%d, eq peer hb timer", pSyncNode->vgId); + sTrace("vgId:%d, eq peer hb timer, rid:%" PRId64 " addr:%" PRId64, pSyncNode->vgId, hbDataRid, pData->destId.addr); - if (pSyncNode->replicaNum > 1) { + if (pSyncNode->totalReplicaNum > 1) { int64_t timerLogicClock = atomic_load_64(&pSyncTimer->logicClock); int64_t msgLogicClock = atomic_load_64(&pData->logicClock); @@ -2130,6 +2210,7 @@ static void syncNodeEqPeerHeartbeatTimer(void* param, void* tmrId) { pSyncTimer->timeStamp = tsNow; // send msg + sTrace("vgId:%d, send heartbeat to dnode:%d", pSyncNode->vgId, DID(&(pSyncMsg->destId))); syncLogSendHeartbeat(pSyncNode, pSyncMsg, false, timerElapsed, pData->execTime); syncNodeSendHeartbeat(pSyncNode, &pSyncMsg->destId, &rpcMsg); } else { @@ -2182,7 +2263,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { if (syncLogBufferAppend(ths->pLogBuf, ths, pEntry) < 0) { sError("vgId:%d, failed to enqueue sync log buffer, index:%" PRId64, ths->vgId, pEntry->index); ASSERT(terrno != 0); - (void)syncLogFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); + (void)syncFsmExecute(ths, ths->pFsm, ths->state, raftStoreGetTerm(ths), pEntry, terrno); syncEntryDestroy(pEntry); return -1; } @@ -2212,7 +2293,7 @@ int32_t syncNodeAppend(SSyncNode* ths, SSyncRaftEntry* pEntry) { } bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { - if (pSyncNode->replicaNum == 1) { + if (pSyncNode->totalReplicaNum == 1) { return false; } @@ -2237,7 +2318,7 @@ bool syncNodeHeartbeatReplyTimeout(SSyncNode* pSyncNode) { bool syncNodeSnapshotSending(SSyncNode* pSyncNode) { if (pSyncNode == NULL) return false; bool b = false; - for (int32_t i = 0; i < pSyncNode->replicaNum; ++i) { + for (int32_t i = 0; i < pSyncNode->totalReplicaNum; ++i) { if (pSyncNode->senders[i] != NULL && pSyncNode->senders[i]->start) { b = true; break; @@ -2328,35 +2409,40 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { pMsgReply->startTime = ths->startTime; pMsgReply->timeStamp = tsMs; + sTrace( + "vgId:%d, heartbeat msg from dnode:%d, cluster:%d, Msgterm:%" PRId64 " currentTerm:%" PRId64, + ths->vgId, DID(&(pMsg->srcId)), CID(&(pMsg->srcId)), pMsg->term, currentTerm); + if (pMsg->term == currentTerm && ths->state != TAOS_SYNC_STATE_LEADER) { syncIndexMgrSetRecvTime(ths->pNextIndex, &(pMsg->srcId), tsMs); resetElect = true; ths->minMatchIndex = pMsg->minMatchIndex; - if (ths->state == TAOS_SYNC_STATE_FOLLOWER) { + if (ths->state == TAOS_SYNC_STATE_FOLLOWER || ths->state == TAOS_SYNC_STATE_LEARNER) { SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); SyncLocalCmd* pSyncMsg = rpcMsgLocalCmd.pCont; - pSyncMsg->cmd = SYNC_LOCAL_CMD_FOLLOWER_CMT; + pSyncMsg->cmd = + (ths->state == TAOS_SYNC_STATE_LEARNER) ? SYNC_LOCAL_CMD_LEARNER_CMT : SYNC_LOCAL_CMD_FOLLOWER_CMT; pSyncMsg->commitIndex = pMsg->commitIndex; pSyncMsg->currentTerm = pMsg->term; - SyncIndex fcIndex = pSyncMsg->commitIndex; if (ths->syncEqMsg != NULL && ths->msgcb != NULL) { int32_t code = ths->syncEqMsg(ths->msgcb, &rpcMsgLocalCmd); if (code != 0) { - sError("vgId:%d, sync enqueue fc-commit msg error, code:%d", ths->vgId, code); + sError("vgId:%d, failed to enqueue commit msg from heartbeat since %s, code:%d", ths->vgId, terrstr(), code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue fc-commit msg, fc-index:%" PRId64, ths->vgId, fcIndex); + sTrace("vgId:%d, enqueue commit msg from heartbeat, commit-index:%" PRId64 ", term:%" PRId64, ths->vgId, + pMsg->commitIndex, pMsg->term); } } } } - if (pMsg->term >= currentTerm && ths->state != TAOS_SYNC_STATE_FOLLOWER) { + if (pMsg->term >= currentTerm && ths->state == TAOS_SYNC_STATE_LEADER) { SRpcMsg rpcMsgLocalCmd = {0}; (void)syncBuildLocalCmd(&rpcMsgLocalCmd, ths->vgId); @@ -2371,7 +2457,7 @@ int32_t syncNodeOnHeartbeat(SSyncNode* ths, const SRpcMsg* pRpcMsg) { sError("vgId:%d, sync enqueue step-down msg error, code:%d", ths->vgId, code); rpcFreeCont(rpcMsgLocalCmd.pCont); } else { - sTrace("vgId:%d, sync enqueue step-down msg, new-term:%" PRId64, ths->vgId, pSyncMsg->currentTerm); + sTrace("vgId:%d, sync enqueue step-down msg, new-term:%" PRId64, ths->vgId, pMsg->term); } } } @@ -2400,7 +2486,7 @@ int32_t syncNodeOnHeartbeatReply(SSyncNode* ths, const SRpcMsg* pRpcMsg) { syncIndexMgrSetRecvTime(ths->pMatchIndex, &pMsg->srcId, tsMs); - return syncLogReplMgrProcessHeartbeatReply(pMgr, ths, pMsg); + return syncLogReplProcessHeartbeatReply(pMgr, ths, pMsg); } int32_t syncNodeOnHeartbeatReplyOld(SSyncNode* ths, const SRpcMsg* pRpcMsg) { @@ -2426,7 +2512,7 @@ int32_t syncNodeOnLocalCmd(SSyncNode* ths, const SRpcMsg* pRpcMsg) { if (pMsg->cmd == SYNC_LOCAL_CMD_STEP_DOWN) { syncNodeStepDown(ths, pMsg->currentTerm); - } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT) { + } else if (pMsg->cmd == SYNC_LOCAL_CMD_FOLLOWER_CMT || pMsg->cmd == SYNC_LOCAL_CMD_LEARNER_CMT) { if (syncLogBufferIsEmpty(ths->pLogBuf)) { sError("vgId:%d, sync log buffer is empty.", ths->vgId); return 0; @@ -2502,13 +2588,15 @@ const char* syncStr(ESyncState state) { return "error"; case TAOS_SYNC_STATE_OFFLINE: return "offline"; + case TAOS_SYNC_STATE_LEARNER: + return "learner"; default: return "unknown"; } } int32_t syncNodeUpdateNewConfigIndex(SSyncNode* ths, SSyncCfg* pNewCfg) { - for (int32_t i = 0; i < pNewCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pNewCfg->totalReplicaNum; ++i) { SRaftId raftId = { .addr = SYNC_ADDR(&pNewCfg->nodeInfo[i]), .vgId = ths->vgId, @@ -2528,7 +2616,7 @@ bool syncNodeIsOptimizedOneReplica(SSyncNode* ths, SRpcMsg* pMsg) { } bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(&((ths->replicasId)[i]), pRaftId)) { return true; } @@ -2538,7 +2626,7 @@ bool syncNodeInRaftGroup(SSyncNode* ths, SRaftId* pRaftId) { SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) { SSyncSnapshotSender* pSender = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pSender = (ths->senders)[i]; } @@ -2548,7 +2636,7 @@ SSyncSnapshotSender* syncNodeGetSnapshotSender(SSyncNode* ths, SRaftId* pDestId) SSyncTimer* syncNodeGetHbTimer(SSyncNode* ths, SRaftId* pDestId) { SSyncTimer* pTimer = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pTimer = &((ths->peerHeartbeatTimerArr)[i]); } @@ -2558,7 +2646,7 @@ SSyncTimer* syncNodeGetHbTimer(SSyncNode* ths, SRaftId* pDestId) { SPeerState* syncNodeGetPeerState(SSyncNode* ths, const SRaftId* pDestId) { SPeerState* pState = NULL; - for (int32_t i = 0; i < ths->replicaNum; ++i) { + for (int32_t i = 0; i < ths->totalReplicaNum; ++i) { if (syncUtilSameId(pDestId, &((ths->replicasId)[i]))) { pState = &((ths->peerStates)[i]); } diff --git a/source/libs/sync/src/syncPipeline.c b/source/libs/sync/src/syncPipeline.c index 6600b505c18405c40abe262655898d20c078fd60..8bb72de518cd7ae4f91d0a6db631f7bc34aeeffc 100644 --- a/source/libs/sync/src/syncPipeline.c +++ b/source/libs/sync/src/syncPipeline.c @@ -53,8 +53,15 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _err; } + if (pNode->restoreFinish && index - pBuf->commitIndex >= TSDB_SYNC_NEGOTIATION_WIN) { + terrno = TSDB_CODE_SYN_NEGOTIATION_WIN_FULL; + sError("vgId:%d, failed to append since %s, index:%" PRId64 ", commit-index:%" PRId64, pNode->vgId, terrstr(), + index, pBuf->commitIndex); + goto _err; + } + SyncIndex appliedIndex = pNode->pFsm->FpAppliedIndexCb(pNode->pFsm); - if (pNode->restoreFinish && pBuf->commitIndex - appliedIndex >= pBuf->size) { + if (pNode->restoreFinish && pBuf->commitIndex - appliedIndex >= TSDB_SYNC_APPLYQ_SIZE_LIMIT) { terrno = TSDB_CODE_SYN_WRITE_STALL; sError("vgId:%d, failed to append since %s. index:%" PRId64 ", commit-index:%" PRId64 ", applied-index:%" PRId64, pNode->vgId, terrstr(), index, pBuf->commitIndex, appliedIndex); @@ -83,10 +90,11 @@ int32_t syncLogBufferAppend(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt _err: syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); + taosMsleep(1); return -1; } -SyncTerm syncLogReplMgrGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +SyncTerm syncLogReplGetPrevLogTerm(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SSyncRaftEntry* pEntry = NULL; SyncIndex prevIndex = index - 1; @@ -242,6 +250,8 @@ int32_t syncLogBufferInitWithoutLock(SSyncLogBuffer* pBuf, SSyncNode* pNode) { // update startIndex pBuf->startIndex = takeDummy ? index : index + 1; + pBuf->isCatchup = false; + sInfo("vgId:%d, init sync log buffer. buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -316,7 +326,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm term = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index + 1); + SyncTerm term = syncLogReplGetPrevLogTerm(NULL, pNode, index + 1); ASSERT(pEntry->term >= 0); if (term == pEntry->term) { ret = 0; @@ -324,6 +334,15 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt goto _out; } + if(pNode->raftCfg.cfg.nodeInfo[pNode->raftCfg.cfg.myIndex].nodeRole == TAOS_SYNC_ROLE_LEARNER && + index > 0 && index > pBuf->totalIndex){ + pBuf->totalIndex = index; + sTrace("vgId:%d, update learner progress. index:%" PRId64 ", term:%" PRId64 ": prevterm:%" PRId64 + " != lastmatch:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", + pNode->vgId, pEntry->index, pEntry->term, prevTerm, lastMatchTerm, pBuf->startIndex, pBuf->commitIndex, + pBuf->matchIndex, pBuf->endIndex); + } + if (index - pBuf->startIndex >= pBuf->size) { sWarn("vgId:%d, out of buffer range. index:%" PRId64 ", term:%" PRId64 ". log buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", @@ -351,7 +370,7 @@ int32_t syncLogBufferAccept(SSyncLogBuffer* pBuf, SSyncNode* pNode, SSyncRaftEnt " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, pEntry->index, pEntry->term, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); - SyncTerm existPrevTerm = syncLogReplMgrGetPrevLogTerm(NULL, pNode, index); + SyncTerm existPrevTerm = syncLogReplGetPrevLogTerm(NULL, pNode, index); ASSERT(pEntry->term == pExist->term && (pEntry->index > pBuf->matchIndex || prevTerm == existPrevTerm)); ret = 0; goto _out; @@ -482,7 +501,7 @@ _out: return matchIndex; } -int32_t syncLogFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, +int32_t syncFsmExecute(SSyncNode* pNode, SSyncFSM* pFsm, ESyncState role, SyncTerm term, SSyncRaftEntry* pEntry, int32_t applyCode) { if (pNode->replicaNum == 1 && pNode->restoreFinish && pNode->vgId != 1) { return 0; @@ -564,7 +583,7 @@ int32_t syncLogBufferCommit(SSyncLogBuffer* pBuf, SSyncNode* pNode, int64_t comm pEntry->term, TMSG_INFO(pEntry->originalRpcType)); } - if (syncLogFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { + if (syncFsmExecute(pNode, pFsm, role, currentTerm, pEntry, 0) != 0) { sError("vgId:%d, failed to execute sync log entry. index:%" PRId64 ", term:%" PRId64 ", role:%d, current term:%" PRId64, vgId, pEntry->index, pEntry->term, role, currentTerm); @@ -611,7 +630,7 @@ _out: return ret; } -void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { +void syncLogReplReset(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) return; ASSERT(pMgr->startIndex >= 0); @@ -625,22 +644,22 @@ void syncLogReplMgrReset(SSyncLogReplMgr* pMgr) { pMgr->retryBackoff = 0; } -int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->endIndex <= pMgr->startIndex) { return 0; } SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; if (pMgr->retryBackoff == SYNC_MAX_RETRY_BACKOFF) { - syncLogReplMgrReset(pMgr); - sWarn("vgId:%d, reset sync log repl mgr since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, + syncLogReplReset(pMgr); + sWarn("vgId:%d, reset sync log repl since retry backoff exceeding limit. peer:%" PRIx64, pNode->vgId, pDestId->addr); return -1; } int32_t ret = -1; bool retried = false; - int64_t retryWaitMs = syncLogGetRetryBackoffTimeMs(pMgr); + int64_t retryWaitMs = syncLogReplGetRetryBackoffTimeMs(pMgr); int64_t nowMs = taosGetMonoTimestampMs(); int count = 0; int64_t firstIndex = -1; @@ -657,16 +676,16 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->states[pos].acked) { if (pMgr->matchIndex < index && pMgr->states[pos].timeMs + (syncGetRetryMaxWaitMs() << 3) < nowMs) { - syncLogReplMgrReset(pMgr); - sWarn("vgId:%d, reset sync log repl mgr since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, - index, pDestId->addr); + syncLogReplReset(pMgr); + sWarn("vgId:%d, reset sync log repl since stagnation. index:%" PRId64 ", peer:%" PRIx64, pNode->vgId, index, + pDestId->addr); goto _out; } continue; } bool barrier = false; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate sync log entry since %s. index:%" PRId64 ", dest:%" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); goto _out; @@ -687,7 +706,7 @@ int32_t syncLogReplMgrRetryOnNeed(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ret = 0; _out: if (retried) { - pMgr->retryBackoff = syncLogGetNextRetryBackoff(pMgr); + pMgr->retryBackoff = syncLogReplGetNextRetryBackoff(pMgr); SSyncLogBuffer* pBuf = pNode->pLogBuf; sInfo("vgId:%d, resend %d sync log entries. dest:%" PRIx64 ", indexes:%" PRId64 " ..., terms: ... %" PRId64 ", retryWaitMs:%" PRId64 ", mgr: [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 @@ -698,7 +717,7 @@ _out: return ret; } -int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; SRaftId destId = pMsg->srcId; ASSERT(pMgr->restored == false); @@ -708,7 +727,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p ASSERT(pMgr->matchIndex == 0); if (pMsg->matchIndex < 0) { pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -716,7 +735,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } } else { if (pMsg->lastSendIndex < pMgr->startIndex || pMsg->lastSendIndex >= pMgr->endIndex) { - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); return 0; } @@ -725,7 +744,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p if (pMsg->success && pMsg->matchIndex == pMsg->lastSendIndex) { pMgr->matchIndex = pMsg->matchIndex; pMgr->restored = true; - sInfo("vgId:%d, sync log repl mgr restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 + sInfo("vgId:%d, sync log repl restored. peer: dnode:%d (%" PRIx64 "), mgr: rs(%d) [%" PRId64 " %" PRId64 ", %" PRId64 "), buffer: [%" PRId64 " %" PRId64 " %" PRId64 ", %" PRId64 ")", pNode->vgId, DID(&destId), destId.addr, pMgr->restored, pMgr->startIndex, pMgr->matchIndex, pMgr->endIndex, pBuf->startIndex, pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); @@ -750,7 +769,7 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p SyncIndex index = TMIN(pMsg->matchIndex, pNode->pLogBuf->matchIndex); if (pMsg->matchIndex < pNode->pLogBuf->matchIndex) { - term = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index + 1); + term = syncLogReplGetPrevLogTerm(pMgr, pNode, index + 1); if ((index + 1 < firstVer) || (term < 0) || (term != pMsg->lastMatchTerm && (index + 1 == firstVer || index == firstVer))) { ASSERT(term >= 0 || terrno == TSDB_CODE_WAL_LOG_NOT_EXIST); @@ -773,53 +792,52 @@ int32_t syncLogReplMgrProcessReplyAsRecovery(SSyncLogReplMgr* pMgr, SSyncNode* p } // attempt to replicate the raft log at index - (void)syncLogReplMgrReset(pMgr); - return syncLogReplMgrReplicateProbe(pMgr, pNode, index); + (void)syncLogReplReset(pMgr); + return syncLogReplProbe(pMgr, pNode, index); } -int32_t syncLogReplMgrProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { +int32_t syncLogReplProcessHeartbeatReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncHeartbeatReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != 0 && pMsg->startTime != pMgr->peerStartTime) { - sInfo("vgId:%d, reset sync log repl mgr in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", + sInfo("vgId:%d, reset sync log repl in heartbeat. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64 "", pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReply(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); if (pMsg->startTime != pMgr->peerStartTime) { - sInfo("vgId:%d, reset sync log repl mgr in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 - ", old:%" PRId64, + sInfo("vgId:%d, reset sync log repl in appendlog reply. peer:%" PRIx64 ", start time:%" PRId64 ", old:%" PRId64, pNode->vgId, pMsg->srcId.addr, pMsg->startTime, pMgr->peerStartTime); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); pMgr->peerStartTime = pMsg->startTime; } if (pMgr->restored) { - (void)syncLogReplMgrProcessReplyAsNormal(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsNormal(pMgr, pNode, pMsg); } else { - (void)syncLogReplMgrProcessReplyAsRecovery(pMgr, pNode, pMsg); + (void)syncLogReplProcessReplyAsRecovery(pMgr, pNode, pMsg); } taosThreadMutexUnlock(&pBuf->mutex); return 0; } -int32_t syncLogReplMgrReplicateOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplDoOnce(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { if (pMgr->restored) { - (void)syncLogReplMgrReplicateAttempt(pMgr, pNode); + (void)syncLogReplAttempt(pMgr, pNode); } else { - (void)syncLogReplMgrReplicateProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); + (void)syncLogReplProbe(pMgr, pNode, pNode->pLogBuf->matchIndex); } return 0; } -int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { +int32_t syncLogReplProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index) { ASSERT(!pMgr->restored); ASSERT(pMgr->startIndex >= 0); int64_t retryMaxWaitMs = syncGetRetryMaxWaitMs(); @@ -829,12 +847,12 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy nowMs < pMgr->states[pMgr->startIndex % pMgr->size].timeMs + retryMaxWaitMs) { return 0; } - (void)syncLogReplMgrReset(pMgr); + (void)syncLogReplReset(pMgr); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -857,7 +875,7 @@ int32_t syncLogReplMgrReplicateProbe(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy return 0; } -int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { +int32_t syncLogReplAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) { ASSERT(pMgr->restored); SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; @@ -879,7 +897,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) SRaftId* pDestId = &pNode->replicasId[pMgr->peerId]; bool barrier = false; SyncTerm term = -1; - if (syncLogReplMgrReplicateOneTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { + if (syncLogReplSendTo(pMgr, pNode, index, &term, pDestId, &barrier) < 0) { sError("vgId:%d, failed to replicate log entry since %s. index:%" PRId64 ", dest: 0x%016" PRIx64 "", pNode->vgId, terrstr(), index, pDestId->addr); return -1; @@ -902,7 +920,7 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) } } - syncLogReplMgrRetryOnNeed(pMgr, pNode); + syncLogReplRetryOnNeed(pMgr, pNode); SSyncLogBuffer* pBuf = pNode->pLogBuf; sTrace("vgId:%d, replicated %d msgs to peer:%" PRIx64 ". indexes:%" PRId64 "..., terms: ...%" PRId64 @@ -913,14 +931,14 @@ int32_t syncLogReplMgrReplicateAttempt(SSyncLogReplMgr* pMgr, SSyncNode* pNode) return 0; } -int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { +int32_t syncLogReplProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncAppendEntriesReply* pMsg) { ASSERT(pMgr->restored == true); if (pMgr->startIndex <= pMsg->lastSendIndex && pMsg->lastSendIndex < pMgr->endIndex) { if (pMgr->startIndex < pMgr->matchIndex && pMgr->retryBackoff > 0) { - int64_t firstSentMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; - int64_t lastSentMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; - int64_t timeDiffMs = lastSentMs - firstSentMs; - if (timeDiffMs > 0 && timeDiffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { + int64_t firstMs = pMgr->states[pMgr->startIndex % pMgr->size].timeMs; + int64_t lastMs = pMgr->states[(pMgr->endIndex - 1) % pMgr->size].timeMs; + int64_t diffMs = lastMs - firstMs; + if (diffMs > 0 && diffMs < ((int64_t)SYNC_LOG_REPL_RETRY_WAIT_MS << (pMgr->retryBackoff - 1))) { pMgr->retryBackoff -= 1; } } @@ -932,10 +950,10 @@ int32_t syncLogReplMgrProcessReplyAsNormal(SSyncLogReplMgr* pMgr, SSyncNode* pNo pMgr->startIndex = pMgr->matchIndex; } - return syncLogReplMgrReplicateAttempt(pMgr, pNode); + return syncLogReplAttempt(pMgr, pNode); } -SSyncLogReplMgr* syncLogReplMgrCreate() { +SSyncLogReplMgr* syncLogReplCreate() { SSyncLogReplMgr* pMgr = taosMemoryCalloc(1, sizeof(SSyncLogReplMgr)); if (pMgr == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; @@ -949,7 +967,7 @@ SSyncLogReplMgr* syncLogReplMgrCreate() { return pMgr; } -void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { +void syncLogReplDestroy(SSyncLogReplMgr* pMgr) { if (pMgr == NULL) { return; } @@ -957,10 +975,10 @@ void syncLogReplMgrDestroy(SSyncLogReplMgr* pMgr) { return; } -int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { +int32_t syncNodeLogReplInit(SSyncNode* pNode) { + for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { ASSERT(pNode->logReplMgrs[i] == NULL); - pNode->logReplMgrs[i] = syncLogReplMgrCreate(); + pNode->logReplMgrs[i] = syncLogReplCreate(); if (pNode->logReplMgrs[i] == NULL) { terrno = TSDB_CODE_OUT_OF_MEMORY; return -1; @@ -970,9 +988,9 @@ int32_t syncNodeLogReplMgrInit(SSyncNode* pNode) { return 0; } -void syncNodeLogReplMgrDestroy(SSyncNode* pNode) { - for (int i = 0; i < TSDB_MAX_REPLICA; i++) { - syncLogReplMgrDestroy(pNode->logReplMgrs[i]); +void syncNodeLogReplDestroy(SSyncNode* pNode) { + for (int i = 0; i < TSDB_MAX_REPLICA + TSDB_MAX_LEARNER_REPLICA; i++) { + syncLogReplDestroy(pNode->logReplMgrs[i]); pNode->logReplMgrs[i] = NULL; } } @@ -1101,9 +1119,9 @@ int32_t syncLogBufferReset(SSyncLogBuffer* pBuf, SSyncNode* pNode) { pBuf->endIndex = pBuf->matchIndex + 1; // reset repl mgr - for (int i = 0; i < pNode->replicaNum; i++) { + for (int i = 0; i < pNode->totalReplicaNum; i++) { SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); } syncLogBufferValidate(pBuf); taosThreadMutexUnlock(&pBuf->mutex); @@ -1127,8 +1145,8 @@ SSyncRaftEntry* syncLogBufferGetOneEntry(SSyncLogBuffer* pBuf, SSyncNode* pNode, return pEntry; } -int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, - SRaftId* pDestId, bool* pBarrier) { +int32_t syncLogReplSendTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, SyncIndex index, SyncTerm* pTerm, SRaftId* pDestId, + bool* pBarrier) { SSyncRaftEntry* pEntry = NULL; SRpcMsg msgOut = {0}; bool inBuf = false; @@ -1141,16 +1159,16 @@ int32_t syncLogReplMgrReplicateOneTo(SSyncLogReplMgr* pMgr, SSyncNode* pNode, Sy if (terrno == TSDB_CODE_WAL_LOG_NOT_EXIST) { SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); if (pMgr) { - sInfo("vgId:%d, reset sync log repl mgr of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, - pDestId->addr, terrstr(), index); - (void)syncLogReplMgrReset(pMgr); + sInfo("vgId:%d, reset sync log repl of peer:%" PRIx64 " since %s. index:%" PRId64, pNode->vgId, pDestId->addr, + terrstr(), index); + (void)syncLogReplReset(pMgr); } } goto _err; } - *pBarrier = syncLogIsReplicationBarrier(pEntry); + *pBarrier = syncLogReplBarrier(pEntry); - prevLogTerm = syncLogReplMgrGetPrevLogTerm(pMgr, pNode, index); + prevLogTerm = syncLogReplGetPrevLogTerm(pMgr, pNode, index); if (prevLogTerm < 0) { sError("vgId:%d, failed to get prev log term since %s. index:%" PRId64 "", pNode->vgId, terrstr(), index); goto _err; diff --git a/source/libs/sync/src/syncRaftCfg.c b/source/libs/sync/src/syncRaftCfg.c index f780e255ce77eb485ffcd1de8d078718dc1973a5..480ed4b2afab84ee0ae4e281cbd3a01d504f22c8 100644 --- a/source/libs/sync/src/syncRaftCfg.c +++ b/source/libs/sync/src/syncRaftCfg.c @@ -18,6 +18,28 @@ #include "syncUtil.h" #include "tjson.h" +const char* syncRoleToStr(ESyncRole role) { + switch (role) { + case TAOS_SYNC_ROLE_VOTER: + return "true"; + case TAOS_SYNC_ROLE_LEARNER: + return "false"; + default: + return "unknown"; + } +} + +const ESyncRole syncStrToRole(char* str) { + if(strcmp(str, "true") == 0){ + return TAOS_SYNC_ROLE_VOTER; + } + if(strcmp(str, "false") == 0){ + return TAOS_SYNC_ROLE_LEARNER; + } + + return TAOS_SYNC_ROLE_ERROR; +} + static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { SSyncCfg *pCfg = (SSyncCfg *)pObj; if (tjsonAddDoubleToObject(pJson, "replicaNum", pCfg->replicaNum) < 0) return -1; @@ -26,13 +48,14 @@ static int32_t syncEncodeSyncCfg(const void *pObj, SJson *pJson) { SJson *nodeInfo = tjsonCreateArray(); if (nodeInfo == NULL) return -1; if (tjsonAddItemToObject(pJson, "nodeInfo", nodeInfo) < 0) return -1; - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonCreateObject(); if (info == NULL) return -1; if (tjsonAddDoubleToObject(info, "nodePort", pCfg->nodeInfo[i].nodePort) < 0) return -1; if (tjsonAddStringToObject(info, "nodeFqdn", pCfg->nodeInfo[i].nodeFqdn) < 0) return -1; if (tjsonAddIntegerToObject(info, "nodeId", pCfg->nodeInfo[i].nodeId) < 0) return -1; if (tjsonAddIntegerToObject(info, "clusterId", pCfg->nodeInfo[i].clusterId) < 0) return -1; + if (tjsonAddStringToObject(info, "isReplica", syncRoleToStr(pCfg->nodeInfo[i].nodeRole)) < 0) return -1; if (tjsonAddItemToArray(nodeInfo, info) < 0) return -1; } @@ -90,7 +113,8 @@ int32_t syncWriteCfgFile(SSyncNode *pNode) { if (taosRenameFile(file, realfile) != 0) goto _OVER; code = 0; - sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d", pNode->vgId, realfile, len); + sInfo("vgId:%d, succeed to write sync cfg file:%s, len:%d, lastConfigIndex:%" PRId64, pNode->vgId, + realfile, len, pNode->raftCfg.lastConfigIndex); _OVER: if (pJson != NULL) tjsonDelete(pJson); @@ -115,9 +139,9 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { SJson *nodeInfo = tjsonGetObjectItem(pJson, "nodeInfo"); if (nodeInfo == NULL) return -1; - pCfg->replicaNum = tjsonGetArraySize(nodeInfo); + pCfg->totalReplicaNum = tjsonGetArraySize(nodeInfo); - for (int32_t i = 0; i < pCfg->replicaNum; ++i) { + for (int32_t i = 0; i < pCfg->totalReplicaNum; ++i) { SJson *info = tjsonGetArrayItem(nodeInfo, i); if (info == NULL) return -1; tjsonGetUInt16ValueFromDouble(info, "nodePort", pCfg->nodeInfo[i].nodePort, code); @@ -126,6 +150,15 @@ static int32_t syncDecodeSyncCfg(const SJson *pJson, void *pObj) { if (code < 0) return -1; tjsonGetNumberValue(info, "nodeId", pCfg->nodeInfo[i].nodeId, code); tjsonGetNumberValue(info, "clusterId", pCfg->nodeInfo[i].clusterId, code); + char role[10] = {0}; + code = tjsonGetStringValue(info, "isReplica", role); + if(code < 0) return -1; + if(strlen(role) != 0){ + pCfg->nodeInfo[i].nodeRole = syncStrToRole(role); + } + else{ + pCfg->nodeInfo[i].nodeRole = TAOS_SYNC_ROLE_VOTER; + } } return 0; diff --git a/source/libs/sync/src/syncReplication.c b/source/libs/sync/src/syncReplication.c index 1d94b288d3a337d23efa5324a32df5f1ca564821..77e73402fae7f75b8b2240a58d2ed71f48b19e0e 100644 --- a/source/libs/sync/src/syncReplication.c +++ b/source/libs/sync/src/syncReplication.c @@ -52,7 +52,7 @@ int32_t syncNodeReplicateReset(SSyncNode* pNode, SRaftId* pDestId) { SSyncLogBuffer* pBuf = pNode->pLogBuf; taosThreadMutexLock(&pBuf->mutex); SSyncLogReplMgr* pMgr = syncNodeGetLogReplMgr(pNode, pDestId); - syncLogReplMgrReset(pMgr); + syncLogReplReset(pMgr); taosThreadMutexUnlock(&pBuf->mutex); return 0; } @@ -66,15 +66,15 @@ int32_t syncNodeReplicate(SSyncNode* pNode) { } int32_t syncNodeReplicateWithoutLock(SSyncNode* pNode) { - if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->replicaNum == 1) { + if (pNode->state != TAOS_SYNC_STATE_LEADER || pNode->raftCfg.cfg.totalReplicaNum == 1) { return -1; } - for (int32_t i = 0; i < pNode->replicaNum; i++) { + for (int32_t i = 0; i < pNode->totalReplicaNum; i++) { if (syncUtilSameId(&pNode->replicasId[i], &pNode->myRaftId)) { continue; } SSyncLogReplMgr* pMgr = pNode->logReplMgrs[i]; - (void)syncLogReplMgrReplicateOnce(pMgr, pNode); + (void)syncLogReplDoOnce(pMgr, pNode); } return 0; } diff --git a/source/libs/sync/src/syncSnapshot.c b/source/libs/sync/src/syncSnapshot.c index a9d0ddad17d99f929c51ef60e313d2df463377e3..763d4ec5d6b0fa8015d841231fc094d96661fadc 100644 --- a/source/libs/sync/src/syncSnapshot.c +++ b/source/libs/sync/src/syncSnapshot.c @@ -795,13 +795,18 @@ int32_t syncNodeOnSnapshot(SSyncNode *pSyncNode, const SRpcMsg *pRpcMsg) { return -1; } - if (pMsg->term > raftStoreGetTerm(pSyncNode)) { - syncNodeStepDown(pSyncNode, pMsg->term); + if(pSyncNode->raftCfg.cfg.nodeInfo[pSyncNode->raftCfg.cfg.myIndex].nodeRole != TAOS_SYNC_ROLE_LEARNER){ + if (pMsg->term > raftStoreGetTerm(pSyncNode)) { + syncNodeStepDown(pSyncNode, pMsg->term); + } + } + else{ + syncNodeUpdateTermWithoutStepDown(pSyncNode, pMsg->term); } // state, term, seq/ack int32_t code = 0; - if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER) { + if (pSyncNode->state == TAOS_SYNC_STATE_FOLLOWER || pSyncNode->state == TAOS_SYNC_STATE_LEARNER) { if (pMsg->term == raftStoreGetTerm(pSyncNode)) { if (pMsg->seq == SYNC_SNAPSHOT_SEQ_PREP_SNAPSHOT) { syncLogRecvSyncSnapshotSend(pSyncNode, pMsg, "process seq pre-snapshot"); diff --git a/source/libs/sync/src/syncTimeout.c b/source/libs/sync/src/syncTimeout.c index a27be2853e512d73ada59bc224ce028ae208e0cb..5ee67da9ab8d2cef07b7d85ef275199e8cf2ef0e 100644 --- a/source/libs/sync/src/syncTimeout.c +++ b/source/libs/sync/src/syncTimeout.c @@ -58,7 +58,7 @@ static void syncNodeCleanConfigIndex(SSyncNode* ths) { static int32_t syncNodeTimerRoutine(SSyncNode* ths) { ths->tmrRoutineNum++; - if (ths->tmrRoutineNum % 60 == 0 && ths->replicaNum > 1) { + if (ths->tmrRoutineNum % 60 == 0 && ths->totalReplicaNum > 1) { sNInfo(ths, "timer routines"); } else { sNTrace(ths, "timer routines"); diff --git a/source/libs/sync/src/syncUtil.c b/source/libs/sync/src/syncUtil.c index a519c76cda53a5f66faeaefa17f56053c5d083e6..cf796c386297271d1138189ea812fe4b1fbe984c 100644 --- a/source/libs/sync/src/syncUtil.c +++ b/source/libs/sync/src/syncUtil.c @@ -125,7 +125,7 @@ static void syncLogBufferStates2Str(SSyncNode* pSyncNode, char* buf, int32_t buf pBuf->commitIndex, pBuf->matchIndex, pBuf->endIndex); } -static void syncLogReplMgrStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { +static void syncLogReplStates2Str(SSyncNode* pSyncNode, char* buf, int32_t bufLen) { int len = 0; len += snprintf(buf + len, bufLen - len, "%s", "{"); for (int32_t i = 0; i < pSyncNode->replicaNum; i++) { @@ -178,7 +178,7 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo syncCfg2SimpleStr(&pNode->raftCfg.cfg, cfgStr, sizeof(cfgStr)); char replMgrStatesStr[1024] = ""; - syncLogReplMgrStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); + syncLogReplStates2Str(pNode, replMgrStatesStr, sizeof(replMgrStatesStr)); char bufferStatesStr[256] = ""; syncLogBufferStates2Str(pNode, bufferStatesStr, sizeof(bufferStatesStr)); @@ -202,21 +202,22 @@ void syncPrintNodeLog(const char* flags, ELogLevel level, int32_t dflag, SSyncNo // restore error code terrno = errCode; - + SyncIndex appliedIndex = pNode->pFsm->FpAppliedIndexCb(pNode->pFsm); + if (pNode != NULL) { taosPrintLog(flags, level, dflag, - "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", first-ver:%" PRId64 - ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 + "vgId:%d, %s, sync:%s, term:%" PRIu64 ", commit-index:%" PRId64 ", applied-index:%" PRId64 + ", first-ver:%" PRId64 ", last-ver:%" PRId64 ", min:%" PRId64 ", snap:%" PRId64 ", snap-term:%" PRIu64 ", elect-times:%d, as-leader-times:%d, cfg-ch-times:%d, hb-slow:%d, hbr-slow:%d, " "aq-items:%d, snaping:%" PRId64 ", replicas:%d, last-cfg:%" PRId64 ", chging:%d, restore:%d, quorum:%d, elect-lc-timer:%" PRId64 ", hb:%" PRId64 ", buffer:%s, repl-mgrs:%s, members:%s, hb:%s, hb-reply:%s", - pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, logBeginIndex, - logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, pNode->electNum, - pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, aqItems, - pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, - pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, pNode->heartbeatTimerLogicClockUser, - bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); + pNode->vgId, eventLog, syncStr(pNode->state), currentTerm, pNode->commitIndex, appliedIndex, + logBeginIndex, logLastIndex, pNode->minMatchIndex, snapshot.lastApplyIndex, snapshot.lastApplyTerm, + pNode->electNum, pNode->becomeLeaderNum, pNode->configChangeNum, pNode->hbSlowNum, pNode->hbrSlowNum, + aqItems, pNode->snapshottingIndex, pNode->replicaNum, pNode->raftCfg.lastConfigIndex, pNode->changing, + pNode->restoreFinish, syncNodeDynamicQuorum(pNode), pNode->electTimerLogicClock, + pNode->heartbeatTimerLogicClockUser, bufferStatesStr, replMgrStatesStr, cfgStr, hbTimeStr, hbrTimeStr); } } diff --git a/source/libs/sync/src/syncVoteMgr.c b/source/libs/sync/src/syncVoteMgr.c index 6bd9625276c898184341cec291abed6c304700fb..83b0dde8e0c77b5c199a72d4a28bda64108b0810 100644 --- a/source/libs/sync/src/syncVoteMgr.c +++ b/source/libs/sync/src/syncVoteMgr.c @@ -30,7 +30,7 @@ SVotesGranted *voteGrantedCreate(SSyncNode *pNode) { return NULL; } - pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicas = (void*)&pNode->replicasId; pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); @@ -49,7 +49,7 @@ void voteGrantedDestroy(SVotesGranted *pVotesGranted) { } void voteGrantedUpdate(SVotesGranted *pVotesGranted, SSyncNode *pNode) { - pVotesGranted->replicas = &pNode->replicasId; + pVotesGranted->replicas = (void*)&pNode->replicasId; pVotesGranted->replicaNum = pNode->replicaNum; voteGrantedClearVotes(pVotesGranted); @@ -115,7 +115,7 @@ SVotesRespond *votesRespondCreate(SSyncNode *pNode) { return NULL; } - pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicas = (void*)&pNode->replicasId; pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; pVotesRespond->pNode = pNode; @@ -130,7 +130,7 @@ void votesRespondDestory(SVotesRespond *pVotesRespond) { } void votesRespondUpdate(SVotesRespond *pVotesRespond, SSyncNode *pNode) { - pVotesRespond->replicas = &pNode->replicasId; + pVotesRespond->replicas = (void*)&pNode->replicasId; pVotesRespond->replicaNum = pNode->replicaNum; pVotesRespond->term = 0; pVotesRespond->pNode = pNode; diff --git a/source/libs/transport/src/transCli.c b/source/libs/transport/src/transCli.c index 50ed9fa61b181ae02fa934da1108a7e7fc130123..c23d6d0a1f42fc46f3b52b7bba8c1ca21f8e4f39 100644 --- a/source/libs/transport/src/transCli.c +++ b/source/libs/transport/src/transCli.c @@ -462,6 +462,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { if (transQueueEmpty(&pConn->cliMsgs)) { if (pConn->broken == true && CONN_NO_PERSIST_BY_APP(pConn)) { tTrace("%s conn %p handle except, persist:0", CONN_GET_INST_LABEL(pConn), pConn); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); return; } @@ -521,6 +522,7 @@ void cliHandleExceptImpl(SCliConn* pConn, int32_t code) { destroyCmsg(pMsg); tTrace("%s conn %p start to destroy, ref:%d", CONN_GET_INST_LABEL(pConn), pConn, T_REF_VAL_GET(pConn)); } while (!transQueueEmpty(&pConn->cliMsgs)); + if (T_REF_VAL_GET(pConn) > 1) transUnrefCliHandle(pConn); transUnrefCliHandle(pConn); } void cliHandleExcept(SCliConn* conn) { diff --git a/source/libs/wal/src/walRead.c b/source/libs/wal/src/walRead.c index 3e1e36ccc1e7ac862aaad36ebc4d4a4c0a6a2c62..19694a6126ee2dc758e49495827069814d9620dd 100644 --- a/source/libs/wal/src/walRead.c +++ b/source/libs/wal/src/walRead.c @@ -33,7 +33,6 @@ SWalReader *walOpenReader(SWal *pWal, SWalFilterCond *cond) { pReader->pLogFile = NULL; pReader->curVersion = -1; pReader->curFileFirstVer = -1; - pReader->curInvalid = 1; pReader->capacity = 0; if (cond) { pReader->cond = *cond; @@ -81,7 +80,6 @@ int32_t walNextValidMsg(SWalReader *pReader) { wDebug("vgId:%d, wal start to fetch, index:%" PRId64 ", last index:%" PRId64 " commit index:%" PRId64 ", applied index:%" PRId64 ", end index:%" PRId64, pReader->pWal->cfg.vgId, fetchVer, lastVer, committedVer, appliedVer, endVer); - pReader->curStopped = 0; while (fetchVer <= endVer) { if (walFetchHeadNew(pReader, fetchVer) < 0) { return -1; @@ -99,10 +97,18 @@ int32_t walNextValidMsg(SWalReader *pReader) { fetchVer = pReader->curVersion; } } - pReader->curStopped = 1; return -1; } +int64_t walReaderGetCurrentVer(const SWalReader *pReader) { return pReader->curVersion; } + +void walReaderValidVersionRange(SWalReader *pReader, int64_t *sver, int64_t *ever) { + *sver = walGetFirstVer(pReader->pWal); + int64_t lastVer = walGetLastVer(pReader->pWal); + int64_t committedVer = walGetCommittedVer(pReader->pWal); + *ever = pReader->cond.scanUncommited ? lastVer : committedVer; +} + static int64_t walReadSeekFilePos(SWalReader *pReader, int64_t fileFirstVer, int64_t ver) { int64_t ret = 0; @@ -196,32 +202,26 @@ int32_t walReadSeekVerImpl(SWalReader *pReader, int64_t ver) { return -1; } - wDebug("vgId:%d, wal version reset from %" PRId64 "(invalid:%d) to %" PRId64, pReader->pWal->cfg.vgId, - pReader->curVersion, pReader->curInvalid, ver); + wDebug("vgId:%d, wal version reset from %" PRId64 " to %" PRId64, pReader->pWal->cfg.vgId, + pReader->curVersion, ver); pReader->curVersion = ver; - pReader->curInvalid = 0; return 0; } int32_t walReadSeekVer(SWalReader *pReader, int64_t ver) { SWal *pWal = pReader->pWal; - if (!pReader->curInvalid && ver == pReader->curVersion) { + if (ver == pReader->curVersion) { wDebug("vgId:%d, wal index:%" PRId64 " match, no need to reset", pReader->pWal->cfg.vgId, ver); return 0; } -// pReader->curInvalid = 1; -// pReader->curVersion = ver; - if (ver > pWal->vers.lastVer || ver < pWal->vers.firstVer) { - wDebug("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, + wInfo("vgId:%d, invalid index:%" PRId64 ", first index:%" PRId64 ", last index:%" PRId64, pReader->pWal->cfg.vgId, ver, pWal->vers.firstVer, pWal->vers.lastVer); terrno = TSDB_CODE_WAL_LOG_NOT_EXIST; return -1; } -// if (ver < pWal->vers.snapshotVer) { -// } if (walReadSeekVerImpl(pReader, ver) < 0) { return -1; @@ -238,10 +238,8 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { wDebug("vgId:%d, wal starts to fetch head, index:%" PRId64, pRead->pWal->cfg.vgId, fetchVer); - if (pRead->curInvalid || pRead->curVersion != fetchVer) { + if (pRead->curVersion != fetchVer) { if (walReadSeekVer(pRead, fetchVer) < 0) { -// pRead->curVersion = fetchVer; -// pRead->curInvalid = 1; return -1; } seeked = true; @@ -260,7 +258,6 @@ static int32_t walFetchHeadNew(SWalReader *pRead, int64_t fetchVer) { } else { terrno = TSDB_CODE_WAL_FILE_CORRUPTED; } -// pRead->curInvalid = 1; return -1; } } @@ -344,7 +341,7 @@ int32_t walFetchHead(SWalReader *pRead, int64_t ver, SWalCkHead *pHead) { return -1; } - if (pRead->curInvalid || pRead->curVersion != ver) { + if (pRead->curVersion != ver) { code = walReadSeekVer(pRead, ver); if (code < 0) { // pRead->curVersion = ver; @@ -479,7 +476,7 @@ int32_t walReadVer(SWalReader *pReader, int64_t ver) { taosThreadMutexLock(&pReader->mutex); - if (pReader->curInvalid || pReader->curVersion != ver) { + if (pReader->curVersion != ver) { if (walReadSeekVer(pReader, ver) < 0) { wError("vgId:%d, unexpected wal log, index:%" PRId64 ", since %s", pReader->pWal->cfg.vgId, ver, terrstr()); taosThreadMutexUnlock(&pReader->mutex); @@ -575,7 +572,6 @@ void walReadReset(SWalReader *pReader) { taosThreadMutexLock(&pReader->mutex); taosCloseFile(&pReader->pIdxFile); taosCloseFile(&pReader->pLogFile); - pReader->curInvalid = 1; pReader->curFileFirstVer = -1; pReader->curVersion = -1; taosThreadMutexUnlock(&pReader->mutex); diff --git a/source/util/src/tconfig.c b/source/util/src/tconfig.c index f811d2f20316e2aab87cea030ead08613b19d77c..288ea6052b580909198788f528871633048e19f1 100644 --- a/source/util/src/tconfig.c +++ b/source/util/src/tconfig.c @@ -187,7 +187,7 @@ static int32_t cfgSetInt32(SConfigItem *pItem, const char *value, ECfgSrcType st } static int32_t cfgSetInt64(SConfigItem *pItem, const char *value, ECfgSrcType stype) { - int64_t ival = (int64_t)atoi(value); + int64_t ival = (int64_t)atoll(value); if (ival < pItem->imin || ival > pItem->imax) { uError("cfg:%s, type:%s src:%s value:%" PRId64 " out of range[%" PRId64 ", %" PRId64 "]", pItem->name, cfgDtypeStr(pItem->dtype), cfgStypeStr(stype), ival, pItem->imin, pItem->imax); diff --git a/source/util/src/terror.c b/source/util/src/terror.c index 2ea43e2cdea0c3f38659082b73d171c924112f0b..6fc16ad4a98b517f4b5c37c48128e1b7d46248a0 100644 --- a/source/util/src/terror.c +++ b/source/util/src/terror.c @@ -98,7 +98,8 @@ TAOS_DEFINE_ERROR(TSDB_CODE_NO_ENOUGH_DISKSPACE, "No enough disk space" TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STARTING, "Database is starting up") TAOS_DEFINE_ERROR(TSDB_CODE_APP_IS_STOPPING, "Database is closing down") -TAOS_DEFINE_ERROR(TSDB_CODE_IVLD_DATA_FMT, "Invalid data format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_DATA_FMT, "Invalid data format") +TAOS_DEFINE_ERROR(TSDB_CODE_INVALID_CFG_VALUE, "Invalid configuration value") //client TAOS_DEFINE_ERROR(TSDB_CODE_TSC_INVALID_OPERATION, "Invalid operation") @@ -190,6 +191,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_MND_NO_USER_FROM_CONN, "Can not get user from TAOS_DEFINE_ERROR(TSDB_CODE_MND_TOO_MANY_USERS, "Too many users") TAOS_DEFINE_ERROR(TSDB_CODE_MND_INVALID_ALTER_OPER, "Invalid alter operation") TAOS_DEFINE_ERROR(TSDB_CODE_MND_AUTH_FAILURE, "Authentication failure") +TAOS_DEFINE_ERROR(TSDB_CODE_MND_PRIVILEDGE_EXIST, "User already have this priviledge") //mnode-stable-part1 TAOS_DEFINE_ERROR(TSDB_CODE_MND_STB_ALREADY_EXIST, "STable already exists") @@ -320,7 +322,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_ALREADY_DEPLOYED, "Snode already deploye TAOS_DEFINE_ERROR(TSDB_CODE_SNODE_NOT_DEPLOYED, "Snode not deployed") // vnode -TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode moved to another dnode or was deleted") +TAOS_DEFINE_ERROR(TSDB_CODE_VND_INVALID_VGROUP_ID, "Vnode is closed or removed") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NO_WRITE_AUTH, "Database write operation denied") TAOS_DEFINE_ERROR(TSDB_CODE_VND_NOT_EXIST, "Vnode not exist") TAOS_DEFINE_ERROR(TSDB_CODE_VND_ALREADY_EXIST, "Vnode already exist") @@ -412,19 +414,14 @@ TAOS_DEFINE_ERROR(TSDB_CODE_GRANT_TABLE_LIMITED, "Table creation limite // sync TAOS_DEFINE_ERROR(TSDB_CODE_SYN_TIMEOUT, "Sync timeout") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_IS_LEADER, "Sync is leader") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_LEADER, "Sync leader is unreachable") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_ONE_REPLICA, "Sync one replica") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NOT_IN_NEW_CONFIG, "Sync not in new config") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEW_CONFIG_ERROR, "Sync new config error") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RECONFIG_NOT_READY, "Sync not ready for reconfig") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready for propose") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_STANDBY_NOT_READY, "Sync not ready for standby") -TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BATCH_ERROR, "Sync batch error") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_PROPOSE_NOT_READY, "Sync not ready to propose") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_RESTORING, "Sync leader is restoring") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INVALID_SNAPSHOT_MSG, "Sync invalid snapshot msg") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_BUFFER_FULL, "Sync buffer is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_WRITE_STALL, "Sync write stall") +TAOS_DEFINE_ERROR(TSDB_CODE_SYN_NEGOTIATION_WIN_FULL, "Sync negotiation win is full") TAOS_DEFINE_ERROR(TSDB_CODE_SYN_INTERNAL_ERROR, "Sync internal error") //tq @@ -442,8 +439,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_TQ_NO_COMMITTED_OFFSET, "TQ no committed offse // wal TAOS_DEFINE_ERROR(TSDB_CODE_WAL_FILE_CORRUPTED, "WAL file is corrupted") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_SIZE_LIMIT, "WAL size exceeds limit") -TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL use invalid version") +TAOS_DEFINE_ERROR(TSDB_CODE_WAL_INVALID_VER, "WAL invalid version") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_NOT_EXIST, "WAL log not exist") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_CHKSUM_MISMATCH, "WAL checksum mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_WAL_LOG_INCOMPLETE, "WAL log incomplete") @@ -579,11 +575,9 @@ TAOS_DEFINE_ERROR(TSDB_CODE_FUNC_DUP_TIMESTAMP, "Duplicate timestamps TAOS_DEFINE_ERROR(TSDB_CODE_UDF_STOPPING, "udf is stopping") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_READ_ERR, "udf pipe read error") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_CONNECT_ERR, "udf pipe connect error") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NO_PIPE, "udf no pipe") +TAOS_DEFINE_ERROR(TSDB_CODE_UDF_PIPE_NOT_EXIST, "udf pipe not exist") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_LOAD_UDF_FAILURE, "udf load failure") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_STATE, "udf invalid state") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_INPUT, "udf invalid function input") -TAOS_DEFINE_ERROR(TSDB_CODE_UDF_NO_FUNC_HANDLE, "udf no function handle") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_BUFSIZE, "udf invalid bufsize") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_INVALID_OUTPUT_TYPE, "udf invalid output type") TAOS_DEFINE_ERROR(TSDB_CODE_UDF_SCRIPT_NOT_SUPPORTED, "udf program language not supported") @@ -631,6 +625,7 @@ TAOS_DEFINE_ERROR(TSDB_CODE_INDEX_INVALID_FILE, "Index file is inval TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_INVALID_MSG, "Invalid message") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_MISMATCH, "Consumer mismatch") TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_CLOSED, "Consumer closed") +TAOS_DEFINE_ERROR(TSDB_CODE_TMQ_CONSUMER_ERROR, "Consumer error, to see log") // stream TAOS_DEFINE_ERROR(TSDB_CODE_STREAM_TASK_NOT_EXIST, "Stream task not exist") diff --git a/source/util/src/tlog.c b/source/util/src/tlog.c index a3d3c399abcd6040cf042dec045024549e2d3179..baf6a9f319e013af20fa00cb8e23dc3dfc900afd 100644 --- a/source/util/src/tlog.c +++ b/source/util/src/tlog.c @@ -24,10 +24,11 @@ #define LOG_MAX_LINE_SIZE (10024) #define LOG_MAX_LINE_BUFFER_SIZE (LOG_MAX_LINE_SIZE + 3) #define LOG_MAX_LINE_DUMP_SIZE (1024 * 1024) -#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 3) +#define LOG_MAX_LINE_DUMP_BUFFER_SIZE (LOG_MAX_LINE_DUMP_SIZE + 128) #define LOG_FILE_NAME_LEN 300 #define LOG_DEFAULT_BUF_SIZE (20 * 1024 * 1024) // 20MB +#define LOG_SLOW_BUF_SIZE (10 * 1024 * 1024) // 10MB #define LOG_DEFAULT_INTERVAL 25 #define LOG_INTERVAL_STEP 5 @@ -51,6 +52,8 @@ typedef struct { int32_t stop; TdThread asyncThread; TdThreadMutex buffMutex; + int32_t writeInterval; + int32_t lastDuration; } SLogBuff; typedef struct { @@ -62,6 +65,7 @@ typedef struct { pid_t pid; char logName[LOG_FILE_NAME_LEN]; SLogBuff *logHandle; + SLogBuff *slowHandle; TdThreadMutex logMutex; } SLogObj; @@ -69,7 +73,6 @@ extern SConfig *tsCfg; static int8_t tsLogInited = 0; static SLogObj tsLogObj = {.fileNum = 1}; static int64_t tsAsyncLogLostLines = 0; -static int32_t tsWriteInterval = LOG_DEFAULT_INTERVAL; static int32_t tsDaylightActive; /* Currently in daylight saving time. */ bool tsLogEmbedded = 0; @@ -82,6 +85,7 @@ int64_t tsNumOfErrorLogs = 0; int64_t tsNumOfInfoLogs = 0; int64_t tsNumOfDebugLogs = 0; int64_t tsNumOfTraceLogs = 0; +int64_t tsNumOfSlowLogs = 0; // log int32_t dDebugFlag = 131; @@ -136,6 +140,34 @@ static int32_t taosStartLog() { return 0; } +int32_t taosInitSlowLog() { + char fullName[PATH_MAX] = {0}; + char logFileName[64] = {0}; +#ifdef CUS_PROMPT + snprintf(logFileName, 64, "%sSlowLog", CUS_PROMPT); +#else + snprintf(logFileName, 64, "taosSlowLog"); +#endif + + if (strlen(tsLogDir) != 0) { + snprintf(fullName, PATH_MAX, "%s" TD_DIRSEP "%s", tsLogDir, logFileName); + } else { + snprintf(fullName, PATH_MAX, "%s", logFileName); + } + + tsLogObj.slowHandle = taosLogBuffNew(LOG_SLOW_BUF_SIZE); + if (tsLogObj.slowHandle == NULL) return -1; + + taosUmaskFile(0); + tsLogObj.slowHandle->pFile = taosOpenFile(fullName, TD_FILE_CREATE | TD_FILE_WRITE | TD_FILE_APPEND); + if (tsLogObj.slowHandle->pFile == NULL) { + printf("\nfailed to open slow log file:%s, reason:%s\n", fullName, strerror(errno)); + return -1; + } + + return 0; +} + int32_t taosInitLog(const char *logName, int32_t maxFiles) { if (atomic_val_compare_exchange_8(&tsLogInited, 0, 1) != 0) return 0; osUpdate(); @@ -151,6 +183,8 @@ int32_t taosInitLog(const char *logName, int32_t maxFiles) { tsLogObj.logHandle = taosLogBuffNew(LOG_DEFAULT_BUF_SIZE); if (tsLogObj.logHandle == NULL) return -1; if (taosOpenLogFile(fullName, tsNumOfLogLines, maxFiles) < 0) return -1; + + if (taosInitSlowLog() < 0) return -1; if (taosStartLog() < 0) return -1; return 0; } @@ -159,25 +193,34 @@ static void taosStopLog() { if (tsLogObj.logHandle) { tsLogObj.logHandle->stop = 1; } + if (tsLogObj.slowHandle) { + tsLogObj.slowHandle->stop = 1; + } } void taosCloseLog() { + taosStopLog(); + + if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { + taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); + taosThreadClear(&tsLogObj.logHandle->asyncThread); + } + + if (tsLogObj.slowHandle != NULL) { + taosThreadMutexDestroy(&tsLogObj.slowHandle->buffMutex); + taosCloseFile(&tsLogObj.slowHandle->pFile); + taosMemoryFreeClear(tsLogObj.slowHandle->buffer); + taosMemoryFreeClear(tsLogObj.slowHandle); + } + if (tsLogObj.logHandle != NULL) { - taosStopLog(); - if (tsLogObj.logHandle != NULL && taosCheckPthreadValid(tsLogObj.logHandle->asyncThread)) { - taosThreadJoin(tsLogObj.logHandle->asyncThread, NULL); - taosThreadClear(&tsLogObj.logHandle->asyncThread); - } tsLogInited = 0; taosThreadMutexDestroy(&tsLogObj.logHandle->buffMutex); taosCloseFile(&tsLogObj.logHandle->pFile); taosMemoryFreeClear(tsLogObj.logHandle->buffer); - memset(&tsLogObj.logHandle->buffer, 0, sizeof(tsLogObj.logHandle->buffer)); taosThreadMutexDestroy(&tsLogObj.logMutex); taosMemoryFreeClear(tsLogObj.logHandle); - memset(&tsLogObj.logHandle, 0, sizeof(tsLogObj.logHandle)); - tsLogObj.logHandle = NULL; } } @@ -513,10 +556,9 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons va_list argpointer; va_start(argpointer, format); - len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - len, format, argpointer); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 - len, format, argpointer); va_end(argpointer); - if (len > LOG_MAX_LINE_DUMP_SIZE) len = LOG_MAX_LINE_DUMP_SIZE; buffer[len++] = '\n'; buffer[len] = 0; @@ -524,6 +566,31 @@ void taosPrintLongString(const char *flags, ELogLevel level, int32_t dflag, cons taosMemoryFree(buffer); } +void taosPrintSlowLog(const char *format, ...) { + if (!osLogSpaceAvailable()) return; + + char *buffer = taosMemoryMalloc(LOG_MAX_LINE_DUMP_BUFFER_SIZE); + int32_t len = taosBuildLogHead(buffer, ""); + + va_list argpointer; + va_start(argpointer, format); + len += vsnprintf(buffer + len, LOG_MAX_LINE_DUMP_BUFFER_SIZE - 2 - len, format, argpointer); + va_end(argpointer); + + buffer[len++] = '\n'; + buffer[len] = 0; + + atomic_add_fetch_64(&tsNumOfSlowLogs, 1); + + if (tsAsyncLog) { + taosPushLogBuffer(tsLogObj.slowHandle, buffer, len); + } else { + taosWriteFile(tsLogObj.slowHandle->pFile, buffer, len); + } + + taosMemoryFree(buffer); +} + void taosDumpData(unsigned char *msg, int32_t len) { if (!osLogSpaceAvailable()) return; taosUpdateLogNums(DEBUG_DUMP); @@ -568,6 +635,7 @@ static SLogBuff *taosLogBuffNew(int32_t bufSize) { LOG_BUF_SIZE(pLogBuf) = bufSize; pLogBuf->minBuffSize = bufSize / 10; pLogBuf->stop = 0; + pLogBuf->writeInterval = LOG_DEFAULT_INTERVAL; if (taosThreadMutexInit(&LOG_BUF_MUTEX(pLogBuf), NULL) < 0) goto _err; // tsem_init(&(pLogBuf->buffNotEmpty), 0, 0); @@ -651,83 +719,78 @@ static int32_t taosGetLogRemainSize(SLogBuff *pLogBuf, int32_t start, int32_t en } static void taosWriteLog(SLogBuff *pLogBuf) { - static int32_t lastDuration = 0; - int32_t remainChecked = 0; - int32_t start, end, pollSize; - - do { - if (remainChecked == 0) { - start = LOG_BUF_START(pLogBuf); - end = LOG_BUF_END(pLogBuf); - - if (start == end) { - dbgEmptyW++; - tsWriteInterval = LOG_MAX_INTERVAL; - return; - } + int32_t start = LOG_BUF_START(pLogBuf); + int32_t end = LOG_BUF_END(pLogBuf); - pollSize = taosGetLogRemainSize(pLogBuf, start, end); - if (pollSize < pLogBuf->minBuffSize) { - lastDuration += tsWriteInterval; - if (lastDuration < LOG_MAX_WAIT_MSEC) { - break; - } - } + if (start == end) { + dbgEmptyW++; + pLogBuf->writeInterval = LOG_MAX_INTERVAL; + return; + } - lastDuration = 0; + int32_t pollSize = taosGetLogRemainSize(pLogBuf, start, end); + if (pollSize < pLogBuf->minBuffSize) { + pLogBuf->lastDuration += pLogBuf->writeInterval; + if (pLogBuf->lastDuration < LOG_MAX_WAIT_MSEC) { + return; } + } - if (start < end) { - taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize); - } else { - int32_t tsize = LOG_BUF_SIZE(pLogBuf) - start; - taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize); + pLogBuf->lastDuration = 0; - taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end); - } + if (start < end) { + taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, pollSize); + } else { + int32_t tsize = LOG_BUF_SIZE(pLogBuf) - start; + taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf) + start, tsize); - dbgWN++; - dbgWSize += pollSize; + taosWriteFile(pLogBuf->pFile, LOG_BUF_BUFFER(pLogBuf), end); + } - if (pollSize < pLogBuf->minBuffSize) { - dbgSmallWN++; - if (tsWriteInterval < LOG_MAX_INTERVAL) { - tsWriteInterval += LOG_INTERVAL_STEP; - } - } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 3) { - dbgBigWN++; - tsWriteInterval = LOG_MIN_INTERVAL; - } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 4) { - if (tsWriteInterval > LOG_MIN_INTERVAL) { - tsWriteInterval -= LOG_INTERVAL_STEP; - } - } + dbgWN++; + dbgWSize += pollSize; - LOG_BUF_START(pLogBuf) = (LOG_BUF_START(pLogBuf) + pollSize) % LOG_BUF_SIZE(pLogBuf); + if (pollSize < pLogBuf->minBuffSize) { + dbgSmallWN++; + if (pLogBuf->writeInterval < LOG_MAX_INTERVAL) { + pLogBuf->writeInterval += LOG_INTERVAL_STEP; + } + } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 3) { + dbgBigWN++; + pLogBuf->writeInterval = LOG_MIN_INTERVAL; + } else if (pollSize > LOG_BUF_SIZE(pLogBuf) / 4) { + if (pLogBuf->writeInterval > LOG_MIN_INTERVAL) { + pLogBuf->writeInterval -= LOG_INTERVAL_STEP; + } + } - start = LOG_BUF_START(pLogBuf); - end = LOG_BUF_END(pLogBuf); + LOG_BUF_START(pLogBuf) = (LOG_BUF_START(pLogBuf) + pollSize) % LOG_BUF_SIZE(pLogBuf); - pollSize = taosGetLogRemainSize(pLogBuf, start, end); - if (pollSize < pLogBuf->minBuffSize) { - break; - } + start = LOG_BUF_START(pLogBuf); + end = LOG_BUF_END(pLogBuf); - tsWriteInterval = LOG_MIN_INTERVAL; + pollSize = taosGetLogRemainSize(pLogBuf, start, end); + if (pollSize < pLogBuf->minBuffSize) { + return; + } - remainChecked = 1; - } while (1); + pLogBuf->writeInterval = 0; } static void *taosAsyncOutputLog(void *param) { - SLogBuff *pLogBuf = (SLogBuff *)param; + SLogBuff *pLogBuf = (SLogBuff *)tsLogObj.logHandle; + SLogBuff *pSlowBuf = (SLogBuff *)tsLogObj.slowHandle; + setThreadName("log"); int32_t count = 0; int32_t updateCron = 0; + int32_t writeInterval = 0; + while (1) { - count += tsWriteInterval; + writeInterval = TMIN(pLogBuf->writeInterval, pSlowBuf->writeInterval); + count += writeInterval; updateCron++; - taosMsleep(tsWriteInterval); + taosMsleep(writeInterval); if (count > 1000) { osUpdate(); count = 0; @@ -735,13 +798,14 @@ static void *taosAsyncOutputLog(void *param) { // Polling the buffer taosWriteLog(pLogBuf); + taosWriteLog(pSlowBuf); if (updateCron >= 3600 * 24 * 40 / 2) { taosUpdateDaylight(); updateCron = 0; } - if (pLogBuf->stop) break; + if (pLogBuf->stop || pSlowBuf->stop) break; } return NULL; diff --git a/source/util/src/tworker.c b/source/util/src/tworker.c index 631bcb443ebecdc89ab3338d5dd5254964232093..6edee27c052d474355812a4b46d53331a5ab6324 100644 --- a/source/util/src/tworker.c +++ b/source/util/src/tworker.c @@ -217,7 +217,7 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem int32_t queueNum = taosGetQueueNumber(pool->qset); int32_t curWorkerNum = taosArrayGetSize(pool->workers); - int32_t dstWorkerNum = ceil(queueNum * pool->ratio); + int32_t dstWorkerNum = ceilf(queueNum * pool->ratio); if (dstWorkerNum < 1) dstWorkerNum = 1; // spawn a thread to process queue @@ -248,7 +248,8 @@ STaosQueue *tAutoQWorkerAllocQueue(SAutoQWorkerPool *pool, void *ahandle, FItem } taosThreadAttrDestroy(&thAttr); - uInfo("worker:%s:%d is launched, total:%d", pool->name, worker->id, (int32_t)taosArrayGetSize(pool->workers)); + int32_t numOfThreads = taosArrayGetSize(pool->workers); + uInfo("worker:%s:%d is launched, total:%d, expect:%d", pool->name, worker->id, numOfThreads, dstWorkerNum); curWorkerNum++; } diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py index b06b0890ff2b5f671f4d347d1f5f77d305e2f367..35ed2982130bb7cb3cef31414a9225b3411aeb7c 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py +++ b/tests/develop-test/5-taos-tools/taosbenchmark/default_json.py @@ -59,9 +59,9 @@ class TDTestCase: os.system("%s" % cmd) tdSql.execute("reset query cache") tdSql.query("show db.tables") - tdSql.checkRows(10) + tdSql.checkRows(8) tdSql.query("select count(*) from db.stb") - tdSql.checkData(0, 0, 100) + tdSql.checkData(0, 0, 80) def stop(self): tdSql.close() diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json index da22ef75e2a648e897341338bb762e4fedc6662b..c841e90b5168afc11bcf8248a0f37c9ba16d30d0 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/default.json @@ -15,13 +15,18 @@ "num_of_records_per_req": 10, "databases": [{ "dbinfo": { - "name": "db" + "name": "db", + "drop": "yes" + }, "super_tables": [{ + "child_table_exists":"no", "name": "stb", "childtable_prefix": "stb_", "childtable_count": 10, "insert_rows": 10, + "childtable_from": 1, + "childtable_to": 9, "columns": [{"type": "INT"}], "tags": [{"type": "INT"}] }] diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json index 9b99521f52794b94558daef935a8aefbe57b5c6b..078688bb7e075753bf972d4275daa21369ae10fd 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/rest_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb3-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json index 7b87919a6dbebd5f4abc05bb1139f309f23fed6f..e808c9d3f3901b2ccaf63b315de164bb11a729cb 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/sml_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb4-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json index baf0384e46d51f802f7dd842b1b49925efd00abc..0054d985ee6e68012d7840b1cb088d5d73147bec 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/stmt_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -54,7 +54,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb2-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json index f683cc016b1e068c6ebf2933084977ffeb11966c..bed3598bfe8a32319968388b435ed0cd1a8a415a 100644 --- a/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json +++ b/tests/develop-test/5-taos-tools/taosbenchmark/json/taosc_auto_create_table.json @@ -11,6 +11,7 @@ "confirm_parameter_prompt": "no", "prepared_rand": 100, "chinese": "no", + "escape_character": "yes", "insert_interval": 0, "num_of_records_per_req": 10, "databases": [{ @@ -29,7 +30,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", @@ -55,7 +55,6 @@ "child_table_exists":"no", "childtable_count": 8, "childtable_prefix": "stb1-2_", - "escape_character": "yes", "auto_create_table": "yes", "batch_create_tbl_num": 10, "data_source": "rand", diff --git a/tests/parallel_test/cases.task b/tests/parallel_test/cases.task index 39415ed0bd666a5add74789f1f165782bb430742..9f9679ae353e471d38831057ed764bcc2cea6c1d 100644 --- a/tests/parallel_test/cases.task +++ b/tests/parallel_test/cases.task @@ -89,7 +89,7 @@ ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -# ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py +,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-1ctb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropStbCtb.py ,,y,system-test,./pytest.sh python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py @@ -785,6 +785,7 @@ ,,y,script,./test.sh -f tsim/insert/query_multi_file.sim ,,y,script,./test.sh -f tsim/insert/tcp.sim ,,y,script,./test.sh -f tsim/insert/update0.sim +,,y,script,./test.sh -f tsim/insert/delete0.sim ,,y,script,./test.sh -f tsim/insert/update1_sort_merge.sim ,,y,script,./test.sh -f tsim/insert/update2.sim ,,y,script,./test.sh -f tsim/parser/alter__for_community_version.sim @@ -878,6 +879,7 @@ ,,y,script,./test.sh -f tsim/query/udf_with_const.sim ,,y,script,./test.sh -f tsim/query/join_interval.sim ,,y,script,./test.sh -f tsim/query/unionall_as_table.sim +,,y,script,./test.sh -f tsim/query/multi_order_by.sim ,,y,script,./test.sh -f tsim/query/sys_tbname.sim ,,y,script,./test.sh -f tsim/query/groupby.sim ,,y,script,./test.sh -f tsim/query/event.sim @@ -885,6 +887,7 @@ ,,y,script,./test.sh -f tsim/query/emptyTsRange.sim ,,y,script,./test.sh -f tsim/query/partitionby.sim ,,y,script,./test.sh -f tsim/query/tableCount.sim +,,y,script,./test.sh -f tsim/query/tag_scan.sim ,,y,script,./test.sh -f tsim/query/nullColSma.sim ,,y,script,./test.sh -f tsim/qnode/basic1.sim ,,y,script,./test.sh -f tsim/snode/basic1.sim diff --git a/tests/pytest/util/autogen.py b/tests/pytest/util/autogen.py index b85348d0ba36a026e9400ece76e18de54185aeee..90cd347b9aed49816bc0de4242bacff8891173bc 100644 --- a/tests/pytest/util/autogen.py +++ b/tests/pytest/util/autogen.py @@ -51,7 +51,7 @@ class AutoGen: metas = [] for i in range(cnt): colname = f"{pre}{i}" - sel = len(types) % len(types) + sel = i % len(types) coltype = types[sel] sql = f"{colname} {coltype}" if sqls != "": diff --git a/tests/pytest/util/sql.py b/tests/pytest/util/sql.py index 783ee476cb5d3b34ffc4461e336412b0259dc975..c041282bfce6d5cf1ba6723ddf476a50bc1f2c7a 100644 --- a/tests/pytest/util/sql.py +++ b/tests/pytest/util/sql.py @@ -431,7 +431,7 @@ class TDSql: time.sleep(1) continue - def execute(self, sql,queryTimes=10): + def execute(self, sql,queryTimes=30): self.sql = sql i=1 while i <= queryTimes: diff --git a/tests/script/api/batchprepare.c b/tests/script/api/batchprepare.c index 0903095dc98dcf3652546859003fa65fd88e1569..99507ef5c3ae58d786dbfe8f91e72400d471ec3f 100644 --- a/tests/script/api/batchprepare.c +++ b/tests/script/api/batchprepare.c @@ -122,9 +122,11 @@ int insertAUTOTest2(TAOS_STMT *stmt, TAOS *taos); int insertAUTOTest3(TAOS_STMT *stmt, TAOS *taos); int queryColumnTest(TAOS_STMT *stmt, TAOS *taos); int queryMiscTest(TAOS_STMT *stmt, TAOS *taos); +int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos); enum { TTYPE_INSERT = 1, + TTYPE_INSERT_NG, TTYPE_QUERY, }; @@ -187,6 +189,8 @@ CaseCfg gCase[] = { {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 10, 10, 1, 3, 0, 0, 1, 2}, {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 10, 10, 1, 3, 0, 0, 1, 2}, + {"query:NG-TBNEXISTS",tListLen(fullColList), fullColList, TTYPE_INSERT_NG,0, false, false, insertNonExistsTb, 10, 10, 1, 3, 0, 0, 1, -1}, + // {"query:SUBT-COLUMN", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryColumnTest, 1, 10, 1, 1, 0, 0, 1, 2}, // {"query:SUBT-MISC", tListLen(fullColList), fullColList, TTYPE_QUERY, 0, false, false, queryMiscTest, 2, 10, 1, 1, 0, 0, 1, 2}, @@ -250,7 +254,7 @@ CaseCtrl gCaseCtrl = { .funcIdxList = NULL, .checkParamNum = false, .runTimes = 0, - .caseIdx = 24, + .caseIdx = 26, .caseNum = 1, .caseRunIdx = -1, .caseRunNum = -1, @@ -2191,6 +2195,47 @@ int queryMiscTest(TAOS_STMT *stmt, TAOS *taos) { } +int insertNonExistsTb(TAOS_STMT *stmt, TAOS *taos) { + BindData data = {0}; + prepareInsertData(&data); + + int code = taos_stmt_prepare(stmt, data.sql, 0); + if (code != 0){ + printf("!!!failed to execute taos_stmt_prepare. error:%s\n", taos_stmt_errstr(stmt)); + exit(1); + } + + bpCheckIsInsert(stmt, 1); + + char *buf = "tbnexist"; + code = bpSetTableNameTags(&data, 0, buf, stmt); + if (code == 0){ + printf("!!!taos_stmt_set_tbname expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_bind_param_batch(stmt, data.pBind)) { + printf("!!!taos_stmt_bind_param_batch expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_add_batch(stmt)) { + printf("!!!taos_stmt_add_batch expected error not occurred\n"); + exit(1); + } + + if (0 == taos_stmt_execute(stmt)) { + printf("!!!taos_stmt_execute expected error not occurred\n"); + exit(1); + } + + destroyData(&data); + + return 0; +} + + + int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { BindData data = {0}; @@ -2213,6 +2258,10 @@ int errorSQLTest1(TAOS_STMT *stmt, TAOS *taos) { } void prepareCheckResultImpl(TAOS * taos, char *tname, bool printr, int expected, bool silent) { + if (TTYPE_INSERT_NG == gCurCase->testType) { + return; + } + char sql[255] = "SELECT * FROM "; int32_t rows = 0; diff --git a/tests/script/api/makefile b/tests/script/api/makefile index 6739794cc8e09025ec18ee92100453edce082f77..6d55d8a75f7efe4383c81781165b2246e7a893a0 100644 --- a/tests/script/api/makefile +++ b/tests/script/api/makefile @@ -15,8 +15,11 @@ exe: gcc $(CFLAGS) ./stopquery.c -o $(ROOT)stopquery $(LFLAGS) gcc $(CFLAGS) ./dbTableRoute.c -o $(ROOT)dbTableRoute $(LFLAGS) gcc $(CFLAGS) ./insertSameTs.c -o $(ROOT)insertSameTs $(LFLAGS) + gcc $(CFLAGS) ./passwdTest.c -o $(ROOT)passwdTest $(LFLAGS) clean: rm $(ROOT)batchprepare rm $(ROOT)stopquery rm $(ROOT)dbTableRoute + rm $(ROOT)insertSameTs + rm $(ROOT)passwdTest diff --git a/tests/script/api/passwdTest.c b/tests/script/api/passwdTest.c new file mode 100644 index 0000000000000000000000000000000000000000..8a2b0a0390dad3acef1fca93abc89095164bb401 --- /dev/null +++ b/tests/script/api/passwdTest.c @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2019 TAOS Data, Inc. + * + * This program is free software: you can use, redistribute, and/or modify + * it under the terms of the GNU Affero General Public License, version 3 + * or later ("AGPL"), as published by the Free Software Foundation. + * + * This program is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. + * + * You should have received a copy of the GNU Affero General Public License + * along with this program. If not, see . + */ + +// TAOS standard API example. The same syntax as MySQL, but only a subset +// to compile: gcc -o demo demo.c -ltaos + +/** + * passwdTest.c + * - Run the test case in clear TDengine environment with default root passwd 'taosdata' + */ + +#include +#include +#include +#include +#include +#include "taos.h" // TAOS header file + +#define nDup 1 +#define nRoot 10 +#define nUser 10 +#define USER_LEN 24 + +void Test(TAOS *taos, char *qstr); +void createUers(TAOS *taos, const char *host, char *qstr); +void passVerTestMulti(const char *host, char *qstr); + +int nPassVerNotified = 0; +TAOS *taosu[nRoot] = {0}; +char users[nUser][USER_LEN] = {0}; + +void __taos_notify_cb(void *param, void *ext, int type) { + switch (type) { + case TAOS_NOTIFY_PASSVER: { + ++nPassVerNotified; + printf("%s:%d type:%d user:%s ver:%d\n", __func__, __LINE__, type, param ? (char *)param : "NULL", *(int *)ext); + break; + } + default: + printf("%s:%d unknown type:%d\n", __func__, __LINE__, type); + break; + } +} + +static void queryDB(TAOS *taos, char *command) { + int i; + TAOS_RES *pSql = NULL; + int32_t code = -1; + + for (i = 0; i < nDup; ++i) { + if (NULL != pSql) { + taos_free_result(pSql); + pSql = NULL; + } + + pSql = taos_query(taos, command); + code = taos_errno(pSql); + if (0 == code) { + break; + } + } + + if (code != 0) { + fprintf(stderr, "failed to run: %s, reason: %s\n", command, taos_errstr(pSql)); + taos_free_result(pSql); + taos_close(taos); + exit(EXIT_FAILURE); + } else { + fprintf(stderr, "success to run: %s\n", command); + } + + taos_free_result(pSql); +} + +int main(int argc, char *argv[]) { + char qstr[1024]; + + // connect to server + if (argc < 2) { + printf("please input server-ip \n"); + return 0; + } + + TAOS *taos = taos_connect(argv[1], "root", "taosdata", NULL, 0); + if (taos == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + createUers(taos, argv[1], qstr); + passVerTestMulti(argv[1], qstr); + + taos_close(taos); + taos_cleanup(); +} + +void createUers(TAOS *taos, const char *host, char *qstr) { + // users + for (int i = 0; i < nUser; ++i) { + sprintf(users[i], "user%d", i); + sprintf(qstr, "CREATE USER %s PASS 'taosdata'", users[i]); + queryDB(taos, qstr); + + taosu[i] = taos_connect(host, users[i], "taosdata", NULL, 0); + if (taosu[i] == NULL) { + printf("failed to connect to server, user:%s, reason:%s\n", users[i], "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taosu[i], __taos_notify_cb, users[i], TAOS_NOTIFY_PASSVER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb for user:%s since %d\n", users[i], code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb for user:%s\n", users[i]); + } + + // alter pass for users + sprintf(qstr, "alter user %s pass 'taos'", users[i]); + queryDB(taos, qstr); + } +} + +void passVerTestMulti(const char *host, char *qstr) { + // root + TAOS *taos[nRoot] = {0}; + char userName[USER_LEN] = "root"; + + for (int i = 0; i < nRoot; ++i) { + taos[i] = taos_connect(host, "root", "taosdata", NULL, 0); + if (taos[i] == NULL) { + printf("failed to connect to server, reason:%s\n", "null taos" /*taos_errstr(taos)*/); + exit(1); + } + + int code = taos_set_notify_cb(taos[i], __taos_notify_cb, userName, TAOS_NOTIFY_PASSVER); + + if (code != 0) { + fprintf(stderr, "failed to run: taos_set_notify_cb since %d\n", code); + } else { + fprintf(stderr, "success to run: taos_set_notify_cb\n"); + } + } + + queryDB(taos[0], "create database if not exists demo1 vgroups 1 minrows 10"); + queryDB(taos[0], "create database if not exists demo2 vgroups 1 minrows 10"); + queryDB(taos[0], "create database if not exists demo3 vgroups 1 minrows 10"); + + queryDB(taos[0], "create table demo1.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taos[0], "create table demo2.stb (ts timestamp, c1 int) tags(t1 int)"); + queryDB(taos[0], "create table demo3.stb (ts timestamp, c1 int) tags(t1 int)"); + + strcpy(qstr, "alter user root pass 'taos'"); + queryDB(taos[0], qstr); + + // calculate the nPassVerNotified for root and users + int nConn = nRoot + nUser; + + for (int i = 0; i < 15; ++i) { + if (nPassVerNotified >= nConn) break; + sleep(1); + } + + // close the taos_conn + for (int i = 0; i < nRoot; ++i) { + taos_close(taos[i]); + printf("%s:%d close taos[%d]\n", __func__, __LINE__, i); + sleep(1); + } + + for (int i = 0; i < nUser; ++i) { + taos_close(taosu[i]); + printf("%s:%d close taosu[%d]\n", __func__, __LINE__, i); + sleep(1); + } + + if (nPassVerNotified >= nConn) { + fprintf(stderr, "succeed to get passVer notification since nNotify %d >= nConn %d\n", nPassVerNotified, nConn); + } else { + fprintf(stderr, "failed to get passVer notification since nNotify %d < nConn %d\n", nPassVerNotified, nConn); + } + // sleep(300); +} \ No newline at end of file diff --git a/tests/script/local.supp b/tests/script/local.supp new file mode 100644 index 0000000000000000000000000000000000000000..562cddba46a7f5990965707ded9ce55eb961b717 --- /dev/null +++ b/tests/script/local.supp @@ -0,0 +1,227 @@ +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE8allocateERS9_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE16_M_allocate_nodeIJRKSt21piecewise_construct_tSt5tupleIJRS3_EESF_IJEEEEEPS8_DpOT_ + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE12_Scoped_nodeC1IJRKSt21piecewise_construct_tSt5tupleIJRS1_EESO_IJEEEEEPNS7_16_Hashtable_allocISaINS7_10_Hash_nodeIS5_Lb0EEEEEEDpOT_ + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZNSt13unordered_mapIjPFvPvESt4hashIjESt8equal_toIjESaISt4pairIKjS2_EEEixERS8_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb14ThreadLocalPtrC1EPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb10VersionSetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaINSt8__detail10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE8allocateERS9_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE16_M_allocate_nodeIJRKSt21piecewise_construct_tSt5tupleIJRS3_EESF_IJEEEEEPS8_DpOT_ + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE12_Scoped_nodeC1IJRKSt21piecewise_construct_tSt5tupleIJRS1_EESO_IJEEEEEPNS7_16_Hashtable_allocISaINS7_10_Hash_nodeIS5_Lb0EEEEEEDpOT_ + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZNSt13unordered_mapIjPFvPvESt4hashIjESt8equal_toIjESaISt4pairIKjS2_EEEixERS8_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb14ThreadLocalPtrC1EPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPm + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:calloc + fun:__cxa_thread_atexit_impl + fun:__cxa_thread_atexit + fun:__tls_init + fun:_ZTWN7rocksdb12perf_contextE + fun:_ZN7rocksdb17InstrumentedMutex4LockEv + fun:_ZN7rocksdb21InstrumentedMutexLockC1EPNS_17InstrumentedMutexE + fun:_ZN7rocksdb5Timer8ShutdownEv + fun:_ZN7rocksdb5TimerD1Ev + fun:_ZNKSt14default_deleteIN7rocksdb5TimerEEclEPS1_ + fun:_ZNSt10unique_ptrIN7rocksdb5TimerESt14default_deleteIS1_EED1Ev + fun:_ZN7rocksdb21PeriodicWorkSchedulerD1Ev + fun:__run_exit_handlers + fun:exit + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E + fun:_ZN7rocksdb13InternalStatsC1EiPNS_11SystemClockEPNS_16ColumnFamilyDataE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPm + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmProcessCreateVnodeReq +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb12_GLOBAL__N_111GetRegistryEv + fun:_ZN7rocksdb23CopyCacheDeleterRoleMapEv + fun:_ZN7rocksdb13InternalStats19CacheEntryRoleStats15BeginCollectionEPNS_5CacheEPNS_11SystemClockEm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE12CollectStatsEii + fun:_ZN7rocksdb13InternalStats22CollectCacheEntryStatsEb + fun:_ZN7rocksdb6DBImpl9DumpStatsEv + fun:_ZZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS_6DBImplEjjENKUlvE_clEv + fun:_ZSt13__invoke_implIvRZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS0_6DBImplEjjEUlvE_JEET_St14__invoke_otherOT0_DpOT1_ + fun:_ZSt10__invoke_rIvRZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS0_6DBImplEjjEUlvE_JEENSt9enable_ifIXsrSt6__and_IJSt7is_voidIT_ESt14__is_invocableIT0_JDpT1_EEEE5valueES9_E4typeEOSC_DpOSD_ + fun:_ZNSt17_Function_handlerIFvvEZN7rocksdb21PeriodicWorkScheduler8RegisterEPNS1_6DBImplEjjEUlvE_E9_M_invokeERKSt9_Any_data + fun:_ZNKSt8functionIFvvEEclEv + fun:_ZN7rocksdb5Timer3RunEv + fun:_ZSt13__invoke_implIvMN7rocksdb5TimerEFvvEPS1_JEET_St21__invoke_memfun_derefOT0_OT1_DpOT2_ + fun:_ZSt8__invokeIMN7rocksdb5TimerEFvvEJPS1_EENSt15__invoke_resultIT_JDpT0_EE4typeEOS6_DpOS7_ + fun:_ZNSt6thread8_InvokerISt5tupleIJMN7rocksdb5TimerEFvvEPS3_EEE9_M_invokeIJLm0ELm1EEEEvSt12_Index_tupleIJXspT_EEE + fun:_ZNSt6thread8_InvokerISt5tupleIJMN7rocksdb5TimerEFvvEPS3_EEEclEv + fun:_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJMN7rocksdb5TimerEFvvEPS4_EEEEE6_M_runEv + obj:/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28 + fun:start_thread +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorIPNSt8__detail15_Hash_node_baseEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaIPNSt8__detail15_Hash_node_baseEEE8allocateERS3_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeIPN7rocksdb16ThreadStatusDataELb0EEEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE13_M_rehash_auxEmSt17integral_constantIbLb1EE + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE9_M_rehashEmRKm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE21_M_insert_unique_nodeERKS2_mmPNS4_10_Hash_nodeIS2_Lb0EEEm + fun:_ZNSt10_HashtableIPN7rocksdb16ThreadStatusDataES2_SaIS2_ENSt8__detail9_IdentityESt8equal_toIS2_ESt4hashIS2_ENS4_18_Mod_range_hashingENS4_20_Default_ranged_hashENS4_20_Prime_rehash_policyENS4_17_Hashtable_traitsILb0ELb1ELb1EEEE9_M_insertIRKS2_NS4_10_AllocNodeISaINS4_10_Hash_nodeIS2_Lb0EEEEEEEESt4pairINS4_14_Node_iteratorIS2_Lb1ELb0EEEbEOT_RKT0_St17integral_constantIbLb1EEm + fun:_ZNSt8__detail12_Insert_baseIPN7rocksdb16ThreadStatusDataES3_SaIS3_ENS_9_IdentityESt8equal_toIS3_ESt4hashIS3_ENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb1ELb1EEEE6insertERKS3_ + fun:_ZNSt13unordered_setIPN7rocksdb16ThreadStatusDataESt4hashIS2_ESt8equal_toIS2_ESaIS2_EE6insertERKS2_ + fun:_ZN7rocksdb19ThreadStatusUpdater14RegisterThreadENS_12ThreadStatus10ThreadTypeEm + fun:_ZN7rocksdb16ThreadStatusUtil14RegisterThreadEPKNS_3EnvENS_12ThreadStatus10ThreadTypeE + fun:_ZN7rocksdb14ThreadPoolImpl4Impl15BGThreadWrapperEPv + fun:_ZSt13__invoke_implIvPFvPvEJPN7rocksdb16BGThreadMetadataEEET_St14__invoke_otherOT0_DpOT1_ + fun:_ZSt8__invokeIPFvPvEJPN7rocksdb16BGThreadMetadataEEENSt15__invoke_resultIT_JDpT0_EE4typeEOS7_DpOS8_ + fun:_ZNSt6thread8_InvokerISt5tupleIJPFvPvEPN7rocksdb16BGThreadMetadataEEEE9_M_invokeIJLm0ELm1EEEEvSt12_Index_tupleIJXspT_EEE + fun:_ZNSt6thread8_InvokerISt5tupleIJPFvPvEPN7rocksdb16BGThreadMetadataEEEEclEv + fun:_ZNSt6thread11_State_implINS_8_InvokerISt5tupleIJPFvPvEPN7rocksdb16BGThreadMetadataEEEEEE6_M_runEv + obj:/usr/lib/x86_64-linux-gnu/libstdc++.so.6.0.28 +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN9__gnu_cxx13new_allocatorIPNSt8__detail15_Hash_node_baseEE8allocateEmPKv + fun:_ZNSt16allocator_traitsISaIPNSt8__detail15_Hash_node_baseEEE8allocateERS3_m + fun:_ZNSt8__detail16_Hashtable_allocISaINS_10_Hash_nodeISt4pairIKjPFvPvEELb0EEEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE19_M_allocate_bucketsEm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE13_M_rehash_auxEmSt17integral_constantIbLb1EE + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE9_M_rehashEmRKm + fun:_ZNSt10_HashtableIjSt4pairIKjPFvPvEESaIS5_ENSt8__detail10_Select1stESt8equal_toIjESt4hashIjENS7_18_Mod_range_hashingENS7_20_Default_ranged_hashENS7_20_Prime_rehash_policyENS7_17_Hashtable_traitsILb0ELb0ELb1EEEE21_M_insert_unique_nodeERS1_mmPNS7_10_Hash_nodeIS5_Lb0EEEm + fun:_ZNSt8__detail9_Map_baseIjSt4pairIKjPFvPvEESaIS6_ENS_10_Select1stESt8equal_toIjESt4hashIjENS_18_Mod_range_hashingENS_20_Default_ranged_hashENS_20_Prime_rehash_policyENS_17_Hashtable_traitsILb0ELb0ELb1EEELb1EEixERS2_ + fun:_ZNSt13unordered_mapIjPFvPvESt4hashIjESt8equal_toIjESaISt4pairIKjS2_EEEixERS8_ + fun:_ZN7rocksdb14ThreadLocalPtr10StaticMeta10SetHandlerEjPFvPvE + fun:_ZN7rocksdb14ThreadLocalPtrC1EPFvPvE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb10VersionSetC1ERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_5CacheEPNS_18WriteBufferManagerEPNS_15WriteControllerEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb6DBImplC1ERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEbbb + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb12_GLOBAL__N_125CreateThreadStatusUpdaterEv + fun:_ZN7rocksdb12_GLOBAL__N_18PosixEnvC1Ev + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:_ZN7rocksdb7OptionsC1Ev + fun:_ZN7rocksdb18ImmutableCFOptionsC1Ev + fun:_Z41__static_initialization_and_destruction_0ii + fun:_GLOBAL__sub_I_cf_options.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb14ThreadLocalPtr8InstanceEv + fun:_ZN7rocksdb14ThreadLocalPtr14InitSingletonsEv + fun:_ZN7rocksdb3Env7DefaultEv + fun:_ZN7rocksdb9DBOptionsC1Ev + fun:_ZN7rocksdb7OptionsC1Ev + fun:_ZN7rocksdb18ImmutableCFOptionsC1Ev + fun:_Z41__static_initialization_and_destruction_0ii + fun:_GLOBAL__sub_I_cf_options.cc + fun:__libc_csu_init + fun:(below main) +} +{ + + Memcheck:Leak + match-leak-kinds: reachable + fun:_Znwm + fun:_ZN7rocksdb24CacheEntryStatsCollectorINS_13InternalStats19CacheEntryRoleStatsEE9GetSharedEPNS_5CacheEPNS_11SystemClockEPSt10shared_ptrIS3_E + fun:_ZN7rocksdb13InternalStatsC1EiPNS_11SystemClockEPNS_16ColumnFamilyDataE + fun:_ZN7rocksdb16ColumnFamilyDataC1EjRKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPNS_7VersionEPNS_5CacheEPNS_18WriteBufferManagerERKNS_19ColumnFamilyOptionsERKNS_18ImmutableDBOptionsERKNS_11FileOptionsEPNS_15ColumnFamilySetEPNS_16BlockCacheTracerERKSt10shared_ptrINS_8IOTracerEES8_ + fun:_ZN7rocksdb15ColumnFamilySet18CreateColumnFamilyERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEjPNS_7VersionERKNS_19ColumnFamilyOptionsE + fun:_ZN7rocksdb10VersionSet18CreateColumnFamilyERKNS_19ColumnFamilyOptionsEPKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler15CreateCfAndInitERKNS_19ColumnFamilyOptionsERKNS_11VersionEditE + fun:_ZN7rocksdb18VersionEditHandler10InitializeEv + fun:_ZN7rocksdb22VersionEditHandlerBase7IterateERNS_3log6ReaderEPNS_6StatusE + fun:_ZN7rocksdb10VersionSet7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbPNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEE + fun:_ZN7rocksdb6DBImpl7RecoverERKSt6vectorINS_22ColumnFamilyDescriptorESaIS2_EEbbbPm + fun:_ZN7rocksdb6DBImpl4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPNS_2DBEbb + fun:_ZN7rocksdb2DB4OpenERKNS_9DBOptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEERKSt6vectorINS_22ColumnFamilyDescriptorESaISD_EEPSC_IPNS_18ColumnFamilyHandleESaISJ_EEPPS0_ + fun:_ZN7rocksdb2DB4OpenERKNS_7OptionsERKNSt7__cxx1112basic_stringIcSt11char_traitsIcESaIcEEEPPS0_ + fun:rocksdb_open + fun:tsdbOpenRocksCache + fun:tsdbOpenCache + fun:tsdbOpen + fun:vnodeOpen + fun:vmOpenVnodeInThread +} diff --git a/tests/script/sh/exec.sh b/tests/script/sh/exec.sh index f548a4cc418d34ac5d4fe27fb944181c54b3a5c2..5061688de2aa3e892774227677e7a31438e72cd9 100755 --- a/tests/script/sh/exec.sh +++ b/tests/script/sh/exec.sh @@ -109,8 +109,10 @@ if [ "$EXEC_OPTON" = "start" ]; then if [ "$VALGRIND_OPTION" = "true" ]; then TT=`date +%s` #mkdir ${LOG_DIR}/${TT} - echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" - nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + #echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" + #nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --gen-suppressions=all --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & + echo "nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --suppressions=${SCRIPT_DIR}/local.supp --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" + nohup valgrind --log-file=${LOG_DIR}/valgrind-taosd-${NODE_NAME}-${TT}.log --suppressions=${SCRIPT_DIR}/local.supp --tool=memcheck --leak-check=full --show-reachable=no --track-origins=yes --show-leak-kinds=all --num-callers=20 -v -v --workaround-gcc296-bugs=yes $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 & else echo "nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2>&1 &" nohup $EXE_DIR/taosd -c $CFG_DIR > /dev/null 2> $ASAN_DIR/$NODE_NAME.asan & diff --git a/tests/script/sh/max_vol.c b/tests/script/sh/max_vol.c new file mode 100644 index 0000000000000000000000000000000000000000..4f9ecd33a7e3471a6502857a81db09470236b059 --- /dev/null +++ b/tests/script/sh/max_vol.c @@ -0,0 +1,101 @@ +#include +#include +#include +#include + +#include "taosudf.h" + +#define STR_MAX_LEN 256 // inter buffer length + +// init +DLL_EXPORT int32_t max_vol_init() +{ + return 0; +} + +// destory +DLL_EXPORT int32_t max_vol_destroy() +{ + return 0; +} + +// start +DLL_EXPORT int32_t max_vol_start(SUdfInterBuf *buf) +{ + memset(buf->buf, 0, sizeof(float) + STR_MAX_LEN); + // set init value + *((float*)buf->buf) = -10000000; + buf->bufLen = sizeof(float) + STR_MAX_LEN; + buf->numOfResult = 0; + return 0; +} + +DLL_EXPORT int32_t max_vol(SUdfDataBlock *block, SUdfInterBuf *interBuf, SUdfInterBuf *newInterBuf) { + float maxValue = *(float *)interBuf->buf; + char strBuff[STR_MAX_LEN] = "inter1buf"; + + if (block->numOfCols < 2) + { + return TSDB_CODE_UDF_INVALID_INPUT; + } + + // check data type + for (int32_t i = 0; i < block->numOfCols; ++i) + { + SUdfColumn *col = block->udfCols[i]; + if( i == block->numOfCols - 1) { + // last column is device id , must varchar + if (col->colMeta.type != TSDB_DATA_TYPE_VARCHAR ) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } else { + if (col->colMeta.type != TSDB_DATA_TYPE_FLOAT) { + return TSDB_CODE_UDF_INVALID_INPUT; + } + } + } + + // calc max voltage + SUdfColumn *lastCol = block->udfCols[block->numOfCols - 1]; + for (int32_t i = 0; i < (block->numOfCols - 1); ++i) { + for (int32_t j = 0; j < block->numOfRows; ++j) { + SUdfColumn *col = block->udfCols[i]; + if (udfColDataIsNull(col, j)) { + continue; + } + char *data = udfColDataGetData(col, j); + float voltage = *(float *)data; + if (voltage > maxValue) { + maxValue = voltage; + char *valData = udfColDataGetData(lastCol, j); + // get device id + char *deviceId = valData + sizeof(uint16_t); + sprintf(strBuff, "%s_(%d,%d)_%f", deviceId, j, i, maxValue); + } + } + } + + *(float*)newInterBuf->buf = maxValue; + strcpy(newInterBuf->buf + sizeof(float), strBuff); + newInterBuf->bufLen = sizeof(float) + strlen(strBuff)+1; + newInterBuf->numOfResult = 1; + return 0; +} + +DLL_EXPORT int32_t max_vol_finish(SUdfInterBuf *buf, SUdfInterBuf *resultData) +{ + char * str = buf->buf + sizeof(float); + // copy to des + char * des = resultData->buf + sizeof(uint16_t); + strcpy(des, str); + + // set binary type len + uint16_t len = strlen(str); + *((uint16_t*)resultData->buf) = len; + + // set buf len + resultData->bufLen = len + sizeof(uint16_t); + // set row count + resultData->numOfResult = 1; + return 0; +} diff --git a/tests/script/tsim/insert/delete0.sim b/tests/script/tsim/insert/delete0.sim new file mode 100644 index 0000000000000000000000000000000000000000..56538536439bbf3d7e9937353ee65acaea34fb43 --- /dev/null +++ b/tests/script/tsim/insert/delete0.sim @@ -0,0 +1,161 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +print =============== create database with different precision +sql create database d0 keep 365 +sql create database d1 keep 365 precision 'ms' +sql create database d2 keep 365 precision 'us' +sql create database d3 keep 365 precision 'ns' + +sql select * from information_schema.ins_databases +if $rows != 6 then + return -1 +endi + +print $data00 $data01 $data02 + + +sql create table if not exists d0.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d1.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d2.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d3.stb (ts timestamp, c1 int, c2 float, c3 double) tags (t1 int unsigned) +sql create table if not exists d0.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d1.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d2.ntb (ts timestamp, c1 int, c2 float, c3 double) +sql create table if not exists d3.ntb (ts timestamp, c1 int, c2 float, c3 double) + +sql create table d0.ct1 using d0.stb tags(1000) +sql create table d1.ct1 using d1.stb tags(1000) +sql create table d2.ct1 using d2.stb tags(1000) +sql create table d3.ct1 using d3.stb tags(1000) +sql create table d0.ct2 using d0.stb tags(1000) +sql create table d1.ct2 using d1.stb tags(1000) +sql create table d2.ct2 using d2.stb tags(1000) +sql create table d3.ct2 using d3.stb tags(1000) + + +sql insert into d0.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct1 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ct2 values(now+0s, 10, 2.0, 3.0) +sql insert into d0.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d1.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d2.ntb values(now+0s, 10, 2.0, 3.0) +sql insert into d3.ntb values(now+0s, 10, 2.0, 3.0) + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 2 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 2 then + return -1 +endi + +print =============== delete from child table +sql delete from d0.ct1 where ts < now() +sql delete from d1.ct1 where ts < now() +sql delete from d2.ct1 where ts < now() +sql delete from d3.ct1 where ts < now() + + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 1 then + return -1 +endi +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 1 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 1 then + return -1 +endi + +print =============== delete from super table +sql delete from d0.stb where ts < now() +sql delete from d1.stb where ts < now() +sql delete from d2.stb where ts < now() +sql delete from d3.stb where ts < now() + +print =============== query data from super table +sql select count(*) from d0.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.stb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.stb +if $data00 != 0 then + return -1 +endi + +print =============== delete from normal table +sql delete from d0.ntb where ts < now() +sql delete from d1.ntb where ts < now() +sql delete from d2.ntb where ts < now() +sql delete from d3.ntb where ts < now() + +print =============== query data from normal table +sql select count(*) from d0.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d1.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d2.ntb +if $data00 != 0 then + return -1 +endi +sql select count(*) from d3.ntb +if $data00 != 0 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/parser/last_cache.sim b/tests/script/tsim/parser/last_cache.sim index 9a41a9f5aa6de119eddfc7cf31eeddccacd1fe5e..3f1a29d928eed421ee23aab79444495ecbeed6f2 100644 --- a/tests/script/tsim/parser/last_cache.sim +++ b/tests/script/tsim/parser/last_cache.sim @@ -54,6 +54,7 @@ sql insert into tbd values ("2021-05-11 10:12:29",NULL,NULL,NULL,NULL ) run tsim/parser/last_cache_query.sim +sql flush database $db system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/parser/limit1.sim b/tests/script/tsim/parser/limit1.sim index d1a75f3ba9603ff2d97a029258dc91f0fa097d43..bae5eba7a4395099ad50105b6259cc69a7b5947e 100644 --- a/tests/script/tsim/parser/limit1.sim +++ b/tests/script/tsim/parser/limit1.sim @@ -55,6 +55,7 @@ print ====== tables created run tsim/parser/limit1_tb.sim run tsim/parser/limit1_stb.sim +sql flush database $db print ================== restart server to commit data into disk system sh/exec.sh -n dnode1 -s stop -x SIGINT system sh/exec.sh -n dnode1 -s start diff --git a/tests/script/tsim/query/multi_order_by.sim b/tests/script/tsim/query/multi_order_by.sim new file mode 100644 index 0000000000000000000000000000000000000000..7b7f2abe365333b2f63a5825ed7945450bddc0d5 --- /dev/null +++ b/tests/script/tsim/query/multi_order_by.sim @@ -0,0 +1,57 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect + +sql create database test; +sql use test; + +sql create table t(ts timestamp, f int); +sql insert into t values(now,0)(now+1s, 1)(now+2s, 2)(now+3s,3)(now+4s,4)(now+5s,5)(now+6s,6)(now+7s,7)(now+8s,8)(now+9s,9) +sql select * from (select * from t order by ts desc limit 3 offset 2) order by ts; +print $data01 $data11 $data21 +if $data01 != 5 then + return -1 +endi +if $data11 != 6 then + return -1 +endi +if $data21 != 7 then + return -1 +endi +sql select * from (select * from t order by ts limit 3 offset 2) order by ts desc; +print $data01 $data11 $data21 +if $data01 != 4 then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data21 != 2 then + return -1 +endi +sql select * from (select * from t order by ts desc limit 3 offset 2) order by ts desc; +print $data01 $data11 $data21 +if $data01 != 7 then + return -1 +endi +if $data11 != 6 then + return -1 +endi +if $data21 != 5 then + return -1 +endi +sql select * from (select * from t order by ts limit 3 offset 2) order by ts; +print $data01 $data11 $data21 +if $data01 != 2 then + return -1 +endi +if $data11 != 3 then + return -1 +endi +if $data21 != 4 then + return -1 +endi + + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/tag_scan.sim b/tests/script/tsim/query/tag_scan.sim new file mode 100644 index 0000000000000000000000000000000000000000..03e3a2063218ad83520507b418f92eb3969f0f84 --- /dev/null +++ b/tests/script/tsim/query/tag_scan.sim @@ -0,0 +1,48 @@ +system sh/stop_dnodes.sh +system sh/deploy.sh -n dnode1 -i 1 +system sh/exec.sh -n dnode1 -s start +sql connect +sql drop database if exists test +sql create database test; +sql use test; + +sql create table st(ts timestamp, f int) tags (t int); +sql insert into ct1 using st tags(1) values(now, 1); +sql insert into ct2 using st tags(2) values(now, 2); +sql insert into ct3 using st tags(3) values(now, 3); +sql insert into ct4 using st tags(4) values(now, 4); + +sql create table st2(ts timestamp, f int) tags (t int); +sql insert into ct21 using st2 tags(1) values(now, 1); +sql insert into ct22 using st2 tags(2) values(now, 2); +sql insert into ct23 using st2 tags(3) values(now, 3); +sql insert into ct24 using st2 tags(4) values(now, 4); + +sql select tbname, 1 from st group by tbname order by tbname; +print $rows $data00 $data10 $data20 +if $rows != 4 then + return -1 +endi +if $data00 != @ct1@ then + return -1 +endi +if $data10 != @ct2@ then + return -1 +endi +sql select tbname, 1 from st group by tbname slimit 0, 1; +print $rows +if $rows != 1 then + return -1 +endi +sql select tbname, 1 from st group by tbname slimit 2, 2; +print $rows $data00 $data10 +if $rows != 2 then + return -1 +endi +sql select tbname, 1 from st group by tbname order by tbname slimit 0, 1; +print $rows $data00 $data10 $data20 +if $rows != 4 then + return -1 +endi + +system sh/exec.sh -n dnode1 -s stop -x SIGINT diff --git a/tests/script/tsim/query/udfpy.sim b/tests/script/tsim/query/udfpy.sim index 025df7984be1f199d336e3041421f5960d3a3097..9e0492ffd95e3baed20a77ddf3423a1e8872c1d0 100644 --- a/tests/script/tsim/query/udfpy.sim +++ b/tests/script/tsim/query/udfpy.sim @@ -280,7 +280,37 @@ if $data20 != 8.000000000 then return -1 endi +sql create or replace function bit_and as '/tmp/udf/libbitand.so' outputtype int +sql select func_version from information_schema.ins_functions where name='bit_and' +if $data00 != 1 then + return -1 +endi +sql select bit_and(f1, f2) from t2; +print $rows , $data00 , $data10 , $data20 , $data30 , $data40 , $data50 +if $rows != 6 then + return -1 +endi +if $data00 != 0 then + return -1 +endi +if $data10 != 1 then + return -1 +endi +if $data20 != NULL then + return -1 +endi + +if $data30 != NULL then + return -1 +endi + +if $data40 != 0 then + return -1 +endi +if $data50 != 1 then + return -1 +endi #sql drop function bit_and; #sql show functions; #if $rows != 1 then diff --git a/tests/script/tsim/stream/basic1.sim b/tests/script/tsim/stream/basic1.sim index e69875d69f20b01f91ea2b90fd912c84b88d2455..15ca6bf7c924bb7cffba8f0607cb0a5c1fea95cb 100644 --- a/tests/script/tsim/stream/basic1.sim +++ b/tests/script/tsim/stream/basic1.sim @@ -37,7 +37,7 @@ if $loop_count == 20 then endi if $rows != 4 then - print =====rows=$rows + print =====rows=$rows, expect 4 goto loop0 endi @@ -53,7 +53,7 @@ if $data02 != 2 then endi if $data03 != 5 then - print =====data03=$data03 + print =====data03=$data03, expect:5 goto loop0 endi diff --git a/tests/script/tsim/testsuit.sim b/tests/script/tsim/testsuit.sim index c5fbf41b66fcba4088e6480c721d4a6e21811d7a..0abe56ab3c3d7d09b6f441aed813445be0d2df92 100644 --- a/tests/script/tsim/testsuit.sim +++ b/tests/script/tsim/testsuit.sim @@ -114,6 +114,7 @@ run tsim/insert/basic1.sim run tsim/insert/commit-merge0.sim run tsim/insert/basic0.sim run tsim/insert/update0.sim +run tsim/insert/delete0.sim run tsim/insert/backquote.sim run tsim/insert/null.sim run tsim/catalog/alterInCurrent.sim diff --git a/tests/script/tsim/tmq/basic1.sim b/tests/script/tsim/tmq/basic1.sim index b296290214d223a54153e27ee6e1386fc7722541..4551228f2f67e6da9beed93f984e93113a77055f 100644 --- a/tests/script/tsim/tmq/basic1.sim +++ b/tests/script/tsim/tmq/basic1.sim @@ -117,8 +117,8 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -140,7 +140,7 @@ if $data[0][2] != $expectmsgcnt then print expect $expectmsgcnt , actual $data02 return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi $loop_cnt = $loop_cnt + 1 @@ -192,8 +192,8 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -211,7 +211,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -266,8 +266,8 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -285,7 +285,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic1Of2Cons.sim b/tests/script/tsim/tmq/basic1Of2Cons.sim index 4c966c370e9bc57c4eb453f72234d027de972576..51d39e8d110f0be00e3a16cf7cdd47e6be226a78 100644 --- a/tests/script/tsim/tmq/basic1Of2Cons.sim +++ b/tests/script/tsim/tmq/basic1Of2Cons.sim @@ -117,11 +117,11 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -148,14 +148,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -223,10 +223,10 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -253,14 +253,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -328,10 +328,10 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -358,14 +358,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_4 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_4 endi endi diff --git a/tests/script/tsim/tmq/basic2.sim b/tests/script/tsim/tmq/basic2.sim index 6d49b46c854678930a30e88a23720faa2e6c1844..8356a60b672c54feb37663d5c3b8f3391c99a456 100644 --- a/tests/script/tsim/tmq/basic2.sim +++ b/tests/script/tsim/tmq/basic2.sim @@ -86,8 +86,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -108,7 +108,7 @@ endi if $data[0][2] != $expectmsgcnt then return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi @@ -146,8 +146,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -165,7 +165,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -206,8 +206,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -225,7 +225,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic2Of2Cons.sim b/tests/script/tsim/tmq/basic2Of2Cons.sim index db660a0c93be23b2b42fc118f522b7944b5f4649..63e7e2dcf4e600e340f6c3767ab91dfd19fc5338 100644 --- a/tests/script/tsim/tmq/basic2Of2Cons.sim +++ b/tests/script/tsim/tmq/basic2Of2Cons.sim @@ -85,10 +85,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -115,14 +115,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfStb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfStb -if $data[0][2] == $totalMsgOfStb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfStb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -175,10 +175,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -205,14 +205,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -265,10 +265,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -295,14 +295,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_4 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_4 endi endi diff --git a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim index 54e10126f1bd2827ce219a7f8d5cfc1511b6ed88..cfdae059dc862d4177d12891ffd9499900ba3b41 100644 --- a/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim +++ b/tests/script/tsim/tmq/basic2Of2ConsOverlap.sim @@ -83,8 +83,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 3 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_stb_all @@ -92,7 +92,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_stb_function $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -121,40 +121,40 @@ endi # $data[0][2]/$data[1][2] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][2] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][2] > $totalMsgOfStb then - return -1 -endi -if $data[1][2] < $totalMsgOfOneTopic then +#if $data[0][2] < $totalMsgOfOneTopic then +# return -1 +#endi +if $data[0][2] > $expectmsgcnt then return -1 endi -if $data[1][2] > $totalMsgOfStb then +#if $data[1][2] < $totalMsgOfOneTopic then +# return -1 +#endi +if $data[1][2] > $expectmsgcnt then return -1 endi -$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb +#$totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfMsgCnt = $data[0][2] + $data[1][2] -if $sumOfMsgCnt != $totalMsgCons then - print total: $totalMsgCons +if $sumOfMsgCnt != $expectmsgcnt then + print total: $expectmsgcnt print sum: $sumOfMsgCnt return -1 endi # $data[0][3]/$data[1][3] should be between $totalMsgOfOneTopic and $totalMsgOfStb. -if $data[0][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[0][3] > $totalMsgOfStb then - return -1 -endi -if $data[1][3] < $totalMsgOfOneTopic then - return -1 -endi -if $data[1][3] > $totalMsgOfStb then - return -1 -endi +#if $data[0][3] < $totalMsgOfStb then +# return -1 +#endi +#if $data[0][3] > $totalMsgOfStb then +# return -1 +#endi +#if $data[1][3] < $totalMsgOfStb then +# return -1 +#endi +#if $data[1][3] > $totalMsgOfStb then +# return -1 +#endi $totalMsgCons = $totalMsgOfOneTopic + $totalMsgOfStb $sumOfRows = $data[0][3] + $data[1][3] @@ -195,15 +195,15 @@ $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfCtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ctb_function $topicList = $topicList . , $topicList = $topicList . topic_ctb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now +1s, $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -232,24 +232,24 @@ endi # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfCtb # or $data[0][2] $totalMsgOfCtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfCtb then +if $data[0][2] == $topicNum then + if $data[1][2] == 1 then goto check_ok_0 endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfCtb then +elif $data[0][2] == 1 then + if $data[1][2] == $topicNum then goto check_ok_0 endi endi return -1 check_ok_0: -if $data[0][3] == $totalMsgOfOneTopic then - if $data[1][3] == $totalMsgOfCtb then +if $data[0][3] == $totalMsgOfCtb then + if $data[1][3] == $totalMsgOfOneTopic then goto check_ok_1 endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfCtb then +elif $data[0][3] == $totalMsgOfOneTopic then + if $data[1][3] == $totalMsgOfCtb then goto check_ok_1 endi endi @@ -289,8 +289,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfOneTopic = $rowsPerCtb $totalMsgOfNtb = $totalMsgOfOneTopic * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $topicList = ' . topic_ntb_function @@ -298,7 +298,7 @@ $topicList = $topicList . , $topicList = $topicList . topic_ntb_all $topicList = $topicList . ' $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -327,12 +327,12 @@ endi # either $data[0][2] $totalMsgOfOneTopic and $data[1][2] == $totalMsgOfNtb # or $data[0][2] $totalMsgOfNtb and $data[1][2] == $totalMsgOfOneTopic -if $data[0][2] == $totalMsgOfOneTopic then - if $data[1][2] == $totalMsgOfNtb then +if $data[0][2] == $expectmsgcnt then + if $data[1][2] == 1 then goto check_ok_2 endi -elif $data[1][2] == $totalMsgOfOneTopic then - if $data[0][2] == $totalMsgOfNtb then +elif $data[0][2] == 1 then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi @@ -343,8 +343,8 @@ if $data[0][3] == $totalMsgOfOneTopic then if $data[1][3] == $totalMsgOfNtb then goto check_ok_3 endi -elif $data[1][3] == $totalMsgOfOneTopic then - if $data[0][3] == $totalMsgOfNtb then +elif $data[0][3] == $totalMsgOfNtb then + if $data[1][3] == $totalMsgOfOneTopic then goto check_ok_3 endi endi diff --git a/tests/script/tsim/tmq/basic3.sim b/tests/script/tsim/tmq/basic3.sim index 1e95fa90a5bf603a2afc1f2fd90ee9e3f6dacdfd..a64dd6924d7c30dec12e315924439b37019af500 100644 --- a/tests/script/tsim/tmq/basic3.sim +++ b/tests/script/tsim/tmq/basic3.sim @@ -117,8 +117,8 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -136,10 +136,10 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $expectmsgcnt then - return -1 -endi -if $data[0][3] != $expectmsgcnt then +#if $data[0][2] != $expectmsgcnt then +# return -1 +#endi +if $data[0][3] != $totalMsgOfStb then return -1 endi $loop_cnt = $loop_cnt + 1 @@ -191,8 +191,8 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -210,7 +210,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -265,8 +265,8 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -284,7 +284,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic3Of2Cons.sim b/tests/script/tsim/tmq/basic3Of2Cons.sim index be0292c57b23be1bf2342f50229076c90ebc4ead..4e47e3dbf9a0eeaf4380e43ceb6caef81f3c8770 100644 --- a/tests/script/tsim/tmq/basic3Of2Cons.sim +++ b/tests/script/tsim/tmq/basic3Of2Cons.sim @@ -116,10 +116,10 @@ endi $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 3 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $cdbName -s start @@ -169,19 +169,19 @@ endi if $data[0][3] <= 0 then return -1 endi -if $data[0][3] >= $expectmsgcnt then +if $data[0][3] >= $totalMsgOfStb then return -1 endi if $data[1][3] <= 0 then return -1 endi -if $data[1][3] >= $expectmsgcnt then +if $data[1][3] >= $totalMsgOfStb then return -1 endi $sumOfMsgRows = $data[0][3] + $data[1][3] -if $sumOfMsgRows != $expectmsgcnt then +if $sumOfMsgRows != $totalMsgOfStb then return -1 endi @@ -234,10 +234,10 @@ endi $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -264,13 +264,13 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi -elif $data[1][2] == $totalMsgOfCtb then +elif $data[1][2] == $expectmsgcnt then if $data[0][2] == 0 then goto check_ok_0 endi @@ -339,10 +339,10 @@ endi $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 1 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -369,13 +369,13 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi -elif $data[1][2] == $totalMsgOfNtb then +elif $data[1][2] == $expectmsgcnt then if $data[0][2] == 0 then goto check_ok_2 endi diff --git a/tests/script/tsim/tmq/basic4.sim b/tests/script/tsim/tmq/basic4.sim index 33a66628d0b3871740754d15225477015eb8e055..6b35342ad119431d1eece232486069c9792015d9 100644 --- a/tests/script/tsim/tmq/basic4.sim +++ b/tests/script/tsim/tmq/basic4.sim @@ -83,8 +83,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 9 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -105,7 +105,7 @@ endi if $data[0][2] != $expectmsgcnt then return -1 endi -if $data[0][3] != $expectmsgcnt then +if $data[0][3] != $totalMsgOfStb then return -1 endi @@ -143,8 +143,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -162,7 +162,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfCtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfCtb then @@ -203,8 +203,8 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -222,7 +222,7 @@ endi if $data[0][1] != $consumerId then return -1 endi -if $data[0][2] != $totalMsgOfNtb then +if $data[0][2] != $expectmsgcnt then return -1 endi if $data[0][3] != $totalMsgOfNtb then diff --git a/tests/script/tsim/tmq/basic4Of2Cons.sim b/tests/script/tsim/tmq/basic4Of2Cons.sim index fdee3f633e0a887c650f5ee67ae699c8642b537d..122a91af3627fb12d00be61a503942e32dca604d 100644 --- a/tests/script/tsim/tmq/basic4Of2Cons.sim +++ b/tests/script/tsim/tmq/basic4Of2Cons.sim @@ -82,10 +82,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfStb = $ctbNum * $rowsPerCtb $totalMsgOfStb = $totalMsgOfStb * $topicNum -$expectmsgcnt = $totalMsgOfStb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = 9 +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfStb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from stb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -w $dbName -s start @@ -134,19 +134,19 @@ endi if $data[0][3] <= 0 then return -1 endi -if $data[0][3] >= $expectmsgcnt then +if $data[0][3] >= $totalMsgOfStb then return -1 endi if $data[1][3] <= 0 then return -1 endi -if $data[1][3] >= $expectmsgcnt then +if $data[1][3] >= $totalMsgOfStb then return -1 endi $sumOfConsRow = $data[0][3] + $data[1][3] -if $sumOfConsRow != $expectmsgcnt then +if $sumOfConsRow != $totalMsgOfStb then return -1 endi @@ -184,10 +184,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfCtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfCtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfCtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ctb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -214,14 +214,14 @@ if $data[0][1] == 1 then endi endi -# either $data[0][2] == $totalMsgOfCtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfCtb -if $data[0][2] == $totalMsgOfCtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_0 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfCtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_0 endi endi @@ -275,10 +275,10 @@ $topicList = $topicList . ' $consumerId = 0 $totalMsgOfNtb = $rowsPerCtb * $topicNum -$expectmsgcnt = $totalMsgOfNtb -sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +$expectmsgcnt = $topicNum +sql insert into consumeinfo values (now , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) $consumerId = 1 -sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $expectmsgcnt , $ifcheckdata , $ifmanualcommit ) +sql insert into consumeinfo values (now+1s , $consumerId , $topicList , $keyList , $totalMsgOfNtb , $ifcheckdata , $ifmanualcommit ) print == start consumer to pull msgs from ntb print == tsim/tmq/consume.sh -d $dbName -y $pullDelay -g $showMsg -r $showRow -s start @@ -305,14 +305,14 @@ if $data[1][1] == 0 then endi endi -# either $data[0][2] == $totalMsgOfNtb and $data[1][2] == 0 -# or $data[0][2] == 0 and $data[1][2] == $totalMsgOfNtb -if $data[0][2] == $totalMsgOfNtb then +# either $data[0][2] == $expectmsgcnt and $data[1][2] == 0 +# or $data[0][2] == 0 and $data[1][2] == $expectmsgcnt +if $data[0][2] == $expectmsgcnt then if $data[1][2] == 0 then goto check_ok_2 endi elif $data[0][2] == 0 then - if $data[1][2] == $totalMsgOfNtb then + if $data[1][2] == $expectmsgcnt then goto check_ok_2 endi endi diff --git a/tests/script/tsim/valgrind/checkUdf.sim b/tests/script/tsim/valgrind/checkUdf.sim index dc703e155d3bacf2ed17648c0020192f5bf77eab..e316c104ed68bcc32bc4f84fa3ee803b5a91ae1a 100644 --- a/tests/script/tsim/valgrind/checkUdf.sim +++ b/tests/script/tsim/valgrind/checkUdf.sim @@ -29,10 +29,10 @@ sql select udf1(f) from t; if $rows != 2 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -51,10 +51,10 @@ sql select udf1(f1, f2) from t2; if $rows != 2 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -72,10 +72,10 @@ print $rows , $data00 , $data10 , $data20 , $data30 if $rows != 4 then return -1 endi -if $data00 != 88 then +if $data00 != 1 then return -1 endi -if $data10 != 88 then +if $data10 != 1 then return -1 endi @@ -114,10 +114,10 @@ print $rows , $data00 , $data01 if $rows != 1 then return -1 endi -if $data00 != 176.000000000 then +if $data00 != 2.000000000 then return -1 endi -if $data01 != 152.420471066 then +if $data01 != 1.732050808 then return -1 endi diff --git a/tests/system-test/0-others/udfTest.py b/tests/system-test/0-others/udfTest.py index da77078208d5f54eb8b4d09ab5e489bb9eb2f1ff..78020cb9586e6c59f7c0f84e5aeacdbb596b421c 100644 --- a/tests/system-test/0-others/udfTest.py +++ b/tests/system-test/0-others/udfTest.py @@ -191,20 +191,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -213,7 +213,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -226,13 +226,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -375,14 +375,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -401,9 +401,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -412,20 +412,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) diff --git a/tests/system-test/0-others/udf_cfg2.py b/tests/system-test/0-others/udf_cfg2.py index cc6da81847126d91e2aed8c14f11ebfbc0729b01..b535b4f626cdf92de1a406185d9c1af807ea6c7c 100644 --- a/tests/system-test/0-others/udf_cfg2.py +++ b/tests/system-test/0-others/udf_cfg2.py @@ -193,20 +193,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -228,13 +228,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -377,14 +377,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -403,9 +403,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -414,20 +414,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) diff --git a/tests/system-test/0-others/udf_create.py b/tests/system-test/0-others/udf_create.py index d35688c8dae3c419084e2f0414c5fb9031a4bea3..f467e802ac5d5f8adb15ef6b705d273e744270a3 100644 --- a/tests/system-test/0-others/udf_create.py +++ b/tests/system-test/0-others/udf_create.py @@ -193,20 +193,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -215,7 +215,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -228,13 +228,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -377,14 +377,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -403,9 +403,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -414,20 +414,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) diff --git a/tests/system-test/0-others/udf_restart_taosd.py b/tests/system-test/0-others/udf_restart_taosd.py index a0f70ccd49be9884ad1c0eeb5b7ee91c67ea4bee..61b6a4ea684f6150834cfc1e7c6ce1dc03a8b91b 100644 --- a/tests/system-test/0-others/udf_restart_taosd.py +++ b/tests/system-test/0-others/udf_restart_taosd.py @@ -190,20 +190,20 @@ class TDTestCase: tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) tdSql.checkData(0,2,1) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(0,4,1.000000000) - tdSql.checkData(0,5,88) + tdSql.checkData(0,5,1) tdSql.checkData(0,6,"binary1") - tdSql.checkData(0,7,88) + tdSql.checkData(0,7,1) tdSql.checkData(3,0,3) - tdSql.checkData(3,1,88) + tdSql.checkData(3,1,1) tdSql.checkData(3,2,33333) - tdSql.checkData(3,3,88) + tdSql.checkData(3,3,1) tdSql.checkData(3,4,33.000000000) - tdSql.checkData(3,5,88) + tdSql.checkData(3,5,1) tdSql.checkData(3,6,"binary1") - tdSql.checkData(3,7,88) + tdSql.checkData(3,7,1) tdSql.checkData(11,0,None) tdSql.checkData(11,1,None) @@ -212,7 +212,7 @@ class TDTestCase: tdSql.checkData(11,4,None) tdSql.checkData(11,5,None) tdSql.checkData(11,6,"binary1") - tdSql.checkData(11,7,88) + tdSql.checkData(11,7,1) tdSql.query("select c1 , udf1(c1) ,c2 ,udf1(c2), c3 ,udf1(c3), c4 ,udf1(c4) from stb1 order by c1") tdSql.checkData(0,0,None) @@ -225,13 +225,13 @@ class TDTestCase: tdSql.checkData(0,7,None) tdSql.checkData(20,0,8) - tdSql.checkData(20,1,88) + tdSql.checkData(20,1,1) tdSql.checkData(20,2,88888) - tdSql.checkData(20,3,88) + tdSql.checkData(20,3,1) tdSql.checkData(20,4,888) - tdSql.checkData(20,5,88) + tdSql.checkData(20,5,1) tdSql.checkData(20,6,88) - tdSql.checkData(20,7,88) + tdSql.checkData(20,7,1) # aggregate functions @@ -374,14 +374,14 @@ class TDTestCase: tdSql.checkRows(25) tdSql.checkData(0,0,None) tdSql.checkData(0,1,None) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,8) tdSql.query("select abs(udf1(c1)) , abs(ceil(c1)) from ct1 order by ts;") tdSql.checkRows(13) - tdSql.checkData(0,0,88) + tdSql.checkData(0,0,1) tdSql.checkData(0,1,8) - tdSql.checkData(1,0,88) + tdSql.checkData(1,0,1) tdSql.checkData(1,1,7) # bug fix for crash @@ -400,9 +400,9 @@ class TDTestCase: tdSql.query("select c1 ,udf1(c1) , c6 ,udf1(c6) from stb1 where c1 > 8 order by ts") tdSql.checkRows(3) tdSql.checkData(0,0,9) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,-99.990000000) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.query("select sub1.c1, sub2.c2 from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) @@ -411,20 +411,20 @@ class TDTestCase: tdSql.checkData(1,1,10) tdSql.query("select udf1(sub1.c1), udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") - tdSql.checkData(0,0,88) - tdSql.checkData(0,1,88) - tdSql.checkData(1,0,88) - tdSql.checkData(1,1,88) + tdSql.checkData(0,0,1) + tdSql.checkData(0,1,1) + tdSql.checkData(1,0,1) + tdSql.checkData(1,1,1) tdSql.query("select sub1.c1 , udf1(sub1.c1), sub2.c2 ,udf1(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,0) - tdSql.checkData(0,1,88) + tdSql.checkData(0,1,1) tdSql.checkData(0,2,0) - tdSql.checkData(0,3,88) + tdSql.checkData(0,3,1) tdSql.checkData(1,0,1) - tdSql.checkData(1,1,88) + tdSql.checkData(1,1,1) tdSql.checkData(1,2,10) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select udf2(sub1.c1), udf2(sub2.c2) from sub1, sub2 where sub1.ts=sub2.ts and sub1.c1 is not null") tdSql.checkData(0,0,16.881943016) @@ -468,12 +468,12 @@ class TDTestCase: tdSql.checkData(1,0,1) tdSql.checkData(1,1,1) tdSql.checkData(1,2,1.110000000) - tdSql.checkData(1,3,88) + tdSql.checkData(1,3,1) tdSql.query("select c1,c6,udf1(c1,c6) from stb1 order by ts") tdSql.checkData(1,0,8) tdSql.checkData(1,1,88.880000000) - tdSql.checkData(1,2,88) + tdSql.checkData(1,2,1) tdSql.query("select abs(udf1(c1,c6,c1,c6)) , abs(ceil(c1)) from stb1 where c1 is not null order by ts;") tdSql.checkRows(22) diff --git a/tests/system-test/0-others/user_control.py b/tests/system-test/0-others/user_control.py index d27892b588a832ff1edeb581b1975e9771510383..7b5578ba45f66bd3a993f0ce2ee666dcf802d3b3 100644 --- a/tests/system-test/0-others/user_control.py +++ b/tests/system-test/0-others/user_control.py @@ -444,7 +444,7 @@ class TDTestCase: self.__grant_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , self.__grant_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , f"GRANT {self.__privilege[0]} ON * TO {self.__user_list[0]}" , - f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" , + # f"GRANT {self.__privilege[0]} ON {DBNAME}.{NTBNAME} TO {self.__user_list[0]}" , ] def __revoke_err(self): @@ -456,7 +456,7 @@ class TDTestCase: self.__revoke_user_privileges(privilege="", dbname="db", user_name=self.__user_list[0]) , self.__revoke_user_privileges(privilege=" ".join(self.__privilege), user_name=self.__user_list[0]) , f"REVOKE {self.__privilege[0]} ON * FROM {self.__user_list[0]}" , - f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" , + # f"REVOKE {self.__privilege[0]} ON {DBNAME}.{NTBNAME} FROM {self.__user_list[0]}" , ] def test_grant_err(self): diff --git a/tests/system-test/1-insert/delete_childtable.py b/tests/system-test/1-insert/delete_childtable.py index e3144edb4594521bb849b579b9c264e44b156de5..a12f8849812bcf5d72bf24c654e92cc8c8da2cb8 100644 --- a/tests/system-test/1-insert/delete_childtable.py +++ b/tests/system-test/1-insert/delete_childtable.py @@ -27,7 +27,7 @@ class TDTestCase: def init(self, conn, logSql, replicaVar=1): self.replicaVar = int(replicaVar) tdLog.debug("start to execute %s" % __file__) - tdSql.init(conn.cursor()) + tdSql.init(conn.cursor(), True) self.dbname = 'db_test' self.setsql = TDSetSql() self.stbname = 'stb' diff --git a/tests/system-test/2-query/To_unixtimestamp.py b/tests/system-test/2-query/To_unixtimestamp.py index 8ee2007450328b2edbb02822a68fa4ac9d9ca9a7..424ebff6c5764dec9a2395eaa2254acfb47056c5 100644 --- a/tests/system-test/2-query/To_unixtimestamp.py +++ b/tests/system-test/2-query/To_unixtimestamp.py @@ -26,7 +26,7 @@ class TDTestCase: 'c1':'int', 'c2':'float', 'c3':'binary(20)' - + } # structure of tag self.tag_dict = { @@ -60,7 +60,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(self.values_list)*tb_num) + tdSql.checkRows(len(self.values_list)*tb_num) for time in ['2020-01-32T08:00:00','2020-13-32T08:00:00','acd']: tdSql.query(f"select to_unixtimestamp('{time}') from {tbname}") if tb_type == 'ntb' or tb_type == 'ctb': @@ -74,7 +74,7 @@ class TDTestCase: if tb_type == 'ntb' or tb_type == 'ctb': tdSql.checkRows(len(values_list)) elif tb_type == 'stb': - tdSql.checkRows(len(values_list)*tb_num) + tdSql.checkRows(len(values_list)*tb_num) for time in self.error_param: tdSql.error(f"select to_unixtimestamp({time}) from {tbname}") def timestamp_change_check_ntb(self): @@ -95,9 +95,20 @@ class TDTestCase: self.data_check(f'{self.stbname}_{i}',self.values_list,'ctb') self.data_check(self.stbname,self.values_list,'stb',self.tbnum) tdSql.execute(f'drop database {self.dbname}') + def timestamp_change_return_type(self): + tdSql.query(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 0);") + tdSql.checkEqual(tdSql.queryResult[0][0], 0) + tdSql.query(f"select to_unixtimestamp('1970-01-01 00:00:00', 1);") + tdSql.checkData(0, 0, '1970-01-01 00:00:00') + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 2);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1.5);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 'abc');") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', true);") + tdSql.error(f"select to_unixtimestamp('1970-01-01 08:00:00+08:00', 1, 3);") def run(self): # sourcery skip: extract-duplicate-method self.timestamp_change_check_ntb() self.timestamp_change_check_stb() + self.timestamp_change_return_type() def stop(self): tdSql.close() tdLog.success(f"{__file__} successfully executed") diff --git a/tests/system-test/2-query/interp.py b/tests/system-test/2-query/interp.py index d7344c631f5a150d3b9bac907b263bb2d41ec91e..ddf3f2534dd42d80b4b9a7da647748a8193902f0 100644 --- a/tests/system-test/2-query/interp.py +++ b/tests/system-test/2-query/interp.py @@ -19,6 +19,7 @@ class TDTestCase: dbname = "db" tbname = "tb" tbname1 = "tb1" + tbname2 = "tb2" stbname = "stb" ctbname1 = "ctb1" ctbname2 = "ctb2" @@ -1948,6 +1949,7 @@ class TDTestCase: tdSql.checkData(59, 1, 60) tdSql.checkData(60, 1, 60) # + tdLog.printNoPrefix("==========step11:test multi-interp cases") tdSql.query(f"select interp(c0),interp(c1),interp(c2),interp(c3) from {dbname}.{tbname} range('2020-02-09 00:00:05', '2020-02-13 00:00:05') every(1d) fill(null)") tdSql.checkRows(5) @@ -2002,7 +2004,362 @@ class TDTestCase: for i in range (tdSql.queryCols): tdSql.checkData(0, i, 13) - tdLog.printNoPrefix("==========step12:test error cases") + tdLog.printNoPrefix("==========step12:test interp with boolean type") + tdSql.execute( + f'''create table if not exists {dbname}.{tbname2} + (ts timestamp, c0 bool) + ''' + ) + + + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:01', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:03', true)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:05', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:07', true)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:09', true)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:11', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:13', false)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:15', NULL)") + tdSql.execute(f"insert into {dbname}.{tbname2} values ('2020-02-02 00:00:17', NULL)") + + # test fill null + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(NULL)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, None) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, None) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, None) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, None) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, None) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, None) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, None) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, None) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + # test fill prev + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(prev)") + tdSql.checkRows(18) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:01.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) + + tdSql.checkData(17, 0, '2020-02-02 00:00:18.000') + + # test fill next + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(next)") + tdSql.checkRows(18) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + tdSql.checkData(17, 2, None) + + tdSql.checkData(17, 0, '2020-02-02 00:00:17.000') + + # test fill value + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, 0)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, 1234)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, True) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, True) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, True) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, True) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, True) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, True) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, false)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, true)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, True) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, True) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, True) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, True) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, True) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, True) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, '0')") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, '123')") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, True) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, True) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, True) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, True) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, True) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, True) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, True) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, 'abc')") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(value, NULL)") + tdSql.checkRows(19) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:00.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, False) + tdSql.checkData(3, 2, True) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, False) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, False) + tdSql.checkData(9, 2, True) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, False) + tdSql.checkData(14, 2, False) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, False) + tdSql.checkData(17, 2, None) + tdSql.checkData(18, 2, False) + + tdSql.checkData(18, 0, '2020-02-02 00:00:18.000') + + # test fill linear + tdSql.query(f"select _irowts,_isfilled,interp(c0) from {dbname}.{tbname2} range('2020-02-02 00:00:00', '2020-02-02 00:00:18') every(1s) fill(linear)") + tdSql.checkRows(17) + tdSql.checkCols(3) + + tdSql.checkData(0, 0, '2020-02-02 00:00:01.000') + + tdSql.checkData(0, 2, False) + tdSql.checkData(1, 2, False) + tdSql.checkData(2, 2, True) + tdSql.checkData(3, 2, False) + tdSql.checkData(4, 2, False) + tdSql.checkData(5, 2, False) + tdSql.checkData(6, 2, True) + tdSql.checkData(7, 2, True) + tdSql.checkData(8, 2, True) + tdSql.checkData(9, 2, False) + tdSql.checkData(10, 2, False) + tdSql.checkData(11, 2, False) + tdSql.checkData(12, 2, False) + tdSql.checkData(13, 2, None) + tdSql.checkData(14, 2, None) + tdSql.checkData(15, 2, None) + tdSql.checkData(16, 2, None) + + tdSql.checkData(16, 0, '2020-02-02 00:00:17.000') + + tdLog.printNoPrefix("==========step13:test error cases") tdSql.error(f"select interp(c0) from {dbname}.{tbname}") tdSql.error(f"select interp(c0) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05')") @@ -2013,7 +2370,7 @@ class TDTestCase: # input can only be numerical types tdSql.error(f"select interp(ts) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") - tdSql.error(f"select interp(c6) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + #tdSql.error(f"select interp(c6) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp(c7) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp(c8) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") @@ -2025,6 +2382,14 @@ class TDTestCase: tdSql.error(f"select interp('abcd') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") tdSql.error(f"select interp('中文字符') from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + # invalid pseudo column usage + tdSql.error(f"select interp(_irowts) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(_isfilled) from {dbname}.{tbname} range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _isfilled = true range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + tdSql.error(f"select interp(c0) from {dbname}.{tbname} where _irowts > 0 range('2020-02-10 00:00:05', '2020-02-15 00:00:05') every(1d) fill(null)") + + + tdLog.printNoPrefix("==========step13:stable cases") @@ -2040,6 +2405,8 @@ class TDTestCase: #tdSql.query(f"select _irowts,interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:04', '2020-02-02 00:00:16') every(1h) fill(prev)") #tdSql.query(f"select tbname,_irowts,interp(c0) from {dbname}.{stbname} partition by tbname range('2020-02-01 00:00:04', '2020-02-02 00:00:16') every(1h) fill(prev)") + tdLog.printNoPrefix("======step 14: test interp pseudo columns") + tdSql.error(f"select _irowts, c6 from {dbname}.{tbname}") def stop(self): tdSql.close() diff --git a/tests/system-test/2-query/max.py b/tests/system-test/2-query/max.py index b8da02b9a6da6bfadc93d301d135464a1c7566f1..ba6ab53fc7c2503bfcdcd1b79474dc22f2717211 100644 --- a/tests/system-test/2-query/max.py +++ b/tests/system-test/2-query/max.py @@ -20,8 +20,8 @@ class TDTestCase: intData = [] floatData = [] tdSql.execute(f'''create table {dbname}.stb(ts timestamp, col1 tinyint, col2 smallint, col3 int, col4 bigint, col5 tinyint unsigned, col6 smallint unsigned, - col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(loc nchar(20))''') - tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags('beijing')") + col7 int unsigned, col8 bigint unsigned, col9 float, col10 double, col11 bool, col12 binary(20), col13 nchar(20)) tags(t0 tinyint, t1 float, loc nchar(20))''') + tdSql.execute(f"create table {dbname}.stb_1 using {dbname}.stb tags(5, 5.5, 'beijing')") for i in range(self.rowNum): tdSql.execute(f"insert into {dbname}.stb_1 values(%d, %d, %d, %d, %d, %d, %d, %d, %d, %f, %f, %d, '{self.binary_str}%d', '{self.nchar_str}%d')" % (self.ts + i, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 1, i + 0.1, i + 0.1, i % 2, i + 1, i + 1)) @@ -55,13 +55,20 @@ class TDTestCase: tdSql.checkData(0, 1, np.max(intData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) tdSql.query(f"select ts, min(col9) from {dbname}.stb_1") - tdSql.checkRows(1) + tdSql.checkRows(1) tdSql.checkData(0, 1, np.min(floatData)) + # check tags + tdSql.query(f"select max(t0) from {dbname}.stb") + tdSql.checkData(0,0,5) + + tdSql.query(f"select max(t1) from {dbname}.stb") + tdSql.checkData(0,0,5.5) + def max_check_ntb_base(self, dbname="db"): tdSql.prepare() intData = [] diff --git a/tests/system-test/2-query/odbc.py b/tests/system-test/2-query/odbc.py index f9232dddf8e68ce3122df84a414240bcbbd405e7..a05e05704279bfa4a96df9a755da3dcd78126a38 100644 --- a/tests/system-test/2-query/odbc.py +++ b/tests/system-test/2-query/odbc.py @@ -22,7 +22,7 @@ class TDTestCase: tdSql.execute("insert into db.ctb using db.stb tags(1) (ts, c1) values (now, 1)") tdSql.query("select count(*) from information_schema.ins_columns") - tdSql.checkData(0, 0, 274) + tdSql.checkData(0, 0, 277) tdSql.query("select * from information_schema.ins_columns where table_name = 'ntb'") tdSql.checkRows(14) diff --git a/tests/system-test/2-query/sml.py b/tests/system-test/2-query/sml.py index ec6309c71ad295eb504d8f97b493a062a044fed1..f96ed8a3ff3e6656c9b81e5f82efa1fed6d3bb36 100644 --- a/tests/system-test/2-query/sml.py +++ b/tests/system-test/2-query/sml.py @@ -24,7 +24,7 @@ class TDTestCase: tdSql.init(conn.cursor(), True) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, dbname="sml_db"): + def checkContent(self, dbname="sml_db"): simClientCfg="%s/taos.cfg"%tdDnodes.getSimCfgPath() buildPath = tdCom.getBuildPath() cmdStr = '%s/build/bin/sml_test %s'%(buildPath, simClientCfg) @@ -102,7 +102,7 @@ class TDTestCase: def run(self): tdSql.prepare() - self.checkFileContent() + self.checkContent() def stop(self): tdSql.close() diff --git a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py index fa22cad726ee700dd05350d0d03d5463bfbbcdec..7d11684ed81fd79584253e032f67cae1bd833de8 100644 --- a/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py +++ b/tests/system-test/7-tmq/dropDbR3ConflictTransaction.py @@ -32,34 +32,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', diff --git a/tests/system-test/7-tmq/subscribeStb.py b/tests/system-test/7-tmq/subscribeStb.py index c8b66adfa2db404941838e64170be41f27ad24b5..9dcbf5b351ff80023fc0eea9476d3d436364ff03 100644 --- a/tests/system-test/7-tmq/subscribeStb.py +++ b/tests/system-test/7-tmq/subscribeStb.py @@ -226,7 +226,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -307,7 +307,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/subscribeStb0.py b/tests/system-test/7-tmq/subscribeStb0.py index 717cf05bdc6d3dfa4d27c915d57fbe3bf61b8e60..06aa18c9b6c79a1ee11a4e236340023c08714137 100644 --- a/tests/system-test/7-tmq/subscribeStb0.py +++ b/tests/system-test/7-tmq/subscribeStb0.py @@ -228,7 +228,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -303,7 +303,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -315,7 +315,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -333,7 +333,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt: + if totalConsumeRows < expectrowcnt: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") @@ -386,7 +386,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -398,7 +398,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -416,7 +416,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != (expectrowcnt * (1 + 1/4)): + if totalConsumeRows < (expectrowcnt * (1 + 1/4)): tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb1.py b/tests/system-test/7-tmq/subscribeStb1.py index 6d4b7d2380161be0b4baaded864a782f8dbd86f5..25d09c38e2060dc0c5ef213a0c36a4bde4738866 100644 --- a/tests/system-test/7-tmq/subscribeStb1.py +++ b/tests/system-test/7-tmq/subscribeStb1.py @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -320,7 +320,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/subscribeStb2.py b/tests/system-test/7-tmq/subscribeStb2.py index 422cb23ffd1c503873cde2ca6b26ae6763a899c8..cdbc41a59351c0f67205244ffdae69f32bf59e6b 100644 --- a/tests/system-test/7-tmq/subscribeStb2.py +++ b/tests/system-test/7-tmq/subscribeStb2.py @@ -233,7 +233,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 10 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -251,6 +251,7 @@ class TDTestCase: tdLog.info("start consume 1 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -271,6 +272,7 @@ class TDTestCase: tdLog.info("start consume 2 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -338,7 +340,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 20 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -380,6 +382,7 @@ class TDTestCase: tdLog.info("start consume 2 processor") self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) + tdLog.sleep(2) tdLog.info("start one new thread to insert data") parameterDict['actionType'] = actionType.INSERT_DATA @@ -394,7 +397,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*2: + if totalConsumeRows < expectrowcnt*2: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*2)) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb3.py b/tests/system-test/7-tmq/subscribeStb3.py index 7205e846205e87cfa4589f01bc6a76c29fda4986..6f3230e68791c9c1806abf576936cd5878bd7b06 100644 --- a/tests/system-test/7-tmq/subscribeStb3.py +++ b/tests/system-test/7-tmq/subscribeStb3.py @@ -215,7 +215,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -233,7 +234,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume 0 processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -269,7 +270,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt-10000: + if totalConsumeRows < expectrowcnt-10000: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt-10000)) tdLog.exit("tmq consume rows error!") @@ -328,7 +329,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -346,7 +348,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -415,7 +417,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -433,7 +436,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -445,7 +448,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -467,7 +470,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -502,7 +505,8 @@ class TDTestCase: parameterDict["stbName"],\ parameterDict["ctbNum"],\ parameterDict["rowsPerTbl"],\ - parameterDict["batchNum"]) + parameterDict["batchNum"],\ + parameterDict["startTs"]) tdLog.info("create topics from stb1") topicFromStb1 = 'topic_stb1' @@ -520,7 +524,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt/4,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -532,7 +536,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt/4: + if totalConsumeRows < expectrowcnt/4: tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt/4)) tdLog.exit("tmq consume rows error!") @@ -555,7 +559,7 @@ class TDTestCase: for i in range(expectRows): totalConsumeRows += resultList[i] - if totalConsumeRows != expectrowcnt*(1/2+1/4): + if totalConsumeRows < expectrowcnt*(1/2+1/4): tdLog.info("act consume rows: %d, expect consume rows: %d"%(totalConsumeRows, expectrowcnt*(1/2+1/4))) tdLog.exit("tmq consume rows error!") diff --git a/tests/system-test/7-tmq/subscribeStb4.py b/tests/system-test/7-tmq/subscribeStb4.py index bb8afcf14ee300231e44c2766ac61d3420f237f1..ed467b09b32aa862eabd1ef22c1872aeedf8d7d0 100644 --- a/tests/system-test/7-tmq/subscribeStb4.py +++ b/tests/system-test/7-tmq/subscribeStb4.py @@ -231,7 +231,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) @@ -305,7 +305,7 @@ class TDTestCase: self.insertConsumerInfo(consumerId, expectrowcnt,topicList,keyList,ifcheckdata,ifManualCommit) tdLog.info("start consume processor") - pollDelay = 100 + pollDelay = 5 showMsg = 1 showRow = 1 self.startTmqSimProcess(buildPath,cfgPath,pollDelay,parameterDict["dbName"],showMsg, showRow) diff --git a/tests/system-test/7-tmq/tmq3mnodeSwitch.py b/tests/system-test/7-tmq/tmq3mnodeSwitch.py index 22ef8cebdc1f367d8d95010087e15b2eaf2f72e8..0740830696987a6009db020de14bdc768d56c69a 100644 --- a/tests/system-test/7-tmq/tmq3mnodeSwitch.py +++ b/tests/system-test/7-tmq/tmq3mnodeSwitch.py @@ -138,34 +138,6 @@ class TDTestCase: else: tdLog.exit("three mnodes is not ready in 10s ") - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -257,7 +229,7 @@ class TDTestCase: tdLog.exit("0 tmq consume rows error!") if expectRowsList[0] == resultList[0]: - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCheckData.py b/tests/system-test/7-tmq/tmqCheckData.py index a9671241a966ea632f31726349580a8e7152f440..cb5a40642aab7ba2053780b898fa498e2c8b49a3 100644 --- a/tests/system-test/7-tmq/tmqCheckData.py +++ b/tests/system-test/7-tmq/tmqCheckData.py @@ -5,6 +5,7 @@ import time import socket import os import threading +import math from util.log import * from util.sql import * @@ -21,34 +22,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -110,7 +83,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -136,7 +109,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -162,7 +135,7 @@ class TDTestCase: # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) # tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCheckData1.py b/tests/system-test/7-tmq/tmqCheckData1.py index e06c29c5a2770ec7f2e2a34d391495ee24399f0c..b4fec94dcc3a2a266a35400dc789ee13aa463caf 100644 --- a/tests/system-test/7-tmq/tmqCheckData1.py +++ b/tests/system-test/7-tmq/tmqCheckData1.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -110,7 +82,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -135,7 +107,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -160,7 +132,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) tdLog.exit("2 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/7-tmq/tmqCommon.py b/tests/system-test/7-tmq/tmqCommon.py index 1e636e207441e7794f312ffa00e27accde0eccbe..f63c70a4c68ba0d88149326f3cd15bab764b6165 100644 --- a/tests/system-test/7-tmq/tmqCommon.py +++ b/tests/system-test/7-tmq/tmqCommon.py @@ -10,7 +10,7 @@ ################################################################### # -*- coding: utf-8 -*- - +import math from asyncore import loop from collections import defaultdict import subprocess @@ -467,18 +467,24 @@ class TMQCom: for i in range(0,skipRowsOfCons): consumeFile.readline() - lines = 0 while True: dst = queryFile.readline() src = consumeFile.readline() - lines += 1 - if dst: - if dst != src: - tdLog.info("src row: %s"%src) - tdLog.info("dst row: %s"%dst) - tdLog.exit("consumerId %d consume rows[%d] is not match the rows by direct query"%(consumerId, lines)) - else: + dstSplit = dst.split(',') + srcSplit = src.split(',') + + if not dst or not src: break + if len(dstSplit) != len(srcSplit): + tdLog.exit("consumerId %d consume rows len is not match the rows by direct query,len(dstSplit):%d != len(srcSplit):%d, dst:%s, src:%s" + %(consumerId, len(dstSplit), len(srcSplit), dst, src)) + + for i in range(len(dstSplit)): + if srcSplit[i] != dstSplit[i]: + srcFloat = float(srcSplit[i]) + dstFloat = float(dstSplit[i]) + if not math.isclose(srcFloat, dstFloat, abs_tol=1e-9): + tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) return def getResultFileByTaosShell(self, consumerId, queryString): diff --git a/tests/system-test/7-tmq/tmqConsumerGroup.py b/tests/system-test/7-tmq/tmqConsumerGroup.py index b1aef9d762f7d994c0bfa02ab04b0c64b76edffa..d146dca4497b9f60d204ca067f3bcd299c204dae 100644 --- a/tests/system-test/7-tmq/tmqConsumerGroup.py +++ b/tests/system-test/7-tmq/tmqConsumerGroup.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', diff --git a/tests/system-test/7-tmq/tmqDelete-1ctb.py b/tests/system-test/7-tmq/tmqDelete-1ctb.py index 6dc8de7efce5659b46e386eabcd69130a1c08524..aa9c8d25d0364635b11ba6c867cad67f2085c08e 100644 --- a/tests/system-test/7-tmq/tmqDelete-1ctb.py +++ b/tests/system-test/7-tmq/tmqDelete-1ctb.py @@ -238,10 +238,10 @@ class TDTestCase: if self.snapshot == 0: consumerId = 2 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 3 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 + 1/4)) topicList = topicFromStb1 ifcheckdata = 1 @@ -271,7 +271,7 @@ class TDTestCase: if totalConsumeRows != expectrowcnt: tdLog.exit("tmq consume rows error with snapshot = 0!") elif self.snapshot == 1: - if totalConsumeRows != totalRowsFromQuery: + if totalConsumeRows != expectrowcnt: tdLog.exit("tmq consume rows error with snapshot = 1!") # tmqCom.checkFileContent(consumerId, queryString) @@ -324,7 +324,7 @@ class TDTestCase: if self.snapshot == 0: consumerId = 4 - expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1/4 + 3/4)) + expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"]) * 2 elif self.snapshot == 1: consumerId = 5 expectrowcnt = int(paraDict["rowsPerTbl"] * paraDict["ctbNum"] * (1 - 1/4 + 1/4 + 3/4)) @@ -370,11 +370,7 @@ class TDTestCase: tdLog.info("act consume rows: %d, act query rows: %d, expect consume rows: %d, "%(totalConsumeRows, totalRowsFromQuery, expectrowcnt)) if self.snapshot == 0: - # If data writing is completed before consumer get snapshot, will consume 7500 from wal; - # If data writing has not started before consumer get snapshot, will consume 10000 from wal; - minRows = int(expectrowcnt * (1 - 1/4)) # 7500 - tdLog.info("consume rows should be between %d and %d, "%(minRows, expectrowcnt)) - if not ((totalConsumeRows >= minRows) and (totalConsumeRows <= expectrowcnt)): + if (totalConsumeRows != expectrowcnt): tdLog.exit("tmq consume rows error with snapshot = 0!") elif self.snapshot == 1: tdLog.info("consume rows should be between %d and %d, "%(totalRowsFromQuery, expectrowcnt)) @@ -495,7 +491,7 @@ class TDTestCase: tdLog.printNoPrefix("======== test case 4 end ...... ") def run(self): - # tdSql.prepare() + tdSql.prepare() tdLog.printNoPrefix("=============================================") tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") self.snapshot = 0 @@ -521,11 +517,11 @@ class TDTestCase: self.prepareTestEnv() self.tmqCase3() - tdLog.printNoPrefix("=============================================") - tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") - self.snapshot = 0 - self.prepareTestEnv() - self.tmqCase4() + # tdLog.printNoPrefix("=============================================") + # tdLog.printNoPrefix("======== snapshot is 0: only consume from wal") + # self.snapshot = 0 + # self.prepareTestEnv() + # self.tmqCase4() tdLog.printNoPrefix("====================================================================") tdLog.printNoPrefix("======== snapshot is 1: firstly consume from tsbs, and then from wal") self.snapshot = 1 diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py index a923232706c62fa1343b65e936cbc7aac1b6df5f..bee38ca8ee192c542ebc2c09718d4e4224904c1e 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot0.py @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -201,7 +173,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -228,7 +200,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -312,7 +284,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -339,7 +311,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py index bee174376daca3c0244464c85f80124929e00ce8..d3b64d2b2150521c1a3bc98fe41734390609084f 100644 --- a/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py +++ b/tests/system-test/7-tmq/tmqUdf-multCtb-snapshot1.py @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -201,7 +173,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -228,7 +200,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -312,7 +284,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -339,7 +311,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) # tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/7-tmq/tmqUdf.py b/tests/system-test/7-tmq/tmqUdf.py index 5bb8e3034ceec89c1161b6a5c9bfd9eb5cfddc82..5da1625cb125091dddc6ddbe3e635dc404352bbb 100644 --- a/tests/system-test/7-tmq/tmqUdf.py +++ b/tests/system-test/7-tmq/tmqUdf.py @@ -60,34 +60,6 @@ class TDTestCase: tdLog.exit("create udf functions fail") return - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def prepareTestEnv(self): tdLog.printNoPrefix("======== prepare test env include database, stable, ctables, and insert data: ") paraDict = {'dbName': 'dbt', @@ -201,7 +173,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) @@ -229,7 +201,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) @@ -313,7 +285,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("2 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) # reinit consume info, and start tmq_sim, then check consume result @@ -340,7 +312,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("3 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) tdLog.printNoPrefix("consumerId %d check data ok!"%(consumerId)) time.sleep(10) diff --git a/tests/system-test/99-TDcase/TD-16821.py b/tests/system-test/99-TDcase/TD-16821.py index 78ac172f3075733aff7b3608fb29400b32adc01d..2e23002059c5d157b3e1f6edf2f21a5d291739bc 100644 --- a/tests/system-test/99-TDcase/TD-16821.py +++ b/tests/system-test/99-TDcase/TD-16821.py @@ -21,34 +21,6 @@ class TDTestCase: tdSql.init(conn.cursor()) #tdSql.init(conn.cursor(), logSql) # output sql.txt file - def checkFileContent(self, consumerId, queryString): - buildPath = tdCom.getBuildPath() - cfgPath = tdCom.getClientCfgPath() - dstFile = '%s/../log/dstrows_%d.txt'%(cfgPath, consumerId) - cmdStr = '%s/build/bin/taos -c %s -s "%s >> %s"'%(buildPath, cfgPath, queryString, dstFile) - tdLog.info(cmdStr) - os.system(cmdStr) - - consumeRowsFile = '%s/../log/consumerid_%d.txt'%(cfgPath, consumerId) - tdLog.info("rows file: %s, %s"%(consumeRowsFile, dstFile)) - - consumeFile = open(consumeRowsFile, mode='r') - queryFile = open(dstFile, mode='r') - - # skip first line for it is schema - queryFile.readline() - - while True: - dst = queryFile.readline() - src = consumeFile.readline() - - if dst: - if dst != src: - tdLog.exit("consumerId %d consume rows is not match the rows by direct query"%consumerId) - else: - break - return - def tmqCase1(self): tdLog.printNoPrefix("======== test case 1: ") paraDict = {'dbName': 'db1', @@ -114,7 +86,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[0], resultList[0])) tdLog.exit("0 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -140,7 +112,7 @@ class TDTestCase: tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[1], resultList[0])) tdLog.exit("1 tmq consume rows error!") - self.checkFileContent(consumerId, queryString) + tmqCom.checkFileContent(consumerId, queryString) # reinit consume info, and start tmq_sim, then check consume result tmqCom.initConsumerTable() @@ -166,7 +138,7 @@ class TDTestCase: # tdLog.info("expect consume rows: %d, act consume rows: %d"%(expectRowsList[2], resultList[0])) # tdLog.exit("2 tmq consume rows error!") - # self.checkFileContent(consumerId, queryString) + # tmqCom.checkFileContent(consumerId, queryString) time.sleep(10) for i in range(len(topicNameList)): diff --git a/tests/system-test/runAllOne.sh b/tests/system-test/runAllOne.sh new file mode 100644 index 0000000000000000000000000000000000000000..5a8d358d989355127f3971d1127aeb43fa0323bd --- /dev/null +++ b/tests/system-test/runAllOne.sh @@ -0,0 +1,688 @@ +# start -N3 +echo " ********** -N 3 *************" +python3 ./test.py -f 7-tmq/tmqDelete-multiCtb.py -N 3 -n 3 +python3 ./test.py -f 1-insert/alter_database.py -P +python3 ./test.py -f 1-insert/influxdb_line_taosc_insert.py -P +python3 ./test.py -f 1-insert/opentsdb_telnet_line_taosc_insert.py -P +python3 ./test.py -f 1-insert/opentsdb_json_taosc_insert.py -P +python3 ./test.py -f 1-insert/test_stmt_muti_insert_query.py -P +python3 ./test.py -f 1-insert/test_stmt_set_tbname_tag.py -P +python3 ./test.py -f 1-insert/alter_stable.py -P +python3 ./test.py -f 1-insert/alter_table.py -P +python3 ./test.py -f 1-insert/boundary.py -P +python3 ./test.py -f 1-insert/insertWithMoreVgroup.py -P +python3 ./test.py -f 1-insert/table_comment.py -P +python3 ./test.py -f 1-insert/time_range_wise.py -P +python3 ./test.py -f 1-insert/block_wise.py -P +python3 ./test.py -f 1-insert/create_retentions.py -P +python3 ./test.py -f 1-insert/mutil_stage.py -P +python3 ./test.py -f 1-insert/table_param_ttl.py -P +python3 ./test.py -f 1-insert/table_param_ttl.py -P -R +python3 ./test.py -f 1-insert/update_data_muti_rows.py -P +python3 ./test.py -f 1-insert/db_tb_name_check.py -P +python3 ./test.py -f 1-insert/InsertFuturets.py -P +python3 ./test.py -f 1-insert/insert_wide_column.py -P +python3 ./test.py -f 2-query/nestedQuery.py -P +python3 ./test.py -f 2-query/nestedQuery_str.py -P +python3 ./test.py -f 2-query/nestedQuery_math.py -P +python3 ./test.py -f 2-query/nestedQuery_time.py -P +python3 ./test.py -f 2-query/nestedQuery_26.py -P +python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 2 +python3 ./test.py -f 2-query/columnLenUpdated.py -P +python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 2 +python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 3 +python3 ./test.py -f 2-query/columnLenUpdated.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 4 +python3 ./test.py -f 7-tmq/tmqShow.py -P +python3 ./test.py -f 7-tmq/tmqDropStb.py -P +python3 ./test.py -f 7-tmq/subscribeStb0.py -P +python3 ./test.py -f 7-tmq/subscribeStb1.py -P +python3 ./test.py -f 7-tmq/subscribeStb2.py -P +python3 ./test.py -f 7-tmq/subscribeStb3.py -P +python3 ./test.py -f 7-tmq/subscribeDb0.py -P -N 3 -n 3 +python3 ./test.py -f 1-insert/delete_stable.py -P +python3 ./test.py -f 2-query/out_of_order.py -P -Q 3 +python3 ./test.py -f 2-query/out_of_order.py -P +python3 ./test.py -f 2-query/insert_null_none.py -P +python3 ./test.py -f 2-query/insert_null_none.py -P -R +python3 ./test.py -f 2-query/insert_null_none.py -P -Q 2 +python3 ./test.py -f 2-query/insert_null_none.py -P -Q 3 +python3 ./test.py -f 2-query/insert_null_none.py -P -Q 4 +python3 ./test.py -f 1-insert/database_pre_suf.py -P +python3 ./test.py -f 2-query/concat.py -P -Q 3 +python3 ./test.py -f 2-query/out_of_order.py -P -Q 2 +python3 ./test.py -f 2-query/out_of_order.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQuery.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_str.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_math.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_time.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQuery_26.py -P -Q 3 +python3 ./test.py -f 7-tmq/create_wrong_topic.py -P +python3 ./test.py -f 7-tmq/dropDbR3ConflictTransaction.py -P -N 3 +python3 ./test.py -f 7-tmq/basic5.py -P +python3 ./test.py -f 7-tmq/subscribeDb.py -P -N 3 -n 3 +python3 ./test.py -f 7-tmq/subscribeDb1.py -P +python3 ./test.py -f 7-tmq/subscribeDb2.py -P +python3 ./test.py -f 7-tmq/subscribeDb3.py -P +python3 ./test.py -f 7-tmq/subscribeDb4.py -P +python3 ./test.py -f 7-tmq/subscribeStb.py -P +python3 ./test.py -f 7-tmq/subscribeStb4.py -P +python3 ./test.py -f 7-tmq/db.py -P +python3 ./test.py -f 7-tmq/tmqError.py -P +python3 ./test.py -f 7-tmq/schema.py -P +python3 ./test.py -f 7-tmq/stbFilter.py -P +python3 ./test.py -f 7-tmq/tmqCheckData.py -P +python3 ./test.py -f 7-tmq/tmqCheckData1.py -P +python3 ./test.py -f 7-tmq/tmqConsumerGroup.py -P +python3 ./test.py -f 7-tmq/tmqAlterSchema.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb.py -P -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1.py -P -N 3 -n 3 +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-1ctb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb-mutilVg-mutilCtb.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-1ctb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb-funcNFilter.py -P +python3 ./test.py -f 7-tmq/tmqConsFromTsdb1-mutilVg-mutilCtb.py -P +python3 ./test.py -f 7-tmq/tmqAutoCreateTbl.py -P +python3 ./test.py -f 7-tmq/tmqDnodeRestart.py -P +python3 ./test.py -f 7-tmq/tmqDnodeRestart1.py -P +python3 ./test.py -f 7-tmq/tmqUpdate-1ctb.py -P +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot0.py -P +python3 ./test.py -f 7-tmq/tmqUpdate-multiCtb-snapshot1.py -P +python3 ./test.py -f 7-tmq/tmqDropStbCtb.py -P +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot0.py -P +python3 ./test.py -f 7-tmq/tmqDropNtb-snapshot1.py -P +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot0.py -P +python3 ./test.py -f 7-tmq/tmqUdf-multCtb-snapshot1.py -P +python3 ./test.py -f 7-tmq/stbTagFilter-1ctb.py -P +python3 ./test.py -f 7-tmq/dataFromTsdbNWal.py -P +python3 ./test.py -f 7-tmq/dataFromTsdbNWal-multiCtb.py -P +python3 ./test.py -f 7-tmq/tmq_taosx.py -P +python3 ./test.py -f 7-tmq/raw_block_interface_test.py -P +python3 ./test.py -f 7-tmq/stbTagFilter-multiCtb.py -P +python3 ./test.py -f 99-TDcase/TD-19201.py -P +python3 ./test.py -f 99-TDcase/TD-21561.py -P +python3 ./test.py -f 0-others/taosShell.py -P +python3 ./test.py -f 0-others/taosShellError.py -P +python3 ./test.py -f 0-others/taosShellNetChk.py -P +python3 ./test.py -f 0-others/telemetry.py -P +python3 ./test.py -f 0-others/backquote_check.py -P +python3 ./test.py -f 0-others/taosdMonitor.py -P +python3 ./test.py -f 0-others/udfTest.py -P +python3 ./test.py -f 0-others/udf_create.py -P +python3 ./test.py -f 0-others/udf_restart_taosd.py -P +python3 ./test.py -f 0-others/udf_cfg1.py -P +python3 ./test.py -f 0-others/udf_cfg2.py -P +python3 ./test.py -f 0-others/cachemodel.py -P +python3 ./test.py -f 0-others/sysinfo.py -P +python3 ./test.py -f 0-others/user_control.py -P +python3 ./test.py -f 0-others/user_manage.py -P +python3 ./test.py -f 0-others/fsync.py -P +python3 ./test.py -f 0-others/tag_index_basic.py -P +python3 ./test.py -f 0-others/show.py -P +python3 ./test.py -f 0-others/information_schema.py -P +python3 ./test.py -f 2-query/abs.py -P +python3 ./test.py -f 2-query/abs.py -P -R +python3 ./test.py -f 2-query/and_or_for_byte.py -P +python3 ./test.py -f 2-query/and_or_for_byte.py -P -R +python3 ./test.py -f 2-query/apercentile.py -P +python3 ./test.py -f 2-query/apercentile.py -P -R +python3 ./test.py -f 2-query/arccos.py -P +python3 ./test.py -f 2-query/arccos.py -P -R +python3 ./test.py -f 2-query/arcsin.py -P +python3 ./test.py -f 2-query/arcsin.py -P -R +python3 ./test.py -f 2-query/arctan.py -P +python3 ./test.py -f 2-query/arctan.py -P -R +python3 ./test.py -f 2-query/avg.py -P +python3 ./test.py -f 2-query/avg.py -P -R +python3 ./test.py -f 2-query/between.py -P +python3 ./test.py -f 2-query/between.py -P -R +python3 ./test.py -f 2-query/bottom.py -P +python3 ./test.py -f 2-query/bottom.py -P -R +python3 ./test.py -f 2-query/cast.py -P +python3 ./test.py -f 2-query/cast.py -P -R +python3 ./test.py -f 2-query/ceil.py -P +python3 ./test.py -f 2-query/ceil.py -P -R +python3 ./test.py -f 2-query/char_length.py -P +python3 ./test.py -f 2-query/char_length.py -P -R +python3 ./test.py -f 2-query/check_tsdb.py -P +python3 ./test.py -f 2-query/check_tsdb.py -P -R +python3 ./test.py -f 2-query/concat.py -P +python3 ./test.py -f 2-query/concat.py -P -R +python3 ./test.py -f 2-query/concat_ws.py -P +python3 ./test.py -f 2-query/concat_ws.py -P -R +python3 ./test.py -f 2-query/concat_ws2.py -P +python3 ./test.py -f 2-query/concat_ws2.py -P -R +python3 ./test.py -f 2-query/cos.py -P +python3 ./test.py -f 2-query/cos.py -P -R +python3 ./test.py -f 2-query/count_partition.py -P +python3 ./test.py -f 2-query/count_partition.py -P -R +python3 ./test.py -f 2-query/count.py -P +python3 ./test.py -f 2-query/count.py -P -R +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -R +python3 ./test.py -f 2-query/db.py -P +python3 ./test.py -f 2-query/diff.py -P +python3 ./test.py -f 2-query/diff.py -P -R +python3 ./test.py -f 2-query/distinct.py -P +python3 ./test.py -f 2-query/distinct.py -P -R +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -R +python3 ./test.py -f 2-query/distribute_agg_avg.py -P +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -R +python3 ./test.py -f 2-query/distribute_agg_count.py -P +python3 ./test.py -f 2-query/distribute_agg_count.py -P -R +python3 ./test.py -f 2-query/distribute_agg_max.py -P +python3 ./test.py -f 2-query/distribute_agg_max.py -P -R +python3 ./test.py -f 2-query/distribute_agg_min.py -P +python3 ./test.py -f 2-query/distribute_agg_min.py -P -R +python3 ./test.py -f 2-query/distribute_agg_spread.py -P +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -R +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -R +python3 ./test.py -f 2-query/distribute_agg_sum.py -P +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -R +python3 ./test.py -f 2-query/explain.py -P +python3 ./test.py -f 2-query/explain.py -P -R +python3 ./test.py -f 2-query/first.py -P +python3 ./test.py -f 2-query/first.py -P -R +python3 ./test.py -f 2-query/floor.py -P +python3 ./test.py -f 2-query/floor.py -P -R +python3 ./test.py -f 2-query/function_null.py -P +python3 ./test.py -f 2-query/function_null.py -P -R +python3 ./test.py -f 2-query/function_stateduration.py -P +python3 ./test.py -f 2-query/function_stateduration.py -P -R +python3 ./test.py -f 2-query/histogram.py -P +python3 ./test.py -f 2-query/histogram.py -P -R +python3 ./test.py -f 2-query/hyperloglog.py -P +python3 ./test.py -f 2-query/hyperloglog.py -P -R +python3 ./test.py -f 2-query/interp.py -P +python3 ./test.py -f 2-query/interp.py -P -R +python3 ./test.py -f 2-query/irate.py -P +python3 ./test.py -f 2-query/irate.py -P -R +python3 ./test.py -f 2-query/join.py -P +python3 ./test.py -f 2-query/join.py -P -R +python3 ./test.py -f 2-query/last_row.py -P +python3 ./test.py -f 2-query/last_row.py -P -R +python3 ./test.py -f 2-query/last.py -P +python3 ./test.py -f 2-query/last.py -P -R +python3 ./test.py -f 2-query/leastsquares.py -P +python3 ./test.py -f 2-query/leastsquares.py -P -R +python3 ./test.py -f 2-query/length.py -P +python3 ./test.py -f 2-query/length.py -P -R +python3 ./test.py -f 2-query/limit.py -P +python3 ./test.py -f 2-query/log.py -P +python3 ./test.py -f 2-query/log.py -P -R +python3 ./test.py -f 2-query/lower.py -P +python3 ./test.py -f 2-query/lower.py -P -R +python3 ./test.py -f 2-query/ltrim.py -P +python3 ./test.py -f 2-query/ltrim.py -P -R +python3 ./test.py -f 2-query/mavg.py -P +python3 ./test.py -f 2-query/mavg.py -P -R +python3 ./test.py -f 2-query/max_partition.py -P +python3 ./test.py -f 2-query/max_partition.py -P -R +python3 ./test.py -f 2-query/max_min_last_interval.py -P +python3 ./test.py -f 2-query/last_row_interval.py -P +python3 ./test.py -f 2-query/max.py -P +python3 ./test.py -f 2-query/max.py -P -R +python3 ./test.py -f 2-query/min.py -P +python3 ./test.py -f 2-query/min.py -P -R +python3 ./test.py -f 2-query/mode.py -P +python3 ./test.py -f 2-query/mode.py -P -R +python3 ./test.py -f 2-query/Now.py -P +python3 ./test.py -f 2-query/Now.py -P -R +python3 ./test.py -f 2-query/percentile.py -P +python3 ./test.py -f 2-query/percentile.py -P -R +python3 ./test.py -f 2-query/pow.py -P +python3 ./test.py -f 2-query/pow.py -P -R +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -R +python3 ./test.py -f 2-query/round.py -P +python3 ./test.py -f 2-query/round.py -P -R +python3 ./test.py -f 2-query/rtrim.py -P +python3 ./test.py -f 2-query/rtrim.py -P -R +python3 ./test.py -f 1-insert/drop.py -P -N 3 -M 3 -i False -n 3 +python3 ./test.py -f 7-tmq/tmqUpdateWithConsume.py -P -N 3 -n 3 +python3 ./test.py -f 2-query/db.py -P -N 3 -n 3 -R +python3 ./test.py -f 2-query/stablity.py -P +python3 ./test.py -f 2-query/stablity_1.py -P +python3 ./test.py -f 2-query/elapsed.py -P +python3 ./test.py -f 2-query/csum.py -P +python3 ./test.py -f 2-query/function_diff.py -P +python3 ./test.py -f 2-query/tagFilter.py -P +python3 ./test.py -f 2-query/projectionDesc.py -P +python3 ./test.py -f 2-query/queryQnode.py -P +python3 ./test.py -f 6-cluster/5dnode1mnode.py -P + + +# -N 4 +echo " ********** -N 4 *************" +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_createDb_replica1.py -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica1_insertdatas_querys.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_insertdatas_querys.py -P -N 4 -M 1 +python3 ./test.py -f 6-cluster/vnode/4dnode1mnode_basic_replica3_vgroups.py -P -N 4 -M 1 +python3 ./test.py -f 2-query/varchar.py -P -R +python3 ./test.py -f 2-query/case_when.py -P +python3 ./test.py -f 2-query/case_when.py -P -R +python3 ./test.py -f 2-query/blockSMA.py -P +python3 ./test.py -f 2-query/blockSMA.py -P -R +python3 ./test.py -f 2-query/projectionDesc.py -P +python3 ./test.py -f 2-query/projectionDesc.py -P -R +python3 ./test.py -f 1-insert/update_data.py -P +python3 ./test.py -f 1-insert/tb_100w_data_order.py -P +python3 ./test.py -f 1-insert/delete_childtable.py -P +python3 ./test.py -f 1-insert/delete_normaltable.py -P +python3 ./test.py -f 1-insert/keep_expired.py -P +python3 ./test.py -f 1-insert/drop.py -P +python3 ./test.py -f 2-query/join2.py -P +python3 ./test.py -f 2-query/union1.py -P +python3 ./test.py -f 2-query/concat2.py -P +python3 ./test.py -f 2-query/json_tag.py -P +python3 ./test.py -f 2-query/nestedQueryInterval.py -P + + +# -N 5 +echo " ********** -N 5 *************" +python3 ./test.py -f 6-cluster/5dnode2mnode.py -P -N 5 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -P -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 -i False +python3 ./test.py -f 6-cluster/5dnode3mnodeStopLoop.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRecreateMnode.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStopFollowerLeader.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop2Follower.py -P -N 5 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeStop.py -P -N 5 -M 3 +python3 ./test.py -f 0-others/taosdShell.py -P -N 5 -M 3 -Q 3 +python3 ./test.py -f 7-tmq/tmqSubscribeStb-r3.py -P -N 5 +python3 ./test.py -f 2-query/timezone.py -P +python3 ./test.py -f 2-query/timezone.py -P -R +python3 ./test.py -f 2-query/To_iso8601.py -P +python3 ./test.py -f 2-query/To_iso8601.py -P -R +python3 ./test.py -f 2-query/To_unixtimestamp.py -P +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -R +python3 ./test.py -f 2-query/Today.py -P +python3 ./test.py -f 2-query/Today.py -P -R +python3 ./test.py -f 2-query/top.py -P +python3 ./test.py -f 2-query/top.py -P -R +python3 ./test.py -f 2-query/tsbsQuery.py -P +python3 ./test.py -f 2-query/tsbsQuery.py -P -R +python3 ./test.py -f 2-query/ttl_comment.py -P +python3 ./test.py -f 2-query/ttl_comment.py -P -R +python3 ./test.py -f 2-query/twa.py -P +python3 ./test.py -f 2-query/twa.py -P -R +python3 ./test.py -f 2-query/union.py -P +python3 ./test.py -f 2-query/union.py -P -R +python3 ./test.py -f 2-query/unique.py -P +python3 ./test.py -f 2-query/unique.py -P -R +python3 ./test.py -f 2-query/upper.py -P +python3 ./test.py -f 2-query/upper.py -P -R +python3 ./test.py -f 2-query/varchar.py -P + +# -N6 +echo " ********** -N 6 *************" +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateDb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateDb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateDb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeModifyMeta.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeModifyMeta.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopDnodeCreateStb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopMnodeCreateStb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeSep1VnodeStopVnodeCreateStb.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertData.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -P -N 6 -M 3 +python3 ./test.py -f 6-cluster/5dnode3mnodeRestartDnodeInsertDataAsync.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -P -N 6 -M 3 +python3 ./test.py -f 7-tmq/tmq3mnodeSwitch.py -P -N 6 -M 3 -n 3 +python3 ./test.py -f 0-others/balance_vgroups_r1.py -P -N 6 + +python3 ./test.py -f 2-query/sample.py -P +python3 ./test.py -f 2-query/sample.py -P -R +python3 ./test.py -f 2-query/sin.py -P +python3 ./test.py -f 2-query/sin.py -P -R +python3 ./test.py -f 2-query/smaTest.py -P +python3 ./test.py -f 2-query/smaTest.py -P -R +python3 ./test.py -f 2-query/sml.py -P +python3 ./test.py -f 2-query/sml.py -P -R +python3 ./test.py -f 2-query/spread.py -P +python3 ./test.py -f 2-query/spread.py -P -R +python3 ./test.py -f 2-query/sqrt.py -P +python3 ./test.py -f 2-query/sqrt.py -P -R +python3 ./test.py -f 2-query/statecount.py -P +python3 ./test.py -f 2-query/statecount.py -P -R +python3 ./test.py -f 2-query/stateduration.py -P +python3 ./test.py -f 2-query/stateduration.py -P -R +python3 ./test.py -f 2-query/substr.py -P +python3 ./test.py -f 2-query/substr.py -P -R +python3 ./test.py -f 2-query/sum.py -P +python3 ./test.py -f 2-query/sum.py -P -R +python3 ./test.py -f 2-query/tail.py -P +python3 ./test.py -f 2-query/tail.py -P -R +python3 ./test.py -f 2-query/tan.py -P +python3 ./test.py -f 2-query/tan.py -P -R +python3 ./test.py -f 2-query/Timediff.py -P +python3 ./test.py -f 2-query/Timediff.py -P -R +python3 ./test.py -f 2-query/timetruncate.py -P +python3 ./test.py -f 2-query/timetruncate.py -P -R + +# N-7 +echo " ********** -N 7 *************" +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -N 7 -M 3 -C 6 +python3 ./test.py -f 6-cluster/5dnode3mnodeAdd1Ddnoe.py -P -N 7 -M 3 -C 6 -n 3 + +python3 ./test.py -f 2-query/between.py -P -Q 2 +python3 ./test.py -f 2-query/distinct.py -P -Q 2 +python3 ./test.py -f 2-query/varchar.py -P -Q 2 +python3 ./test.py -f 2-query/ltrim.py -P -Q 2 +python3 ./test.py -f 2-query/rtrim.py -P -Q 2 +python3 ./test.py -f 2-query/length.py -P -Q 2 +python3 ./test.py -f 2-query/char_length.py -P -Q 2 +python3 ./test.py -f 2-query/upper.py -P -Q 2 +python3 ./test.py -f 2-query/lower.py -P -Q 2 +python3 ./test.py -f 2-query/join.py -P -Q 2 +python3 ./test.py -f 2-query/join2.py -P -Q 2 +python3 ./test.py -f 2-query/cast.py -P -Q 2 +python3 ./test.py -f 2-query/substr.py -P -Q 2 +python3 ./test.py -f 2-query/union.py -P -Q 2 +python3 ./test.py -f 2-query/union1.py -P -Q 2 +python3 ./test.py -f 2-query/concat.py -P -Q 2 +python3 ./test.py -f 2-query/concat2.py -P -Q 2 +python3 ./test.py -f 2-query/concat_ws.py -P -Q 2 +python3 ./test.py -f 2-query/concat_ws2.py -P -Q 2 +python3 ./test.py -f 2-query/check_tsdb.py -P -Q 2 +python3 ./test.py -f 2-query/spread.py -P -Q 2 +python3 ./test.py -f 2-query/hyperloglog.py -P -Q 2 +python3 ./test.py -f 2-query/explain.py -P -Q 2 +python3 ./test.py -f 2-query/leastsquares.py -P -Q 2 +python3 ./test.py -f 2-query/timezone.py -P -Q 2 +python3 ./test.py -f 2-query/Now.py -P -Q 2 +python3 ./test.py -f 2-query/Today.py -P -Q 2 +python3 ./test.py -f 2-query/max.py -P -Q 2 +python3 ./test.py -f 2-query/min.py -P -Q 2 +python3 ./test.py -f 2-query/mode.py -P -Q 2 +python3 ./test.py -f 2-query/count.py -P -Q 2 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 2 +python3 ./test.py -f 2-query/last.py -P -Q 2 +python3 ./test.py -f 2-query/first.py -P -Q 2 +python3 ./test.py -f 2-query/To_iso8601.py -P -Q 2 +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 2 +python3 ./test.py -f 2-query/timetruncate.py -P -Q 2 +python3 ./test.py -f 2-query/diff.py -P -Q 2 +python3 ./test.py -f 2-query/Timediff.py -P -Q 2 +python3 ./test.py -f 2-query/json_tag.py -P -Q 2 +python3 ./test.py -f 2-query/top.py -P -Q 2 +python3 ./test.py -f 2-query/bottom.py -P -Q 2 +python3 ./test.py -f 2-query/percentile.py -P -Q 2 +python3 ./test.py -f 2-query/apercentile.py -P -Q 2 +python3 ./test.py -f 2-query/abs.py -P -Q 2 +python3 ./test.py -f 2-query/ceil.py -P -Q 2 +python3 ./test.py -f 2-query/floor.py -P -Q 2 +python3 ./test.py -f 2-query/round.py -P -Q 2 +python3 ./test.py -f 2-query/log.py -P -Q 2 +python3 ./test.py -f 2-query/pow.py -P -Q 2 +python3 ./test.py -f 2-query/sqrt.py -P -Q 2 +python3 ./test.py -f 2-query/sin.py -P -Q 2 +python3 ./test.py -f 2-query/cos.py -P -Q 2 +python3 ./test.py -f 2-query/tan.py -P -Q 2 +python3 ./test.py -f 2-query/arcsin.py -P -Q 2 +python3 ./test.py -f 2-query/arccos.py -P -Q 2 +python3 ./test.py -f 2-query/arctan.py -P -Q 2 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 2 +python3 ./test.py -f 2-query/interp.py -P -Q 2 +python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 2 +python3 ./test.py -f 2-query/stablity.py -P -Q 2 +python3 ./test.py -f 2-query/stablity_1.py -P -Q 2 +python3 ./test.py -f 2-query/avg.py -P -Q 2 +python3 ./test.py -f 2-query/elapsed.py -P -Q 2 +python3 ./test.py -f 2-query/csum.py -P -Q 2 +python3 ./test.py -f 2-query/mavg.py -P -Q 2 +python3 ./test.py -f 2-query/sample.py -P -Q 2 +python3 ./test.py -f 2-query/function_diff.py -P -Q 2 +python3 ./test.py -f 2-query/unique.py -P -Q 2 +python3 ./test.py -f 2-query/stateduration.py -P -Q 2 +python3 ./test.py -f 2-query/function_stateduration.py -P -Q 2 +python3 ./test.py -f 2-query/statecount.py -P -Q 2 +python3 ./test.py -f 2-query/tail.py -P -Q 2 +python3 ./test.py -f 2-query/ttl_comment.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 2 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 2 +python3 ./test.py -f 2-query/twa.py -P -Q 2 +python3 ./test.py -f 2-query/irate.py -P -Q 2 +python3 ./test.py -f 2-query/function_null.py -P -Q 2 +python3 ./test.py -f 2-query/count_partition.py -P -Q 2 +python3 ./test.py -f 2-query/max_partition.py -P -Q 2 +python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 2 +python3 ./test.py -f 2-query/last_row_interval.py -P -Q 2 +python3 ./test.py -f 2-query/last_row.py -P -Q 2 +python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 2 +python3 ./test.py -f 2-query/sml.py -P -Q 2 +python3 ./test.py -f 2-query/case_when.py -P -Q 2 +python3 ./test.py -f 2-query/blockSMA.py -P -Q 2 +python3 ./test.py -f 2-query/projectionDesc.py -P -Q 2 +python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 2 + +python3 ./test.py -f 2-query/between.py -P -Q 3 +python3 ./test.py -f 2-query/distinct.py -P -Q 3 +python3 ./test.py -f 2-query/varchar.py -P -Q 3 +python3 ./test.py -f 2-query/ltrim.py -P -Q 3 +python3 ./test.py -f 2-query/rtrim.py -P -Q 3 +python3 ./test.py -f 2-query/length.py -P -Q 3 +python3 ./test.py -f 2-query/char_length.py -P -Q 3 +python3 ./test.py -f 2-query/upper.py -P -Q 3 +python3 ./test.py -f 2-query/lower.py -P -Q 3 +python3 ./test.py -f 2-query/join.py -P -Q 3 +python3 ./test.py -f 2-query/join2.py -P -Q 3 +python3 ./test.py -f 2-query/cast.py -P -Q 3 +python3 ./test.py -f 2-query/substr.py -P -Q 3 +python3 ./test.py -f 2-query/union.py -P -Q 3 +python3 ./test.py -f 2-query/union1.py -P -Q 3 +python3 ./test.py -f 2-query/concat2.py -P -Q 3 +python3 ./test.py -f 2-query/concat_ws.py -P -Q 3 +python3 ./test.py -f 2-query/concat_ws2.py -P -Q 3 +python3 ./test.py -f 2-query/check_tsdb.py -P -Q 3 +python3 ./test.py -f 2-query/spread.py -P -Q 3 +python3 ./test.py -f 2-query/hyperloglog.py -P -Q 3 +python3 ./test.py -f 2-query/explain.py -P -Q 3 +python3 ./test.py -f 2-query/leastsquares.py -P -Q 3 +python3 ./test.py -f 2-query/timezone.py -P -Q 3 +python3 ./test.py -f 2-query/Now.py -P -Q 3 +python3 ./test.py -f 2-query/Today.py -P -Q 3 +python3 ./test.py -f 2-query/max.py -P -Q 3 +python3 ./test.py -f 2-query/min.py -P -Q 3 +python3 ./test.py -f 2-query/mode.py -P -Q 3 +python3 ./test.py -f 2-query/count.py -P -Q 3 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 3 +python3 ./test.py -f 2-query/last.py -P -Q 3 +python3 ./test.py -f 2-query/first.py -P -Q 3 +python3 ./test.py -f 2-query/To_iso8601.py -P -Q 3 +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 3 +python3 ./test.py -f 2-query/timetruncate.py -P -Q 3 +python3 ./test.py -f 2-query/diff.py -P -Q 3 +python3 ./test.py -f 2-query/Timediff.py -P -Q 3 +python3 ./test.py -f 2-query/json_tag.py -P -Q 3 +python3 ./test.py -f 2-query/top.py -P -Q 3 +python3 ./test.py -f 2-query/bottom.py -P -Q 3 +python3 ./test.py -f 2-query/percentile.py -P -Q 3 +python3 ./test.py -f 2-query/apercentile.py -P -Q 3 +python3 ./test.py -f 2-query/abs.py -P -Q 3 +python3 ./test.py -f 2-query/ceil.py -P -Q 3 +python3 ./test.py -f 2-query/floor.py -P -Q 3 +python3 ./test.py -f 2-query/round.py -P -Q 3 +python3 ./test.py -f 2-query/log.py -P -Q 3 +python3 ./test.py -f 2-query/pow.py -P -Q 3 +python3 ./test.py -f 2-query/sqrt.py -P -Q 3 +python3 ./test.py -f 2-query/sin.py -P -Q 3 +python3 ./test.py -f 2-query/cos.py -P -Q 3 +python3 ./test.py -f 2-query/tan.py -P -Q 3 +python3 ./test.py -f 2-query/arcsin.py -P -Q 3 +python3 ./test.py -f 2-query/arccos.py -P -Q 3 +python3 ./test.py -f 2-query/arctan.py -P -Q 3 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 3 +python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 3 +python3 ./test.py -f 2-query/stablity.py -P -Q 3 +python3 ./test.py -f 2-query/stablity_1.py -P -Q 3 +python3 ./test.py -f 2-query/avg.py -P -Q 3 +python3 ./test.py -f 2-query/elapsed.py -P -Q 3 +python3 ./test.py -f 2-query/csum.py -P -Q 3 +python3 ./test.py -f 2-query/mavg.py -P -Q 3 +python3 ./test.py -f 2-query/sample.py -P -Q 3 +python3 ./test.py -f 2-query/function_diff.py -P -Q 3 +python3 ./test.py -f 2-query/unique.py -P -Q 3 +python3 ./test.py -f 2-query/stateduration.py -P -Q 3 +python3 ./test.py -f 2-query/function_stateduration.py -P -Q 3 +python3 ./test.py -f 2-query/statecount.py -P -Q 3 +python3 ./test.py -f 2-query/tail.py -P -Q 3 +python3 ./test.py -f 2-query/ttl_comment.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 3 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 3 +python3 ./test.py -f 2-query/twa.py -P -Q 3 +python3 ./test.py -f 2-query/irate.py -P -Q 3 +python3 ./test.py -f 2-query/function_null.py -P -Q 3 +python3 ./test.py -f 2-query/count_partition.py -P -Q 3 +python3 ./test.py -f 2-query/max_partition.py -P -Q 3 +python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 3 +python3 ./test.py -f 2-query/last_row_interval.py -P -Q 3 +python3 ./test.py -f 2-query/last_row.py -P -Q 3 +python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 3 +python3 ./test.py -f 2-query/sml.py -P -Q 3 +python3 ./test.py -f 2-query/interp.py -P -Q 3 +python3 ./test.py -f 2-query/case_when.py -P -Q 3 +python3 ./test.py -f 2-query/blockSMA.py -P -Q 3 +python3 ./test.py -f 2-query/projectionDesc.py -P -Q 3 +python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 3 +python3 ./test.py -f 2-query/between.py -P -Q 4 +python3 ./test.py -f 2-query/distinct.py -P -Q 4 +python3 ./test.py -f 2-query/varchar.py -P -Q 4 +python3 ./test.py -f 2-query/ltrim.py -P -Q 4 +python3 ./test.py -f 2-query/rtrim.py -P -Q 4 +python3 ./test.py -f 2-query/length.py -P -Q 4 +python3 ./test.py -f 2-query/char_length.py -P -Q 4 +python3 ./test.py -f 2-query/upper.py -P -Q 4 +python3 ./test.py -f 2-query/lower.py -P -Q 4 +python3 ./test.py -f 2-query/join.py -P -Q 4 +python3 ./test.py -f 2-query/join2.py -P -Q 4 +python3 ./test.py -f 2-query/substr.py -P -Q 4 +python3 ./test.py -f 2-query/union.py -P -Q 4 +python3 ./test.py -f 2-query/union1.py -P -Q 4 +python3 ./test.py -f 2-query/concat.py -P -Q 4 +python3 ./test.py -f 2-query/concat2.py -P -Q 4 +python3 ./test.py -f 2-query/concat_ws.py -P -Q 4 +python3 ./test.py -f 2-query/concat_ws2.py -P -Q 4 +python3 ./test.py -f 2-query/check_tsdb.py -P -Q 4 +python3 ./test.py -f 2-query/spread.py -P -Q 4 +python3 ./test.py -f 2-query/hyperloglog.py -P -Q 4 +python3 ./test.py -f 2-query/explain.py -P -Q 4 +python3 ./test.py -f 2-query/leastsquares.py -P -Q 4 +python3 ./test.py -f 2-query/timezone.py -P -Q 4 +python3 ./test.py -f 2-query/Now.py -P -Q 4 +python3 ./test.py -f 2-query/Today.py -P -Q 4 +python3 ./test.py -f 2-query/max.py -P -Q 4 +python3 ./test.py -f 2-query/min.py -P -Q 4 +python3 ./test.py -f 2-query/mode.py -P -Q 4 +python3 ./test.py -f 2-query/count.py -P -Q 4 +python3 ./test.py -f 2-query/countAlwaysReturnValue.py -P -Q 4 +python3 ./test.py -f 2-query/last.py -P -Q 4 +python3 ./test.py -f 2-query/first.py -P -Q 4 +python3 ./test.py -f 2-query/To_iso8601.py -P -Q 4 +python3 ./test.py -f 2-query/To_unixtimestamp.py -P -Q 4 +python3 ./test.py -f 2-query/timetruncate.py -P -Q 4 +python3 ./test.py -f 2-query/diff.py -P -Q 4 +python3 ./test.py -f 2-query/Timediff.py -P -Q 4 +python3 ./test.py -f 2-query/json_tag.py -P -Q 4 +python3 ./test.py -f 2-query/top.py -P -Q 4 +python3 ./test.py -f 2-query/bottom.py -P -Q 4 +python3 ./test.py -f 2-query/percentile.py -P -Q 4 +python3 ./test.py -f 2-query/apercentile.py -P -Q 4 +python3 ./test.py -f 2-query/abs.py -P -Q 4 +python3 ./test.py -f 2-query/ceil.py -P -Q 4 +python3 ./test.py -f 2-query/floor.py -P -Q 4 +python3 ./test.py -f 2-query/round.py -P -Q 4 +python3 ./test.py -f 2-query/log.py -P -Q 4 +python3 ./test.py -f 2-query/pow.py -P -Q 4 +python3 ./test.py -f 2-query/sqrt.py -P -Q 4 +python3 ./test.py -f 2-query/sin.py -P -Q 4 +python3 ./test.py -f 2-query/cos.py -P -Q 4 +python3 ./test.py -f 2-query/tan.py -P -Q 4 +python3 ./test.py -f 2-query/arcsin.py -P -Q 4 +python3 ./test.py -f 2-query/arccos.py -P -Q 4 +python3 ./test.py -f 2-query/arctan.py -P -Q 4 +python3 ./test.py -f 2-query/query_cols_tags_and_or.py -P -Q 4 +python3 ./test.py -f 2-query/nestedQueryInterval.py -P -Q 4 +python3 ./test.py -f 2-query/stablity.py -P -Q 4 +python3 ./test.py -f 2-query/stablity_1.py -P -Q 4 +python3 ./test.py -f 2-query/avg.py -P -Q 4 +python3 ./test.py -f 2-query/elapsed.py -P -Q 4 +python3 ./test.py -f 2-query/csum.py -P -Q 4 +python3 ./test.py -f 2-query/mavg.py -P -Q 4 +python3 ./test.py -f 2-query/sample.py -P -Q 4 +python3 ./test.py -f 2-query/cast.py -P -Q 4 +python3 ./test.py -f 2-query/function_diff.py -P -Q 4 +python3 ./test.py -f 2-query/unique.py -P -Q 4 +python3 ./test.py -f 2-query/stateduration.py -P -Q 4 +python3 ./test.py -f 2-query/function_stateduration.py -P -Q 4 +python3 ./test.py -f 2-query/statecount.py -P -Q 4 +python3 ./test.py -f 2-query/tail.py -P -Q 4 +python3 ./test.py -f 2-query/ttl_comment.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_count.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_max.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_min.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_sum.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_spread.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_apercentile.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_avg.py -P -Q 4 +python3 ./test.py -f 2-query/distribute_agg_stddev.py -P -Q 4 +python3 ./test.py -f 2-query/twa.py -P -Q 4 +python3 ./test.py -f 2-query/irate.py -P -Q 4 +python3 ./test.py -f 2-query/function_null.py -P -Q 4 +python3 ./test.py -f 2-query/count_partition.py -P -Q 4 +python3 ./test.py -f 2-query/max_partition.py -P -Q 4 +python3 ./test.py -f 2-query/max_min_last_interval.py -P -Q 4 +python3 ./test.py -f 2-query/last_row_interval.py -P -Q 4 +python3 ./test.py -f 2-query/last_row.py -P -Q 4 +python3 ./test.py -f 2-query/tsbsQuery.py -P -Q 4 +python3 ./test.py -f 2-query/sml.py -P -Q 4 +python3 ./test.py -f 2-query/interp.py -P -Q 4 +python3 ./test.py -f 2-query/case_when.py -P -Q 4 +python3 ./test.py -f 2-query/insert_select.py -P +python3 ./test.py -f 2-query/insert_select.py -P -R +python3 ./test.py -f 2-query/insert_select.py -P -Q 2 +python3 ./test.py -f 2-query/insert_select.py -P -Q 3 +python3 ./test.py -f 2-query/insert_select.py -P -Q 4 +python3 ./test.py -f 2-query/out_of_order.py -P -R +python3 ./test.py -f 2-query/blockSMA.py -P -Q 4 +python3 ./test.py -f 2-query/projectionDesc.py -P -Q 4 +python3 ./test.py -f 2-query/odbc.py -P +python3 ./test.py -f 99-TDcase/TD-21561.py -P -Q 4 +python3 ./test.py -f 99-TDcase/TD-20582.py -P \ No newline at end of file diff --git a/tests/system-test/test.py b/tests/system-test/test.py index bc7f1d9861c6128de2bcbef00ddd2c77c0df5a33..7082bb0f22f0ae2ab98a5c8ca8e87f895e1dea54 100644 --- a/tests/system-test/test.py +++ b/tests/system-test/test.py @@ -13,6 +13,7 @@ # pip install src/connector/python/ # -*- coding: utf-8 -*- +import os import sys import getopt import subprocess @@ -22,6 +23,7 @@ import json import platform import socket import threading +import importlib import toml sys.path.append("../pytest") @@ -53,8 +55,39 @@ def checkRunTimeError(): if hwnd: os.system("TASKKILL /F /IM taosd.exe") +# +# run case on previous cluster +# +def runOnPreviousCluster(host, config, fileName): + print("enter run on previeous") + + # load case module + sep = "/" + if platform.system().lower() == 'windows': + sep = os.sep + moduleName = fileName.replace(".py", "").replace(sep, ".") + uModule = importlib.import_module(moduleName) + case = uModule.TDTestCase() + + # create conn + conn = taos.connect(host, config) + + # run case + case.init(conn, False) + try: + case.run() + except Exception as e: + tdLog.notice(repr(e)) + tdLog.exit("%s failed" % (fileName)) + # stop + case.stop() + + if __name__ == "__main__": + # + # analysis paramaters + # fileName = "all" deployPath = "" masterIp = "" @@ -75,8 +108,9 @@ if __name__ == "__main__": replicaVar = 1 asan = False independentMnode = True - opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:a', [ - 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode']) + previousCluster = False + opts, args = getopt.gnu_getopt(sys.argv[1:], 'f:p:m:l:scghrd:k:e:N:M:Q:C:RD:n:i:aP', [ + 'file=', 'path=', 'master', 'logSql', 'stop', 'cluster', 'valgrind', 'help', 'restart', 'updateCfgDict', 'killv', 'execCmd','dnodeNums','mnodeNums','queryPolicy','createDnodeNums','restful','adaptercfgupdate','replicaVar','independentMnode','previous']) for key, value in opts: if key in ['-h', '--help']: tdLog.printNoPrefix( @@ -101,6 +135,7 @@ if __name__ == "__main__": tdLog.printNoPrefix('-n the number of replicas') tdLog.printNoPrefix('-i independentMnode Mnode') tdLog.printNoPrefix('-a address sanitizer mode') + tdLog.printNoPrefix('-P run case with [P]revious cluster, do not create new cluster to run case.') sys.exit(0) @@ -182,6 +217,12 @@ if __name__ == "__main__": if key in ['-n', '--replicaVar']: replicaVar = value + if key in ['-P', '--previous']: + previousCluster = True + + # + # do exeCmd command + # if not execCmd == "": if restful: tAdapter.init(deployPath) @@ -191,6 +232,9 @@ if __name__ == "__main__": exec(execCmd) quit() + # + # do stop option + # if (stop != 0): if (valgrind == 0): toBeKilled = "taosd" @@ -248,6 +292,9 @@ if __name__ == "__main__": tdLog.info('stop All dnodes') + # + # get hostname + # if masterIp == "": host = socket.gethostname() else: @@ -256,8 +303,20 @@ if __name__ == "__main__": host = config["host"] except Exception as r: host = masterIp - tdLog.info("Procedures for tdengine deployed in %s" % (host)) + + # + # do previousCluster option + # + if previousCluster: + tdDnodes.init(deployPath, masterIp) + runOnPreviousCluster(host, tdDnodes.getSimCfgPath(), fileName) + tdLog.info("run on previous cluster end.") + quit() + + # + # windows run + # if platform.system().lower() == 'windows': fileName = fileName.replace("/", os.sep) if (masterIp == "" and not fileName == "0-others\\udf_create.py"): @@ -451,6 +510,7 @@ if __name__ == "__main__": tAdapter.stop(force_kill=True) if dnodeNums == 1 : + # dnode is one tdDnodes.deploy(1,updateCfgDict) tdDnodes.start(1) tdCases.logSql(logSql) @@ -491,6 +551,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") else : + # dnode > 1 cluster tdLog.debug("create an cluster with %s nodes and make %s dnode as independent mnode"%(dnodeNums,mnodeNums)) dnodeslist = cluster.configure_cluster(dnodeNums=dnodeNums, mnodeNums=mnodeNums, independentMnode=independentMnode) tdDnodes = ClusterDnodes(dnodeslist) @@ -509,6 +570,7 @@ if __name__ == "__main__": tAdapter.deploy(adapter_cfg_dict) tAdapter.start() + # create taos connect if not restful: conn = taos.connect(host,config=tdDnodes.getSimCfgPath()) else: @@ -527,6 +589,7 @@ if __name__ == "__main__": except Exception as r: print(r) + # do queryPolicy option if queryPolicy != 1: queryPolicy=int(queryPolicy) if restful: @@ -548,6 +611,7 @@ if __name__ == "__main__": tdLog.exit(f"alter queryPolicy to {queryPolicy} failed") + # run case if testCluster: tdLog.info("Procedures for testing cluster") if fileName == "all": @@ -566,6 +630,7 @@ if __name__ == "__main__": else: tdCases.runOneLinux(conn, fileName, replicaVar) + # do restart option if restart: if fileName == "all": tdLog.info("not need to query ") @@ -585,6 +650,7 @@ if __name__ == "__main__": else: tdLog.info("not need to query") + # close for end if conn is not None: conn.close() if asan: diff --git a/tools/shell/src/shellArguments.c b/tools/shell/src/shellArguments.c index 0f5585991e5efcf4d3c69b74a5b0b3e254a99e60..52cb524e3bfb98875e4f93a20bf21fa185674a29 100644 --- a/tools/shell/src/shellArguments.c +++ b/tools/shell/src/shellArguments.c @@ -20,18 +20,6 @@ #include "shellInt.h" #include "version.h" -#ifndef CUS_NAME -char cusName[] = "TDengine"; -#endif - -#ifndef CUS_PROMPT -char cusPrompt[] = "taos"; -#endif - -#ifndef CUS_EMAIL -char cusEmail[] = ""; -#endif - #if defined(CUS_NAME) || defined(CUS_PROMPT) || defined(CUS_EMAIL) #include "cus_name.h" #endif @@ -255,8 +243,8 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { SShellArgs *pArgs = &shell.args; for (int i = 1; i < argc; i++) { - if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 || strcmp(argv[i], "-?") == 0 || - strcmp(argv[i], "/?") == 0) { + if (strcmp(argv[i], "--help") == 0 || strcmp(argv[i], "--usage") == 0 + || strcmp(argv[i], "-?") == 0 || strcmp(argv[i], "/?") == 0) { shellParseSingleOpt('?', NULL); return 0; } @@ -272,8 +260,10 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { return -1; } - if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' || key[1] == 'a' || key[1] == 'c' || key[1] == 's' || - key[1] == 'f' || key[1] == 'd' || key[1] == 'w' || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' + if (key[1] == 'h' || key[1] == 'P' || key[1] == 'u' + || key[1] == 'a' || key[1] == 'c' || key[1] == 's' + || key[1] == 'f' || key[1] == 'd' || key[1] == 'w' + || key[1] == 'n' || key[1] == 'l' || key[1] == 'N' #ifdef WEBSOCKET || key[1] == 'E' || key[1] == 'T' #endif @@ -289,10 +279,12 @@ int32_t shellParseArgsWithoutArgp(int argc, char *argv[]) { } shellParseSingleOpt(key[1], val); i++; - } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' || key[1] == 'r' || key[1] == 'k' || key[1] == 't' || - key[1] == 'V' || key[1] == '?' || key[1] == 1 + } else if (key[1] == 'p' || key[1] == 'A' || key[1] == 'C' + || key[1] == 'r' || key[1] == 'k' + || key[1] == 't' || key[1] == 'V' + || key[1] == '?' || key[1] == 1 #ifdef WEBSOCKET - || key[1] == 'R' + ||key[1] == 'R' #endif ) { shellParseSingleOpt(key[1], NULL); diff --git a/tools/shell/src/shellEngine.c b/tools/shell/src/shellEngine.c index 70130397ba7d4b9539080bff1ad81c193e751a54..478801d3e7878996553c4ef2020d73810dc17088 100644 --- a/tools/shell/src/shellEngine.c +++ b/tools/shell/src/shellEngine.c @@ -348,14 +348,23 @@ void shellDumpFieldToFile(TdFilePtr pFile, const char *val, TAOS_FIELD *field, i taosFprintfFile(pFile, "%" PRIu64, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + if (tsEnableScience) { + taosFprintfFile(pFile, "%e", GET_FLOAT_VAL(val)); + } else { + taosFprintfFile(pFile, "%.5f", GET_FLOAT_VAL(val)); + } break; case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, length)) { - taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); - } else { + if (tsEnableScience) { + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9e", 23, GET_DOUBLE_VAL(val)); taosFprintfFile(pFile, "%s", buf); + } else { + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", length, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, length)) { + taosFprintfFile(pFile, "%*.15e", length, GET_DOUBLE_VAL(val)); + } else { + taosFprintfFile(pFile, "%s", buf); + } } break; case TSDB_DATA_TYPE_BINARY: @@ -543,14 +552,23 @@ void shellPrintField(const char *val, TAOS_FIELD *field, int32_t width, int32_t printf("%*" PRIu64, width, *((uint64_t *)val)); break; case TSDB_DATA_TYPE_FLOAT: - printf("%*ef", width, GET_FLOAT_VAL(val)); + if (tsEnableScience) { + printf("%*e", width, GET_FLOAT_VAL(val)); + } else { + printf("%*.5f", width, GET_FLOAT_VAL(val)); + } break; case TSDB_DATA_TYPE_DOUBLE: - n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); - if (n > TMAX(25, width)) { - printf("%*.15e", width, GET_DOUBLE_VAL(val)); + if (tsEnableScience) { + snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%.9e", GET_DOUBLE_VAL(val)); + printf("%*s", width, buf); } else { - printf("%s", buf); + n = snprintf(buf, TSDB_MAX_BYTES_PER_ROW, "%*.9f", width, GET_DOUBLE_VAL(val)); + if (n > TMAX(25, width)) { + printf("%*.15e", width, GET_DOUBLE_VAL(val)); + } else { + printf("%s", buf); + } } break; case TSDB_DATA_TYPE_BINARY: @@ -1097,6 +1115,7 @@ int32_t shellExecute() { } if (shell.conn == NULL) { + printf("failed to connect to server, reason: %s\n", taos_errstr(NULL)); fflush(stdout); return -1; } diff --git a/tools/shell/src/shellWebsocket.c b/tools/shell/src/shellWebsocket.c index 1d81ce4b2f8c47ef3f8e5344c42ae5afa3b96bcb..d8920cb4c3d79aacb441bf9a1f512f4a5a4f62b6 100644 --- a/tools/shell/src/shellWebsocket.c +++ b/tools/shell/src/shellWebsocket.c @@ -24,7 +24,7 @@ int shell_conn_ws_server(bool first) { ((dsnLen-SHELL_WS_DSN_MASK) > SHELL_WS_DSN_BUFF)? SHELL_WS_DSN_BUFF:(dsnLen-SHELL_WS_DSN_MASK), "%s", shell.args.dsn); - fprintf(stdout, "trying to connect %s*** ", cuttedDsn); + fprintf(stdout, "trying to connect %s****** ", cuttedDsn); fflush(stdout); for (int i = 0; i < shell.args.timeout; i++) { shell.ws_conn = ws_connect_with_dsn(shell.args.dsn); diff --git a/utils/test/c/CMakeLists.txt b/utils/test/c/CMakeLists.txt index eda9c70f159b9c440d530a8036ef01eeb409930c..87b0d11d1c00f631c7c09081952af82174e2432f 100644 --- a/utils/test/c/CMakeLists.txt +++ b/utils/test/c/CMakeLists.txt @@ -6,6 +6,14 @@ add_executable(tmq_taosx_ci tmq_taosx_ci.c) add_executable(write_raw_block_test write_raw_block_test.c) add_executable(sml_test sml_test.c) add_executable(get_db_name_test get_db_name_test.c) +add_executable(tmq_offset tmqOffset.c) +target_link_libraries( + tmq_offset + PUBLIC taos_static + PUBLIC util + PUBLIC common + PUBLIC os +) target_link_libraries( create_table PUBLIC taos_static diff --git a/utils/test/c/sml_test.c b/utils/test/c/sml_test.c index b0cc6f749cb703c601235a3aaf444ce169149057..755ab556251d5f42be27622354f4e950384259fa 100644 --- a/utils/test/c/sml_test.c +++ b/utils/test/c/sml_test.c @@ -773,26 +773,26 @@ int sml_dup_time_Test() { taos_free_result(pRes); const char *sql[] = {//"test_ms,t0=t c0=f 1626006833641", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", - "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." - "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " - "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." - "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=f,c1=1i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"xcxvwjvf\",c8=L\"ncharColValue\",c9=7u64 1626006833639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=T,c1=2i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"fixrzcuq\",c8=L\"ncharColValue\",c9=7u64 1626006834639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=3i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"iupzdqub\",c8=L\"ncharColValue\",c9=7u64 1626006835639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=4i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"yvvtzzof\",c8=L\"ncharColValue\",c9=7u64 1626006836639000000", + "ubzlsr,id=qmtcvgd,t0=t,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11." + "12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" " + "c0=t,c1=5i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22." + "123456789f64,c7=\"vbxpilkj\",c8=L\"ncharColValue\",c9=7u64 1626006837639000000"}; pRes = taos_query(taos, "use sml_db"); taos_free_result(pRes); @@ -942,6 +942,8 @@ int sml_ts2164_Test() { "ts3038,location=l2a,groupid=ca current=L\"11.8\"", "ts3038,location=l2a,groupid=ca voltage=L\"221\"", "ts3038,location=l2a,groupid=ca phase=L\"221\"", +// "qgyltizmkq,id=sub_table_0123456,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64", +// "qgyltizmkq,id=sub_table_0123456,t=3,t0=f,t1=127i8,t2=32767i16,t3=2147483647i32,t4=9223372036854775807i64,t5=11.12345f32,t6=22.123456789f64,t7=\"binaryTagValue\",t8=L\"ncharTagValue\" c0=f,c1=127i8,c2=32767i16,c3=2147483647i32,c4=9223372036854775807i64,c5=11.12345f32,c6=22.123456789f64,c7=\"binaryColValue\",c8=L\"ncharColValue\",c9=7u64" // "meters,location=la,groupid=cb current=11.8,voltage=221,phase=0.27", }; diff --git a/utils/test/c/tmqOffset.c b/utils/test/c/tmqOffset.c new file mode 100644 index 0000000000000000000000000000000000000000..7225cb87bdcbf77807c1861d22cef72f955d49bd --- /dev/null +++ b/utils/test/c/tmqOffset.c @@ -0,0 +1,64 @@ +// +// Created by mingming wanng on 2023/4/7. +// +#include +#include +#include "taoserror.h" +#include "tlog.h" +#include "tmsg.h" + +typedef struct { + int32_t size; +} STqOffsetHead; + +int32_t tqOffsetRestoreFromFile(const char* fname) { + TdFilePtr pFile = taosOpenFile(fname, TD_FILE_READ); + if (pFile != NULL) { + STqOffsetHead head = {0}; + int32_t code; + + while (1) { + if ((code = taosReadFile(pFile, &head, sizeof(STqOffsetHead))) != sizeof(STqOffsetHead)) { + if (code == 0) { + break; + } else { + printf("code:%d != 0\n", code); + return -1; + } + } + int32_t size = htonl(head.size); + void* memBuf = taosMemoryCalloc(1, size); + if (memBuf == NULL) { + printf("memBuf == NULL\n"); + return -1; + } + if ((code = taosReadFile(pFile, memBuf, size)) != size) { + taosMemoryFree(memBuf); + printf("code:%d != size:%d\n", code, size); + return -1; + } + STqOffset offset; + SDecoder decoder; + tDecoderInit(&decoder, memBuf, size); + if (tDecodeSTqOffset(&decoder, &offset) < 0) { + taosMemoryFree(memBuf); + tDecoderClear(&decoder); + printf("tDecodeSTqOffset error\n"); + return -1; + } + + tDecoderClear(&decoder); + printf("subkey:%s, type:%d, uid/version:%"PRId64", ts:%"PRId64"\n", + offset.subKey, offset.val.type, offset.val.uid, offset.val.ts); + taosMemoryFree(memBuf); + } + + taosCloseFile(&pFile); + } + return 0; +} + +int main(int argc, char *argv[]) { + tqOffsetRestoreFromFile("offset-ver0"); + return 0; +}